using System.Security.Cryptography; using System.Text; using dy.net.model.dto; using dy.net.model.entity; using dy.net.storage; using dy.net.utils; using Newtonsoft.Json; using SqlSugar; using MediaStorageType = dy.net.model.dto.StorageType; namespace dy.net.service { public class StorageMigrationService { private readonly ISqlSugarClient _db; private readonly DouyinCommonService _commonService; private readonly OpenListSettingsService _settingsService; private readonly OpenListMediaStorage _openList; public StorageMigrationService( ISqlSugarClient db, DouyinCommonService commonService, OpenListSettingsService settingsService, OpenListMediaStorage openList) { _db = db; _commonService = commonService; _settingsService = settingsService; _openList = openList; } public async Task PreflightAsync(bool verifyCapabilities, CancellationToken cancellationToken) { var result = new StorageMigrationPreflightResult(); if (_commonService.GetConfig()?.StorageType != MediaStorageType.OpenList) { result.Errors.Add("请先完成测试并启用 OpenList 原生存储;迁移不会自动切换存储模式。"); return result; } var settings = await _settingsService.GetAsync(); try { OpenListTransferService.ValidateSettings(settings); if (string.IsNullOrWhiteSpace(await _settingsService.GetPasswordAsync(settings))) result.Errors.Add("OpenList 密码无效,请重新输入并保存配置。"); else result.ConfigurationFingerprint = StorageConfigurationFingerprint.Create(settings); } catch (Exception ex) { result.Errors.Add(ex.GetBaseException().Message); } if (!settings.LastTestedAt.HasValue) result.Errors.Add("当前 OpenList 配置尚未通过服务端复制能力测试。"); if (verifyCapabilities) { var probe = await _openList.ProbeAsync(settings, cancellationToken); if (!probe.Success) result.Errors.Add("OpenList 能力测试失败:" + probe.Message); } if (result.Errors.Count > 0) return result; List planning; try { planning = await BuildPlanningAsync(); } catch (Exception ex) { result.Errors.Add("读取 OpenList 目标失败:" + ex.GetBaseException().Message); return result; } result.RecordCount = planning.Count; result.LocalRecordCount = planning.Count(x => x.Video.StorageType == MediaStorageType.Local); result.LegacyWebDavRecordCount = planning.Count(x => x.Video.StorageType == MediaStorageType.WebDav); result.AdoptableCount = planning.Count(x => x.AdoptExistingTarget); result.TransferRequiredCount = planning.Count(x => !x.AdoptExistingTarget); result.ReadableFileCount = planning.Count(x => x.LocalReadable); result.MissingFileCount = planning.Count(x => !x.AdoptExistingTarget && !x.LocalReadable); result.TotalBytes = planning.Where(x => x.LocalReadable).Sum(x => x.LocalLength); result.InvalidCookieCount = planning.Where(x => x.Cookie == null || x.Cookie.StatusCode != 0 || string.IsNullOrWhiteSpace(x.Cookie.Cookies)) .Select(x => x.Video.CookieId).Distinct().Count(); result.MissingTargetPathCount = planning.Count(x => x.Plan == null); result.ConflictCount = planning.Where(x => x.Plan != null) .GroupBy(x => x.Plan.VideoPath, StringComparer.Ordinal) .Where(x => x.Count() > 1).Sum(x => x.Count()); foreach (var error in planning.Select(x => x.PlanError).Where(x => !string.IsNullOrWhiteSpace(x)).Distinct().Take(20)) result.Errors.Add(error); if (result.ConflictCount > 0) result.Errors.Add($"发现 {result.ConflictCount} 条记录映射到重复目标路径,必须先调整目录或命名配置。"); if (result.MissingFileCount > 0) result.Warnings.Add($"{result.MissingFileCount} 条记录的本地主媒体缺失或不可安全读取,执行时会先尝试记录 URL,再扫描对应抖音全量列表回源。"); if (result.InvalidCookieCount > 0) result.Warnings.Add($"{result.InvalidCookieCount} 个账号 Cookie 无效;这些账号下缺失本地文件的作品可能无法回源。"); result.Warnings.Add($"OpenList 无法统一报告所有后端剩余容量;容量未知,预计需读取本地数据 {FormatBytes(result.TotalBytes)}。"); result.CanStart = result.RecordCount > 0 && result.Errors.Count == 0; if (result.RecordCount == 0) result.Errors.Add("没有需要接管到 OpenList 的旧存储视频记录。"); return result; } public async Task CreateAsync(CreateStorageMigrationRequest request, CancellationToken cancellationToken) { request ??= new CreateStorageMigrationRequest(); if (request.Concurrency is < 1 or > 3) throw new InvalidOperationException("迁移并发只能选择 1–3。"); var preflight = await PreflightAsync(false, cancellationToken); if (!preflight.CanStart) throw new InvalidOperationException(string.Join(";", preflight.Errors)); if (string.IsNullOrWhiteSpace(request.ConfigurationFingerprint) || !string.Equals(request.ConfigurationFingerprint, preflight.ConfigurationFingerprint, StringComparison.Ordinal)) throw new InvalidOperationException("OpenList 配置已变化,请重新执行迁移预检。"); var active = await _db.Queryable().Where(x => x.Status == StorageMigrationTaskStatus.Queued || x.Status == StorageMigrationTaskStatus.Running || x.Status == StorageMigrationTaskStatus.Paused || x.Status == StorageMigrationTaskStatus.Cleaning).AnyAsync(); if (active) throw new InvalidOperationException("已有未结束的存储迁移任务,请先处理当前任务。"); var planning = await BuildPlanningAsync(); if (planning.Any(x => x.Plan == null || !string.IsNullOrWhiteSpace(x.PlanError))) throw new InvalidOperationException("迁移计划在创建前发生变化,请重新执行预检。"); var now = DateTime.Now; var task = new StorageMigrationTask { Id = Guid.NewGuid().ToString("N"), Status = StorageMigrationTaskStatus.Queued, TargetStorageType = MediaStorageType.OpenList, Concurrency = request.Concurrency, TotalCount = planning.Count, PendingCount = planning.Count, TotalBytes = planning.Where(x => x.LocalReadable).Sum(x => x.LocalLength), ConfigurationFingerprint = preflight.ConfigurationFingerprint, CreatedAt = now, UpdatedAt = now }; var items = planning.Select(entry => new StorageMigrationItem { Id = Guid.NewGuid().ToString("N"), TaskId = task.Id, VideoId = entry.Video.Id, SourceStorageType = entry.Video.StorageType, AdoptExistingTarget = entry.AdoptExistingTarget, Stage = StorageMigrationItemStage.Pending, ExpectedLength = entry.LocalLength > 0 ? entry.LocalLength : entry.Video.FileSize, OldVideoPath = entry.Video.VideoSavePath, TargetVideoPath = entry.Plan.VideoPath, TargetCoverPath = entry.Plan.CoverPath, OldSnapshotJson = JsonConvert.SerializeObject(entry.Video), AttachmentPlanJson = JsonConvert.SerializeObject(BuildAttachments(entry)), UploadedPathsJson = "[]", OwnedRemotePathsJson = "[]", CreatedAt = now, UpdatedAt = now }).ToList(); var transaction = await _db.Ado.UseTranAsync(async () => { await _db.Insertable(task).ExecuteCommandAsync(); await _db.Insertable(items).ExecuteCommandAsync(); }); if (!transaction.IsSuccess) throw new InvalidOperationException("创建迁移任务失败:" + transaction.ErrorMessage); return task; } public async Task GetAsync(string id) { var task = await _db.Queryable().InSingleAsync(id) ?? throw new KeyNotFoundException("迁移任务不存在"); return ToDetail(task); } public async Task GetLatestAsync() { var task = await _db.Queryable().Where(x => !x.IsArchived) .OrderByDescending(x => x.CreatedAt).FirstAsync(); return task == null ? null : ToDetail(task); } public async Task ArchiveAsync(string id) { var task = await RequireTaskAsync(id); if (!CanArchive(task)) throw new InvalidOperationException("该迁移仍有成功项等待清理或回滚,请先处理旧文件后再归档。"); task.IsArchived = true; task.UpdatedAt = DateTime.Now; if (await _db.Updateable(task).ExecuteCommandAsync() != 1) throw new InvalidOperationException("归档迁移历史失败。"); } public async Task ArchiveAllSettledAsync() { var tasks = await _db.Queryable().Where(x => !x.IsArchived).ToListAsync(); var archivable = tasks.Where(CanArchive).ToList(); var requiresCleanup = tasks.Count(x => IsTerminal(x.Status) && !CanArchive(x)); if (archivable.Count > 0) { var now = DateTime.Now; foreach (var task in archivable) { task.IsArchived = true; task.UpdatedAt = now; } var changed = await _db.Updateable(archivable).ExecuteCommandAsync(); if (changed != archivable.Count) throw new InvalidOperationException("部分迁移历史归档失败。"); } return new StorageMigrationArchiveResult { ArchivedCount = archivable.Count, RequiresCleanupCount = requiresCleanup, Message = requiresCleanup > 0 ? $"已隐藏 {archivable.Count} 条迁移历史;另有 {requiresCleanup} 条包含已迁移视频,请先清理旧文件或回滚。" : $"已隐藏 {archivable.Count} 条已结束的迁移历史。" }; } public async Task GetItemsAsync(string taskId, StorageMigrationItemPageRequest request) { request ??= new StorageMigrationItemPageRequest(); request.PageIndex = Math.Max(1, request.PageIndex); request.PageSize = Math.Clamp(request.PageSize, 1, 100); var query = _db.Queryable().Where(x => x.TaskId == taskId) .WhereIF(request.Stage.HasValue, x => x.Stage == request.Stage.Value); return new StorageMigrationItemPage { TotalCount = await query.CountAsync(), Items = await query.OrderBy(x => x.CreatedAt).Skip((request.PageIndex - 1) * request.PageSize).Take(request.PageSize).ToListAsync() }; } public async Task PreviewFailedRecordRemovalAsync() { var plan = await BuildFailedRecordRemovalPlanAsync(); var preview = new FailedMigrationRecordPreview { ActiveMigrationCount = plan.ActiveMigrationCount, FailedItemCount = plan.Groups.Sum(x => x.Items.Count), DistinctVideoCount = plan.Groups.Count, EligibleRecordCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.Eligible), AlreadyMissingCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.AlreadyMissing), ChangedRecordCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.Changed), PermanentlyExcludedCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.PermanentlyExcluded), InvalidSnapshotCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.InvalidSnapshot) }; preview.CanExecute = preview.ActiveMigrationCount == 0 && preview.EligibleRecordCount + preview.AlreadyMissingCount > 0; preview.ConfirmationToken = preview.CanExecute ? plan.ConfirmationToken : null; if (preview.ActiveMigrationCount > 0) preview.Errors.Add("存在排队、运行、暂停或清理中的存储迁移任务,请先结束当前迁移再处理历史失败记录。"); else if (preview.FailedItemCount == 0) preview.Errors.Add("当前没有可处理的历史迁移失败项。"); else if (!preview.CanExecute) preview.Errors.Add("没有可安全删除的视频记录;已变化、已迁移、永久排除或快照损坏的记录不会被修改。"); preview.Warnings.Add("此操作只删除数据库视频记录,不删除旧本地文件或 OpenList 文件,也不会自动启动同步。"); if (preview.ChangedRecordCount > 0) preview.Warnings.Add($"{preview.ChangedRecordCount} 条视频记录已变化、已迁移或已重新创建,将安全跳过。"); if (preview.PermanentlyExcludedCount > 0) preview.Warnings.Add($"{preview.PermanentlyExcludedCount} 条视频已被永久排除,请先在任务中心取消排除后再处理。"); if (preview.InvalidSnapshotCount > 0) preview.Warnings.Add($"{preview.InvalidSnapshotCount} 条迁移快照无效,将保留原记录和失败状态。"); return preview; } public async Task RemoveFailedRecordsAsync(RemoveFailedMigrationRecordsRequest request) { if (string.IsNullOrWhiteSpace(request?.ConfirmationToken)) throw new InvalidOperationException("缺少删除确认信息,请重新打开预览后确认。"); RemoveFailedMigrationRecordsResult result = null; var transaction = await _db.Ado.UseTranAsync(async () => { var plan = await BuildFailedRecordRemovalPlanAsync(); if (plan.ActiveMigrationCount > 0) throw new InvalidOperationException("存在未结束的存储迁移任务,请先结束当前迁移。"); if (!string.Equals(plan.ConfirmationToken, request.ConfirmationToken, StringComparison.Ordinal)) throw new InvalidOperationException("迁移失败记录已发生变化,请重新预览并确认。"); var actionable = plan.Groups.Where(x => x.Disposition is FailedRecordDisposition.Eligible or FailedRecordDisposition.AlreadyMissing).ToList(); if (actionable.Count == 0) throw new InvalidOperationException("没有可安全删除的视频记录。"); var now = DateTime.Now; var changedItems = new List(); var deletedRecords = 0; foreach (var group in actionable) { if (group.Disposition == FailedRecordDisposition.Eligible) { var current = group.Current; var expectedStorageType = current.StorageType; var deleted = await _db.Deleteable().Where(x => x.Id == current.Id && x.AwemeId == current.AwemeId && x.StorageType == expectedStorageType && x.VideoSavePath == current.VideoSavePath).ExecuteCommandAsync(); if (deleted != 1) throw new InvalidOperationException($"视频 {current.AwemeId ?? current.Id} 已发生变化,请重新预览。"); deletedRecords++; } foreach (var item in group.Items) { item.Stage = StorageMigrationItemStage.RecordRemoved; item.ErrorMessage = null; item.WarningMessage = group.Disposition == FailedRecordDisposition.Eligible ? "视频记录已删除;旧文件和 OpenList 文件均已保留,等待后续正常同步重新发现。" : "原视频记录已不存在;迁移失败项已归档,等待后续正常同步重新发现。"; item.UpdatedAt = now; item.CompletedAt = now; changedItems.Add(item); } } if (changedItems.Count > 0) { var changed = await _db.Updateable(changedItems).ExecuteCommandAsync(); if (changed != changedItems.Count) throw new InvalidOperationException("更新迁移失败项状态不完整,操作已回滚。"); } var affectedTaskIds = changedItems.Select(x => x.TaskId).Distinct().ToList(); foreach (var taskId in affectedTaskIds) await RefreshCountsAsync(taskId); result = new RemoveFailedMigrationRecordsResult { DeletedRecordCount = deletedRecords, AlreadyMissingCount = actionable.Count(x => x.Disposition == FailedRecordDisposition.AlreadyMissing), RemovedItemCount = changedItems.Count, AffectedTaskCount = affectedTaskIds.Count, SkippedChangedCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.Changed), SkippedExcludedCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.PermanentlyExcluded), InvalidSnapshotCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.InvalidSnapshot), Message = $"已删除 {deletedRecords} 条视频记录,更新 {changedItems.Count} 条迁移失败项;旧文件已保留,将随系统后续正常同步重新发现。" }; }); if (!transaction.IsSuccess) throw transaction.ErrorException ?? new InvalidOperationException("删除迁移失败记录失败,数据库操作已回滚。"); return result; } public Task PauseAsync(string id) => ChangeStatusAsync(id, new[] { StorageMigrationTaskStatus.Queued, StorageMigrationTaskStatus.Running }, StorageMigrationTaskStatus.Paused, null); public async Task ResumeAsync(string id) { var task = await RequireTaskAsync(id); if (task.Status != StorageMigrationTaskStatus.Paused) throw new InvalidOperationException("只有暂停中的任务可以恢复。"); var fingerprint = StorageConfigurationFingerprint.Create(await _settingsService.GetAsync()); if (!string.Equals(fingerprint, task.ConfigurationFingerprint, StringComparison.Ordinal)) throw new InvalidOperationException("OpenList 地址、源挂载、基础目录或账号已变化,请重新预检并创建新任务。"); await ChangeStatusAsync(id, new[] { StorageMigrationTaskStatus.Paused }, StorageMigrationTaskStatus.Queued, null); } public Task CancelAsync(string id) => ChangeStatusAsync(id, new[] { StorageMigrationTaskStatus.Queued, StorageMigrationTaskStatus.Running, StorageMigrationTaskStatus.Paused }, StorageMigrationTaskStatus.Cancelled, null); public async Task RetryFailedAsync(string id) { var task = await RequireTaskAsync(id); if (task.Status is not (StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled or StorageMigrationTaskStatus.Paused)) throw new InvalidOperationException("当前任务没有可重试的失败项。"); var fingerprint = StorageConfigurationFingerprint.Create(await _settingsService.GetAsync()); if (fingerprint != task.ConfigurationFingerprint) throw new InvalidOperationException("OpenList 配置已变化,请重新预检。"); await _db.Updateable() .SetColumns(x => new StorageMigrationItem { Stage = StorageMigrationItemStage.Pending, ErrorMessage = null, UpdatedAt = DateTime.Now }) .Where(x => x.TaskId == id && x.Stage == StorageMigrationItemStage.Failed).ExecuteCommandAsync(); await _db.Updateable() .SetColumns(x => new StorageMigrationTask { Status = StorageMigrationTaskStatus.Queued, CompletedAt = null, ErrorMessage = null, UpdatedAt = DateTime.Now }) .Where(x => x.Id == id).ExecuteCommandAsync(); await RefreshCountsAsync(id); } public async Task CleanupAsync(string id, CancellationToken cancellationToken) { var task = await RequireTaskAsync(id); if (task.Status is not (StorageMigrationTaskStatus.Completed or StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled)) throw new InvalidOperationException("只有已结束的任务可以清理旧文件。"); var targetStorage = await BindTaskTargetAsync(task); await _db.Updateable() .SetColumns(x => new StorageMigrationTask { Status = StorageMigrationTaskStatus.Cleaning, ErrorMessage = null, UpdatedAt = DateTime.Now }) .Where(x => x.Id == id).ExecuteCommandAsync(); var items = await _db.Queryable().Where(x => x.TaskId == id && (x.Stage == StorageMigrationItemStage.Succeeded || x.Stage == StorageMigrationItemStage.SucceededWithWarnings)).ToListAsync(); var errors = new List(); foreach (var item in items) { try { await CleanupItemAsync(item, targetStorage, cancellationToken); } catch (Exception ex) { errors.Add($"{item.VideoId}: {ex.Message}"); item.ErrorMessage = "清理失败:" + ex.Message; item.UpdatedAt = DateTime.Now; await _db.Updateable(item).ExecuteCommandAsync(); } } var remaining = await _db.Queryable().Where(x => x.TaskId == id && (x.Stage == StorageMigrationItemStage.Succeeded || x.Stage == StorageMigrationItemStage.SucceededWithWarnings)).CountAsync(); await _db.Updateable() .SetColumns(x => new StorageMigrationTask { Status = remaining == 0 ? StorageMigrationTaskStatus.Cleaned : StorageMigrationTaskStatus.PartiallyFailed, CleanedAt = remaining == 0 ? DateTime.Now : null, ErrorMessage = errors.Count == 0 ? null : string.Join(";", errors.Take(20)), UpdatedAt = DateTime.Now }).Where(x => x.Id == id).ExecuteCommandAsync(); await RefreshCountsAsync(id); } public async Task RollbackAsync(string id, CancellationToken cancellationToken) { var task = await RequireTaskAsync(id); if (task.Status is not (StorageMigrationTaskStatus.Completed or StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled)) throw new InvalidOperationException("当前任务状态不允许回滚。"); if (task.CleanedAt.HasValue || await _db.Queryable().Where(x => x.TaskId == id && x.Stage == StorageMigrationItemStage.Cleaned).AnyAsync()) throw new InvalidOperationException("旧文件清理已开始,不能再执行回滚。"); var targetStorage = await BindTaskTargetAsync(task); var items = await _db.Queryable().Where(x => x.TaskId == id && (x.Stage == StorageMigrationItemStage.Succeeded || x.Stage == StorageMigrationItemStage.SucceededWithWarnings)).ToListAsync(); var snapshots = items.ToDictionary(x => x.Id, x => JsonConvert.DeserializeObject(x.OldSnapshotJson)); foreach (var pair in snapshots) { var snapshot = pair.Value ?? throw new InvalidOperationException("旧视频快照损坏,无法回滚。"); var item = items.First(x => x.Id == pair.Key); var sourceType = item.SourceStorageType ?? snapshot.StorageType; if (sourceType == MediaStorageType.Local) { var cookie = await _db.Queryable().InSingleAsync(snapshot.CookieId); if (!SafeLocalMigrationFile.TryResolve(snapshot.VideoSavePath, StorageMigrationPathPolicy.GetLocalRoots(cookie), out var local, out var error) || !File.Exists(local) || new FileInfo(local).Length <= 0) throw new InvalidOperationException($"视频 {snapshot.AwemeId} 的旧主媒体不可用:{error ?? "文件不存在或为空"}"); } else if (sourceType == MediaStorageType.WebDav) { var length = await targetStorage.GetLengthAsync(snapshot.VideoSavePath, cancellationToken); if (!length.HasValue || length <= 0 || snapshot.FileSize > 0 && length.Value != snapshot.FileSize) throw new InvalidOperationException($"视频 {snapshot.AwemeId} 的原 WebDAV 文件已不可用,无法恢复旧记录。"); } else { throw new InvalidOperationException($"视频 {snapshot.AwemeId} 的旧存储类型不支持回滚。"); } } var transaction = await _db.Ado.UseTranAsync(async () => { foreach (var item in items) { var snapshot = snapshots[item.Id]; var current = await _db.Queryable().InSingleAsync(item.VideoId) ?? throw new InvalidOperationException($"视频记录 {item.VideoId} 已不存在"); if (current.StorageType != MediaStorageType.OpenList || current.VideoSavePath != item.TargetVideoPath) throw new InvalidOperationException($"视频 {snapshot.AwemeId} 已被其他操作修改,无法回滚"); if (await _db.Updateable(snapshot).ExecuteCommandAsync() != 1) throw new InvalidOperationException($"恢复视频 {snapshot.AwemeId} 失败"); item.Stage = StorageMigrationItemStage.RolledBack; item.UpdatedAt = DateTime.Now; item.CompletedAt = DateTime.Now; await _db.Updateable(item).ExecuteCommandAsync(); } }); if (!transaction.IsSuccess) throw transaction.ErrorException ?? new InvalidOperationException("回滚数据库事务失败"); var warnings = new List(); foreach (var item in items) { var snapshot = snapshots[item.Id]; if ((item.SourceStorageType ?? snapshot.StorageType) == MediaStorageType.WebDav) continue; foreach (var remotePath in DeserializePaths(item.OwnedRemotePathsJson)) { try { if (!await IsRemotePathReferencedAsync(remotePath, cancellationToken)) await targetStorage.DeleteAsync(remotePath, cancellationToken); } catch (Exception ex) { warnings.Add($"{remotePath}: {ex.Message}"); } } } await _db.Updateable() .SetColumns(x => new StorageMigrationTask { Status = StorageMigrationTaskStatus.RolledBack, RolledBackAt = DateTime.Now, ErrorMessage = warnings.Count == 0 ? null : "远端独占文件清理警告:" + string.Join(";", warnings.Take(20)), UpdatedAt = DateTime.Now }).Where(x => x.Id == id).ExecuteCommandAsync(); await RefreshCountsAsync(id); } private async Task CleanupItemAsync(StorageMigrationItem item, IMediaStorage targetStorage, CancellationToken cancellationToken) { var current = await _db.Queryable().InSingleAsync(item.VideoId) ?? throw new InvalidOperationException("视频记录不存在"); if (current.StorageType != MediaStorageType.OpenList || current.VideoSavePath != item.TargetVideoPath) throw new InvalidOperationException("数据库已不再指向本任务的 OpenList 目标"); var remoteLength = await targetStorage.GetLengthAsync(current.VideoSavePath, cancellationToken); if (!remoteLength.HasValue || remoteLength.Value != item.ExpectedLength) throw new InvalidOperationException("远端主媒体不存在或长度已变化"); var snapshot = JsonConvert.DeserializeObject(item.OldSnapshotJson) ?? throw new InvalidOperationException("旧视频快照损坏"); if ((item.SourceStorageType ?? snapshot.StorageType) == MediaStorageType.WebDav) { item.Stage = StorageMigrationItemStage.Cleaned; item.ErrorMessage = null; item.WarningMessage = string.IsNullOrWhiteSpace(item.WarningMessage) ? "原 WebDAV 记录与 OpenList 指向同一文件,无需删除媒体。" : item.WarningMessage + ";原 WebDAV 记录与 OpenList 指向同一文件,无需删除媒体。"; item.UpdatedAt = DateTime.Now; item.CompletedAt = DateTime.Now; await _db.Updateable(item).ExecuteCommandAsync(); return; } var cookie = await _db.Queryable().InSingleAsync(snapshot.CookieId); var roots = StorageMigrationPathPolicy.GetLocalRoots(cookie); var candidates = BuildLocalCleanupCandidates(snapshot); var localVideos = await _db.Queryable().Where(x => x.StorageType == MediaStorageType.Local).ToListAsync(); foreach (var candidate in candidates) { if (string.IsNullOrWhiteSpace(candidate.Path) || !File.Exists(candidate.Path)) continue; if (!SafeLocalMigrationFile.TryResolve(candidate.Path, roots, out var safePath, out var error)) throw new InvalidOperationException($"拒绝清理不安全路径 {candidate.Path}:{error}"); if (IsLocalPathReferenced(candidate.Path, localVideos, snapshot.Id, candidate.Shared)) continue; File.Delete(safePath); } DeleteEmptyParents(snapshot.VideoSavePath, roots); item.Stage = StorageMigrationItemStage.Cleaned; item.ErrorMessage = null; item.UpdatedAt = DateTime.Now; item.CompletedAt = DateTime.Now; await _db.Updateable(item).ExecuteCommandAsync(); } private static List BuildLocalCleanupCandidates(DouyinVideo snapshot) { var result = new List { new(snapshot.VideoSavePath, false), new(snapshot.VideoCoverSavePath, snapshot.ViedoType is VideoTypeEnum.dy_mix or VideoTypeEnum.dy_series), new(snapshot.AuthorAvatar, true) }; if (!string.IsNullOrWhiteSpace(snapshot.VideoSavePath)) { var directory = Path.GetDirectoryName(snapshot.VideoSavePath); var basename = Path.GetFileNameWithoutExtension(snapshot.VideoSavePath); result.Add(new CleanupCandidate(Path.Combine(directory ?? string.Empty, basename + ".nfo"), false)); if (snapshot.ViedoType is VideoTypeEnum.dy_mix or VideoTypeEnum.dy_series) result.Add(new CleanupCandidate(Path.Combine(directory ?? string.Empty, "tvshow.nfo"), true)); } if (!string.IsNullOrWhiteSpace(snapshot.DynamicVideos)) { try { foreach (var attachment in JsonConvert.DeserializeObject>(snapshot.DynamicVideos) ?? new()) result.Add(new CleanupCandidate(attachment.Path, false)); } catch (JsonException) { } } return result.Where(x => !string.IsNullOrWhiteSpace(x.Path)).DistinctBy(x => x.Path).ToList(); } private static bool IsLocalPathReferenced(string path, IEnumerable localVideos, string excludedVideoId, bool shared) { foreach (var video in localVideos.Where(x => x.Id != excludedVideoId)) { if (path == video.VideoSavePath || path == video.VideoCoverSavePath || path == video.AuthorAvatar) return true; if (shared && !string.IsNullOrWhiteSpace(video.VideoSavePath) && string.Equals(Path.GetDirectoryName(video.VideoSavePath), Path.GetDirectoryName(path), StringComparison.Ordinal)) return true; if (!string.IsNullOrWhiteSpace(video.DynamicVideos) && video.DynamicVideos.Contains(path, StringComparison.Ordinal)) return true; } return false; } private async Task IsRemotePathReferencedAsync(string path, CancellationToken cancellationToken) { var videos = await _db.Queryable().Where(x => x.StorageType.IsRemote()).ToListAsync(); foreach (var video in videos) { if (video.VideoSavePath == path || video.VideoCoverSavePath == path || video.AuthorAvatar == path) return true; if (!string.IsNullOrWhiteSpace(video.DynamicVideos) && video.DynamicVideos.Contains(path, StringComparison.Ordinal)) return true; if (NfoContentBuilder.Build(video).Keys.Contains(path)) return true; if (Path.GetFileName(path).Equals("tvshow.nfo", StringComparison.OrdinalIgnoreCase) && StoragePath.DirectoryName(video.VideoSavePath) == StoragePath.DirectoryName(path)) return true; } return false; } private static void DeleteEmptyParents(string filePath, IReadOnlyList roots) { if (string.IsNullOrWhiteSpace(filePath)) return; var directory = Path.GetDirectoryName(Path.GetFullPath(filePath)); var root = roots.Where(x => SafeLocalMigrationFile.IsWithin(filePath, x)).OrderByDescending(x => x.Length).FirstOrDefault(); while (!string.IsNullOrWhiteSpace(directory) && !string.Equals(directory, root, StringComparison.Ordinal) && Directory.Exists(directory) && !Directory.EnumerateFileSystemEntries(directory).Any()) { Directory.Delete(directory, false); directory = Path.GetDirectoryName(directory); } } private static IReadOnlyList DeserializePaths(string json) { try { return JsonConvert.DeserializeObject>(json) ?? new(); } catch { return Array.Empty(); } } public async Task RefreshCountsAsync(string taskId) { var items = await _db.Queryable().Where(x => x.TaskId == taskId).ToListAsync(); var pending = items.Count(i => i.Stage == StorageMigrationItemStage.Pending || i.Stage == StorageMigrationItemStage.Uploading || i.Stage == StorageMigrationItemStage.Verifying || i.Stage == StorageMigrationItemStage.Committing); var success = items.Count(i => i.Stage == StorageMigrationItemStage.Succeeded || i.Stage == StorageMigrationItemStage.SucceededWithWarnings || i.Stage == StorageMigrationItemStage.Cleaned); var warning = items.Count(i => i.Stage == StorageMigrationItemStage.SucceededWithWarnings); var failed = items.Count(i => i.Stage == StorageMigrationItemStage.Failed); var removed = items.Count(i => i.Stage == StorageMigrationItemStage.RecordRemoved); var now = DateTime.Now; await _db.Updateable() .SetColumns(x => new StorageMigrationTask { PendingCount = pending, SuccessCount = success, WarningCount = warning, FailedCount = failed, RemovedCount = removed, UpdatedAt = now }).Where(x => x.Id == taskId).ExecuteCommandAsync(); } private async Task BuildFailedRecordRemovalPlanAsync() { var activeMigrationCount = await _db.Queryable().Where(x => x.Status == StorageMigrationTaskStatus.Queued || x.Status == StorageMigrationTaskStatus.Running || x.Status == StorageMigrationTaskStatus.Paused || x.Status == StorageMigrationTaskStatus.Cleaning).CountAsync(); var terminalTaskIds = await _db.Queryable().Where(x => x.Status == StorageMigrationTaskStatus.Completed || x.Status == StorageMigrationTaskStatus.PartiallyFailed || x.Status == StorageMigrationTaskStatus.Cancelled || x.Status == StorageMigrationTaskStatus.Cleaned || x.Status == StorageMigrationTaskStatus.RolledBack).Select(x => x.Id).ToListAsync(); var failedItems = new List(); foreach (var taskIdBatch in terminalTaskIds.Chunk(500)) failedItems.AddRange(await _db.Queryable().Where(x => taskIdBatch.Contains(x.TaskId) && x.Stage == StorageMigrationItemStage.Failed).ToListAsync()); var groups = failedItems.GroupBy(x => x.VideoId).OrderBy(x => x.Key, StringComparer.Ordinal) .Select(x => new FailedRecordGroup { VideoId = x.Key, Items = x.OrderBy(i => i.Id, StringComparer.Ordinal).ToList() }) .ToList(); foreach (var group in groups) { foreach (var item in group.Items) { try { var snapshot = JsonConvert.DeserializeObject(item.OldSnapshotJson); if (snapshot != null) group.Snapshots.Add(snapshot); } catch (JsonException) { } } } var videoIds = groups.Select(x => x.VideoId).Where(x => !string.IsNullOrWhiteSpace(x)).Distinct().ToList(); var currentVideosById = new List(); foreach (var videoIdBatch in videoIds.Chunk(500)) currentVideosById.AddRange(await _db.Queryable().Where(x => videoIdBatch.Contains(x.Id)).ToListAsync()); var currentById = currentVideosById.ToDictionary(x => x.Id, StringComparer.Ordinal); var snapshotAwemeIds = groups.SelectMany(x => x.Snapshots).Select(x => x.AwemeId) .Where(x => !string.IsNullOrWhiteSpace(x)).Distinct().ToList(); var currentVideosByAweme = new List(); var excludedAwemeIds = new HashSet(StringComparer.Ordinal); foreach (var awemeIdBatch in snapshotAwemeIds.Chunk(500)) { currentVideosByAweme.AddRange(await _db.Queryable() .Where(x => awemeIdBatch.Contains(x.AwemeId)).ToListAsync()); excludedAwemeIds.UnionWith(await _db.Queryable() .Where(x => awemeIdBatch.Contains(x.ViedoId)).Select(x => x.ViedoId).ToListAsync()); } var currentByAweme = currentVideosByAweme.GroupBy(x => x.AwemeId) .ToDictionary(x => x.Key, x => x.ToList(), StringComparer.Ordinal); foreach (var group in groups) { var awemeIds = group.Snapshots.Select(x => x.AwemeId).Where(x => !string.IsNullOrWhiteSpace(x)) .Distinct(StringComparer.Ordinal).ToList(); if (group.Snapshots.Count == 0 || awemeIds.Count != 1 || group.Snapshots.All(x => !string.Equals(x.Id, group.VideoId, StringComparison.Ordinal))) { group.Disposition = FailedRecordDisposition.InvalidSnapshot; continue; } group.AwemeId = awemeIds[0]; if (excludedAwemeIds.Contains(group.AwemeId)) { group.Disposition = FailedRecordDisposition.PermanentlyExcluded; continue; } if (!currentById.TryGetValue(group.VideoId, out var current)) { group.Disposition = currentByAweme.TryGetValue(group.AwemeId, out var replacements) && replacements.Any(x => x.Id != group.VideoId) ? FailedRecordDisposition.Changed : FailedRecordDisposition.AlreadyMissing; continue; } group.Current = current; var matchesSnapshot = group.Snapshots.Any(snapshot => current.StorageType == snapshot.StorageType && string.Equals(current.AwemeId, group.AwemeId, StringComparison.Ordinal) && string.Equals(snapshot.Id, current.Id, StringComparison.Ordinal) && string.Equals(snapshot.AwemeId, current.AwemeId, StringComparison.Ordinal) && string.Equals(snapshot.VideoSavePath, current.VideoSavePath, StringComparison.Ordinal)); group.Disposition = matchesSnapshot ? FailedRecordDisposition.Eligible : FailedRecordDisposition.Changed; } var tokenSource = string.Join("\n", groups.SelectMany(group => group.Items.Select(item => $"{item.Id}|{item.TaskId}|{item.VideoId}|{item.UpdatedAt.Ticks}|{(int)group.Disposition}|{group.AwemeId}|{group.Current?.Id}|{(int?)(group.Current?.StorageType)}|{group.Current?.VideoSavePath}"))); var token = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(tokenSource))).ToLowerInvariant(); return new FailedRecordRemovalPlan { ActiveMigrationCount = activeMigrationCount, Groups = groups, ConfirmationToken = token }; } private async Task ChangeStatusAsync(string id, StorageMigrationTaskStatus[] allowed, StorageMigrationTaskStatus status, string error) { var task = await RequireTaskAsync(id); if (!allowed.Contains(task.Status)) throw new InvalidOperationException($"任务状态 {task.Status} 不允许此操作。"); await _db.Updateable() .SetColumns(x => new StorageMigrationTask { Status = status, ErrorMessage = error, UpdatedAt = DateTime.Now }) .Where(x => x.Id == id).ExecuteCommandAsync(); } private async Task RequireTaskAsync(string id) => await _db.Queryable().InSingleAsync(id) ?? throw new KeyNotFoundException("迁移任务不存在"); private async Task BindTaskTargetAsync(StorageMigrationTask task) { if ((task.TargetStorageType ?? MediaStorageType.WebDav) != MediaStorageType.OpenList) throw new InvalidOperationException("旧 WebDAV 迁移任务只保留审计信息,不能再执行写入、清理或回滚。"); var settings = await _settingsService.GetAsync(); if (!string.Equals(StorageConfigurationFingerprint.Create(settings), task.ConfigurationFingerprint, StringComparison.Ordinal)) throw new InvalidOperationException("OpenList 地址、源挂载、基础目录或账号已变化,请恢复任务原目标后再操作。"); return _openList.Bind(settings); } private async Task> BuildPlanningAsync() { var videos = await _db.Queryable() .Where(x => x.StorageType != MediaStorageType.OpenList) .OrderBy(x => x.CreateTime).ToListAsync(); var cookies = (await _db.Queryable().ToListAsync()).ToDictionary(x => x.Id ?? string.Empty, StringComparer.Ordinal); var categories = (await _db.Queryable().ToListAsync()).ToDictionary(x => x.Id ?? string.Empty, StringComparer.Ordinal); var followed = await _db.Queryable().ToListAsync(); var config = _commonService.GetConfig() ?? new AppConfig(); var episodes = videos.Where(x => x.ViedoType is VideoTypeEnum.dy_mix or VideoTypeEnum.dy_series) .GroupBy(x => x.CateId ?? x.CateXId ?? string.Empty) .SelectMany(group => group.OrderBy(x => x.CreateTime).ThenBy(x => x.Id).Select((video, index) => new { video.Id, Episode = index + 1 })) .ToDictionary(x => x.Id, x => x.Episode); var result = new List(videos.Count); var openListSettings = await _settingsService.GetAsync(); var targetStorage = _openList.Bind(openListSettings); foreach (var video in videos) { cookies.TryGetValue(video.CookieId ?? string.Empty, out var cookie); categories.TryGetValue(video.CateId ?? string.Empty, out var category); var follow = followed.FirstOrDefault(x => x.UperId == video.AuthorId); StorageMigrationPathPlan plan = null; string planError = null; try { plan = video.StorageType == MediaStorageType.WebDav ? new StorageMigrationPathPlan { VideoPath = StoragePath.NormalizeRemote(video.VideoSavePath), CoverPath = string.IsNullOrWhiteSpace(video.VideoCoverSavePath) ? string.Empty : StoragePath.NormalizeRemote(video.VideoCoverSavePath), AvatarPath = string.IsNullOrWhiteSpace(video.AuthorAvatar) ? string.Empty : StoragePath.NormalizeRemote(video.AuthorAvatar) } : StorageMigrationPathPolicy.Build(video, cookie, category, follow, config, episodes.GetValueOrDefault(video.Id, 1)); } catch (Exception ex) { planError = $"视频 {video.AwemeId ?? video.Id}:{ex.Message}"; } var roots = StorageMigrationPathPolicy.GetLocalRoots(cookie); var readable = SafeLocalMigrationFile.TryResolve(video.VideoSavePath, roots, out var fullPath, out _) && new FileInfo(fullPath).Length > 0; long? existingTargetLength = null; if (plan != null && video.StorageType == MediaStorageType.WebDav) { try { existingTargetLength = await targetStorage.GetLengthAsync(plan.VideoPath); } catch (Exception ex) { planError = $"视频 {video.AwemeId ?? video.Id} OpenList 接管检测失败:{ex.GetBaseException().Message}"; } } result.Add(new PlanningEntry { Video = video, Cookie = cookie, Category = category, Followed = follow, Plan = plan, PlanError = planError, LocalRoots = roots, LocalReadable = readable, LocalLength = readable ? new FileInfo(fullPath).Length : 0, AdoptExistingTarget = existingTargetLength is > 0 && (video.FileSize <= 0 || existingTargetLength.Value == video.FileSize) }); } return result; } private static List BuildAttachments(PlanningEntry entry) { var attachments = new List(); Add(attachments, "cover", entry.Video.VideoCoverSavePath, entry.Video.VideoCoverUrl, entry.Plan.CoverPath, false); Add(attachments, "avatar", entry.Video.AuthorAvatar, entry.Video.AuthorAvatarUrl, entry.Plan.AvatarPath, true); if (!string.IsNullOrWhiteSpace(entry.Video.DynamicVideos)) { try { var dynamicItems = JsonConvert.DeserializeObject>(entry.Video.DynamicVideos) ?? new(); var index = 0; foreach (var item in dynamicItems) { index++; var oldName = Path.GetFileName(item.Path); var name = string.IsNullOrWhiteSpace(oldName) ? $"attachment-{index:D3}.bin" : oldName; Add(attachments, "dynamic", item.Path, Uri.IsWellFormedUriString(item.Path, UriKind.Absolute) ? item.Path : null, StoragePath.CombineRemote(entry.Plan.DirectoryPath, name), false, item.Height, item.Width); } } catch (JsonException ex) { Serilog.Log.Warning(ex, "解析迁移附件失败:{VideoId}", entry.Video.Id); } } return attachments; } private static void Add(List list, string kind, string source, string sourceUrl, string target, bool shared, int height = 0, int width = 0) { if (string.IsNullOrWhiteSpace(target)) return; list.Add(new MigrationAttachmentPlan { Kind = kind, SourcePath = source, SourceUrl = sourceUrl, TargetPath = target, Shared = shared, Height = height, Width = width }); } private static StorageMigrationTaskDetail ToDetail(StorageMigrationTask task) { var isOpenListTask = task.TargetStorageType == MediaStorageType.OpenList; return new StorageMigrationTaskDetail { Task = task, StatusText = task.Status switch { StorageMigrationTaskStatus.Queued => "排队", StorageMigrationTaskStatus.Running => "运行中", StorageMigrationTaskStatus.Paused => "已暂停", StorageMigrationTaskStatus.Completed => "已完成", StorageMigrationTaskStatus.PartiallyFailed => "部分失败", StorageMigrationTaskStatus.Cancelled => "已取消", StorageMigrationTaskStatus.Cleaning => "清理中", StorageMigrationTaskStatus.Cleaned => "已清理", StorageMigrationTaskStatus.RolledBack => "已回滚", _ => task.Status.ToString() }, CurrentFile = task.CurrentFile, CanPause = isOpenListTask && task.Status is (StorageMigrationTaskStatus.Queued or StorageMigrationTaskStatus.Running), CanResume = isOpenListTask && task.Status == StorageMigrationTaskStatus.Paused, CanCancel = isOpenListTask && task.Status is (StorageMigrationTaskStatus.Queued or StorageMigrationTaskStatus.Running or StorageMigrationTaskStatus.Paused), CanRetryFailed = isOpenListTask && task.FailedCount > 0 && task.Status is (StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled or StorageMigrationTaskStatus.Paused), CanCleanup = isOpenListTask && task.SuccessCount > 0 && task.Status is (StorageMigrationTaskStatus.Completed or StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled), CanRollback = isOpenListTask && task.SuccessCount > 0 && !task.CleanedAt.HasValue && task.Status is (StorageMigrationTaskStatus.Completed or StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled), CanArchive = CanArchive(task) }; } private static bool CanArchive(StorageMigrationTask task) => task != null && !task.IsArchived && IsTerminal(task.Status) && (task.TargetStorageType != MediaStorageType.OpenList || task.Status is StorageMigrationTaskStatus.Cleaned or StorageMigrationTaskStatus.RolledBack || task.SuccessCount == 0); private static bool IsTerminal(StorageMigrationTaskStatus status) => status is StorageMigrationTaskStatus.Completed or StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled or StorageMigrationTaskStatus.Cleaned or StorageMigrationTaskStatus.RolledBack; private static string FormatBytes(long value) { string[] units = { "B", "KB", "MB", "GB", "TB" }; var size = (double)value; var unit = 0; while (size >= 1024 && unit < units.Length - 1) { size /= 1024; unit++; } return $"{size:0.##} {units[unit]}"; } private sealed class PlanningEntry { public DouyinVideo Video { get; set; } public DouyinCookie Cookie { get; set; } public DouyinCollectCate Category { get; set; } public DouyinFollowed Followed { get; set; } public StorageMigrationPathPlan Plan { get; set; } public string PlanError { get; set; } public IReadOnlyList LocalRoots { get; set; } public bool LocalReadable { get; set; } public long LocalLength { get; set; } public bool AdoptExistingTarget { get; set; } } internal sealed class MigrationAttachmentPlan { public string Kind { get; set; } public string SourcePath { get; set; } public string SourceUrl { get; set; } public string TargetPath { get; set; } public bool Shared { get; set; } public int Height { get; set; } public int Width { get; set; } } private enum FailedRecordDisposition { Eligible, AlreadyMissing, Changed, PermanentlyExcluded, InvalidSnapshot } private sealed class FailedRecordRemovalPlan { public int ActiveMigrationCount { get; set; } public List Groups { get; set; } = new(); public string ConfirmationToken { get; set; } } private sealed class FailedRecordGroup { public string VideoId { get; set; } public string AwemeId { get; set; } public DouyinVideo Current { get; set; } public List Snapshots { get; set; } = new(); public List Items { get; set; } = new(); public FailedRecordDisposition Disposition { get; set; } } private sealed record CleanupCandidate(string Path, bool Shared); } }