feat: add fnOS packaging, storage workflows and release pipeline
This commit is contained in:
@@ -0,0 +1,958 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using dy.net.model.dto;
|
||||
using dy.net.model.entity;
|
||||
using dy.net.storage;
|
||||
using dy.net.utils;
|
||||
using Newtonsoft.Json;
|
||||
using SqlSugar;
|
||||
using MediaStorageType = dy.net.model.dto.StorageType;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
public class StorageMigrationService
|
||||
{
|
||||
private readonly ISqlSugarClient _db;
|
||||
private readonly DouyinCommonService _commonService;
|
||||
private readonly OpenListSettingsService _settingsService;
|
||||
private readonly OpenListMediaStorage _openList;
|
||||
|
||||
public StorageMigrationService(
|
||||
ISqlSugarClient db,
|
||||
DouyinCommonService commonService,
|
||||
OpenListSettingsService settingsService,
|
||||
OpenListMediaStorage openList)
|
||||
{
|
||||
_db = db;
|
||||
_commonService = commonService;
|
||||
_settingsService = settingsService;
|
||||
_openList = openList;
|
||||
}
|
||||
|
||||
public async Task<StorageMigrationPreflightResult> PreflightAsync(bool verifyCapabilities, CancellationToken cancellationToken)
|
||||
{
|
||||
var result = new StorageMigrationPreflightResult();
|
||||
if (_commonService.GetConfig()?.StorageType != MediaStorageType.OpenList)
|
||||
{
|
||||
result.Errors.Add("请先完成测试并启用 OpenList 原生存储；迁移不会自动切换存储模式。");
|
||||
return result;
|
||||
}
|
||||
|
||||
var settings = await _settingsService.GetAsync();
|
||||
try
|
||||
{
|
||||
OpenListTransferService.ValidateSettings(settings);
|
||||
if (string.IsNullOrWhiteSpace(await _settingsService.GetPasswordAsync(settings)))
|
||||
result.Errors.Add("OpenList 密码无效，请重新输入并保存配置。");
|
||||
else
|
||||
result.ConfigurationFingerprint = StorageConfigurationFingerprint.Create(settings);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
result.Errors.Add(ex.GetBaseException().Message);
|
||||
}
|
||||
if (!settings.LastTestedAt.HasValue)
|
||||
result.Errors.Add("当前 OpenList 配置尚未通过服务端复制能力测试。");
|
||||
if (verifyCapabilities)
|
||||
{
|
||||
var probe = await _openList.ProbeAsync(settings, cancellationToken);
|
||||
if (!probe.Success) result.Errors.Add("OpenList 能力测试失败：" + probe.Message);
|
||||
}
|
||||
|
||||
if (result.Errors.Count > 0) return result;
|
||||
|
||||
List<PlanningEntry> planning;
|
||||
try { planning = await BuildPlanningAsync(); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
result.Errors.Add("读取 OpenList 目标失败：" + ex.GetBaseException().Message);
|
||||
return result;
|
||||
}
|
||||
result.RecordCount = planning.Count;
|
||||
result.LocalRecordCount = planning.Count(x => x.Video.StorageType == MediaStorageType.Local);
|
||||
result.LegacyWebDavRecordCount = planning.Count(x => x.Video.StorageType == MediaStorageType.WebDav);
|
||||
result.AdoptableCount = planning.Count(x => x.AdoptExistingTarget);
|
||||
result.TransferRequiredCount = planning.Count(x => !x.AdoptExistingTarget);
|
||||
result.ReadableFileCount = planning.Count(x => x.LocalReadable);
|
||||
result.MissingFileCount = planning.Count(x => !x.AdoptExistingTarget && !x.LocalReadable);
|
||||
result.TotalBytes = planning.Where(x => x.LocalReadable).Sum(x => x.LocalLength);
|
||||
result.InvalidCookieCount = planning.Where(x => x.Cookie == null || x.Cookie.StatusCode != 0 || string.IsNullOrWhiteSpace(x.Cookie.Cookies))
|
||||
.Select(x => x.Video.CookieId).Distinct().Count();
|
||||
result.MissingTargetPathCount = planning.Count(x => x.Plan == null);
|
||||
result.ConflictCount = planning.Where(x => x.Plan != null)
|
||||
.GroupBy(x => x.Plan.VideoPath, StringComparer.Ordinal)
|
||||
.Where(x => x.Count() > 1).Sum(x => x.Count());
|
||||
|
||||
foreach (var error in planning.Select(x => x.PlanError).Where(x => !string.IsNullOrWhiteSpace(x)).Distinct().Take(20))
|
||||
result.Errors.Add(error);
|
||||
if (result.ConflictCount > 0)
|
||||
result.Errors.Add($"发现 {result.ConflictCount} 条记录映射到重复目标路径，必须先调整目录或命名配置。");
|
||||
if (result.MissingFileCount > 0)
|
||||
result.Warnings.Add($"{result.MissingFileCount} 条记录的本地主媒体缺失或不可安全读取，执行时会先尝试记录 URL，再扫描对应抖音全量列表回源。");
|
||||
if (result.InvalidCookieCount > 0)
|
||||
result.Warnings.Add($"{result.InvalidCookieCount} 个账号 Cookie 无效；这些账号下缺失本地文件的作品可能无法回源。");
|
||||
result.Warnings.Add($"OpenList 无法统一报告所有后端剩余容量；容量未知，预计需读取本地数据 {FormatBytes(result.TotalBytes)}。");
|
||||
result.CanStart = result.RecordCount > 0 && result.Errors.Count == 0;
|
||||
if (result.RecordCount == 0) result.Errors.Add("没有需要接管到 OpenList 的旧存储视频记录。");
|
||||
return result;
|
||||
}
|
||||
|
||||
public async Task<StorageMigrationTask> CreateAsync(CreateStorageMigrationRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
request ??= new CreateStorageMigrationRequest();
|
||||
if (request.Concurrency is < 1 or > 3) throw new InvalidOperationException("迁移并发只能选择 1–3。");
|
||||
var preflight = await PreflightAsync(false, cancellationToken);
|
||||
if (!preflight.CanStart) throw new InvalidOperationException(string.Join("；", preflight.Errors));
|
||||
if (string.IsNullOrWhiteSpace(request.ConfigurationFingerprint)
|
||||
|| !string.Equals(request.ConfigurationFingerprint, preflight.ConfigurationFingerprint, StringComparison.Ordinal))
|
||||
throw new InvalidOperationException("OpenList 配置已变化，请重新执行迁移预检。");
|
||||
|
||||
var active = await _db.Queryable<StorageMigrationTask>().Where(x =>
|
||||
x.Status == StorageMigrationTaskStatus.Queued || x.Status == StorageMigrationTaskStatus.Running
|
||||
|| x.Status == StorageMigrationTaskStatus.Paused || x.Status == StorageMigrationTaskStatus.Cleaning).AnyAsync();
|
||||
if (active) throw new InvalidOperationException("已有未结束的存储迁移任务，请先处理当前任务。");
|
||||
|
||||
var planning = await BuildPlanningAsync();
|
||||
if (planning.Any(x => x.Plan == null || !string.IsNullOrWhiteSpace(x.PlanError)))
|
||||
throw new InvalidOperationException("迁移计划在创建前发生变化，请重新执行预检。");
|
||||
var now = DateTime.Now;
|
||||
var task = new StorageMigrationTask
|
||||
{
|
||||
Id = Guid.NewGuid().ToString("N"),
|
||||
Status = StorageMigrationTaskStatus.Queued,
|
||||
TargetStorageType = MediaStorageType.OpenList,
|
||||
Concurrency = request.Concurrency,
|
||||
TotalCount = planning.Count,
|
||||
PendingCount = planning.Count,
|
||||
TotalBytes = planning.Where(x => x.LocalReadable).Sum(x => x.LocalLength),
|
||||
ConfigurationFingerprint = preflight.ConfigurationFingerprint,
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now
|
||||
};
|
||||
var items = planning.Select(entry => new StorageMigrationItem
|
||||
{
|
||||
Id = Guid.NewGuid().ToString("N"),
|
||||
TaskId = task.Id,
|
||||
VideoId = entry.Video.Id,
|
||||
SourceStorageType = entry.Video.StorageType,
|
||||
AdoptExistingTarget = entry.AdoptExistingTarget,
|
||||
Stage = StorageMigrationItemStage.Pending,
|
||||
ExpectedLength = entry.LocalLength > 0 ? entry.LocalLength : entry.Video.FileSize,
|
||||
OldVideoPath = entry.Video.VideoSavePath,
|
||||
TargetVideoPath = entry.Plan.VideoPath,
|
||||
TargetCoverPath = entry.Plan.CoverPath,
|
||||
OldSnapshotJson = JsonConvert.SerializeObject(entry.Video),
|
||||
AttachmentPlanJson = JsonConvert.SerializeObject(BuildAttachments(entry)),
|
||||
UploadedPathsJson = "[]",
|
||||
OwnedRemotePathsJson = "[]",
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now
|
||||
}).ToList();
|
||||
|
||||
var transaction = await _db.Ado.UseTranAsync(async () =>
|
||||
{
|
||||
await _db.Insertable(task).ExecuteCommandAsync();
|
||||
await _db.Insertable(items).ExecuteCommandAsync();
|
||||
});
|
||||
if (!transaction.IsSuccess) throw new InvalidOperationException("创建迁移任务失败：" + transaction.ErrorMessage);
|
||||
return task;
|
||||
}
|
||||
|
||||
public async Task<StorageMigrationTaskDetail> GetAsync(string id)
|
||||
{
|
||||
var task = await _db.Queryable<StorageMigrationTask>().InSingleAsync(id)
|
||||
?? throw new KeyNotFoundException("迁移任务不存在");
|
||||
return ToDetail(task);
|
||||
}
|
||||
|
||||
public async Task<StorageMigrationTaskDetail> GetLatestAsync()
|
||||
{
|
||||
var task = await _db.Queryable<StorageMigrationTask>().Where(x => !x.IsArchived)
|
||||
.OrderByDescending(x => x.CreatedAt).FirstAsync();
|
||||
return task == null ? null : ToDetail(task);
|
||||
}
|
||||
|
||||
public async Task ArchiveAsync(string id)
|
||||
{
|
||||
var task = await RequireTaskAsync(id);
|
||||
if (!CanArchive(task))
|
||||
throw new InvalidOperationException("该迁移仍有成功项等待清理或回滚，请先处理旧文件后再归档。");
|
||||
task.IsArchived = true;
|
||||
task.UpdatedAt = DateTime.Now;
|
||||
if (await _db.Updateable(task).ExecuteCommandAsync() != 1)
|
||||
throw new InvalidOperationException("归档迁移历史失败。");
|
||||
}
|
||||
|
||||
public async Task<StorageMigrationArchiveResult> ArchiveAllSettledAsync()
|
||||
{
|
||||
var tasks = await _db.Queryable<StorageMigrationTask>().Where(x => !x.IsArchived).ToListAsync();
|
||||
var archivable = tasks.Where(CanArchive).ToList();
|
||||
var requiresCleanup = tasks.Count(x => IsTerminal(x.Status) && !CanArchive(x));
|
||||
if (archivable.Count > 0)
|
||||
{
|
||||
var now = DateTime.Now;
|
||||
foreach (var task in archivable)
|
||||
{
|
||||
task.IsArchived = true;
|
||||
task.UpdatedAt = now;
|
||||
}
|
||||
var changed = await _db.Updateable(archivable).ExecuteCommandAsync();
|
||||
if (changed != archivable.Count) throw new InvalidOperationException("部分迁移历史归档失败。");
|
||||
}
|
||||
return new StorageMigrationArchiveResult
|
||||
{
|
||||
ArchivedCount = archivable.Count,
|
||||
RequiresCleanupCount = requiresCleanup,
|
||||
Message = requiresCleanup > 0
|
||||
? $"已隐藏 {archivable.Count} 条迁移历史；另有 {requiresCleanup} 条包含已迁移视频，请先清理旧文件或回滚。"
|
||||
: $"已隐藏 {archivable.Count} 条已结束的迁移历史。"
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<StorageMigrationItemPage> GetItemsAsync(string taskId, StorageMigrationItemPageRequest request)
|
||||
{
|
||||
request ??= new StorageMigrationItemPageRequest();
|
||||
request.PageIndex = Math.Max(1, request.PageIndex);
|
||||
request.PageSize = Math.Clamp(request.PageSize, 1, 100);
|
||||
var query = _db.Queryable<StorageMigrationItem>().Where(x => x.TaskId == taskId)
|
||||
.WhereIF(request.Stage.HasValue, x => x.Stage == request.Stage.Value);
|
||||
return new StorageMigrationItemPage
|
||||
{
|
||||
TotalCount = await query.CountAsync(),
|
||||
Items = await query.OrderBy(x => x.CreatedAt).Skip((request.PageIndex - 1) * request.PageSize).Take(request.PageSize).ToListAsync()
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<FailedMigrationRecordPreview> PreviewFailedRecordRemovalAsync()
|
||||
{
|
||||
var plan = await BuildFailedRecordRemovalPlanAsync();
|
||||
var preview = new FailedMigrationRecordPreview
|
||||
{
|
||||
ActiveMigrationCount = plan.ActiveMigrationCount,
|
||||
FailedItemCount = plan.Groups.Sum(x => x.Items.Count),
|
||||
DistinctVideoCount = plan.Groups.Count,
|
||||
EligibleRecordCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.Eligible),
|
||||
AlreadyMissingCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.AlreadyMissing),
|
||||
ChangedRecordCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.Changed),
|
||||
PermanentlyExcludedCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.PermanentlyExcluded),
|
||||
InvalidSnapshotCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.InvalidSnapshot)
|
||||
};
|
||||
preview.CanExecute = preview.ActiveMigrationCount == 0
|
||||
&& preview.EligibleRecordCount + preview.AlreadyMissingCount > 0;
|
||||
preview.ConfirmationToken = preview.CanExecute ? plan.ConfirmationToken : null;
|
||||
|
||||
if (preview.ActiveMigrationCount > 0)
|
||||
preview.Errors.Add("存在排队、运行、暂停或清理中的存储迁移任务，请先结束当前迁移再处理历史失败记录。");
|
||||
else if (preview.FailedItemCount == 0)
|
||||
preview.Errors.Add("当前没有可处理的历史迁移失败项。");
|
||||
else if (!preview.CanExecute)
|
||||
preview.Errors.Add("没有可安全删除的视频记录；已变化、已迁移、永久排除或快照损坏的记录不会被修改。");
|
||||
|
||||
preview.Warnings.Add("此操作只删除数据库视频记录，不删除旧本地文件或 OpenList 文件，也不会自动启动同步。");
|
||||
if (preview.ChangedRecordCount > 0)
|
||||
preview.Warnings.Add($"{preview.ChangedRecordCount} 条视频记录已变化、已迁移或已重新创建，将安全跳过。");
|
||||
if (preview.PermanentlyExcludedCount > 0)
|
||||
preview.Warnings.Add($"{preview.PermanentlyExcludedCount} 条视频已被永久排除，请先在任务中心取消排除后再处理。");
|
||||
if (preview.InvalidSnapshotCount > 0)
|
||||
preview.Warnings.Add($"{preview.InvalidSnapshotCount} 条迁移快照无效，将保留原记录和失败状态。");
|
||||
return preview;
|
||||
}
|
||||
|
||||
public async Task<RemoveFailedMigrationRecordsResult> RemoveFailedRecordsAsync(RemoveFailedMigrationRecordsRequest request)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(request?.ConfirmationToken))
|
||||
throw new InvalidOperationException("缺少删除确认信息，请重新打开预览后确认。");
|
||||
|
||||
RemoveFailedMigrationRecordsResult result = null;
|
||||
var transaction = await _db.Ado.UseTranAsync(async () =>
|
||||
{
|
||||
var plan = await BuildFailedRecordRemovalPlanAsync();
|
||||
if (plan.ActiveMigrationCount > 0)
|
||||
throw new InvalidOperationException("存在未结束的存储迁移任务，请先结束当前迁移。");
|
||||
if (!string.Equals(plan.ConfirmationToken, request.ConfirmationToken, StringComparison.Ordinal))
|
||||
throw new InvalidOperationException("迁移失败记录已发生变化，请重新预览并确认。");
|
||||
|
||||
var actionable = plan.Groups.Where(x => x.Disposition is FailedRecordDisposition.Eligible
|
||||
or FailedRecordDisposition.AlreadyMissing).ToList();
|
||||
if (actionable.Count == 0)
|
||||
throw new InvalidOperationException("没有可安全删除的视频记录。");
|
||||
|
||||
var now = DateTime.Now;
|
||||
var changedItems = new List<StorageMigrationItem>();
|
||||
var deletedRecords = 0;
|
||||
foreach (var group in actionable)
|
||||
{
|
||||
if (group.Disposition == FailedRecordDisposition.Eligible)
|
||||
{
|
||||
var current = group.Current;
|
||||
var expectedStorageType = current.StorageType;
|
||||
var deleted = await _db.Deleteable<DouyinVideo>().Where(x => x.Id == current.Id
|
||||
&& x.AwemeId == current.AwemeId && x.StorageType == expectedStorageType
|
||||
&& x.VideoSavePath == current.VideoSavePath).ExecuteCommandAsync();
|
||||
if (deleted != 1)
|
||||
throw new InvalidOperationException($"视频 {current.AwemeId ?? current.Id} 已发生变化，请重新预览。");
|
||||
deletedRecords++;
|
||||
}
|
||||
|
||||
foreach (var item in group.Items)
|
||||
{
|
||||
item.Stage = StorageMigrationItemStage.RecordRemoved;
|
||||
item.ErrorMessage = null;
|
||||
item.WarningMessage = group.Disposition == FailedRecordDisposition.Eligible
|
||||
? "视频记录已删除；旧文件和 OpenList 文件均已保留，等待后续正常同步重新发现。"
|
||||
: "原视频记录已不存在；迁移失败项已归档，等待后续正常同步重新发现。";
|
||||
item.UpdatedAt = now;
|
||||
item.CompletedAt = now;
|
||||
changedItems.Add(item);
|
||||
}
|
||||
}
|
||||
|
||||
if (changedItems.Count > 0)
|
||||
{
|
||||
var changed = await _db.Updateable(changedItems).ExecuteCommandAsync();
|
||||
if (changed != changedItems.Count)
|
||||
throw new InvalidOperationException("更新迁移失败项状态不完整，操作已回滚。");
|
||||
}
|
||||
|
||||
var affectedTaskIds = changedItems.Select(x => x.TaskId).Distinct().ToList();
|
||||
foreach (var taskId in affectedTaskIds) await RefreshCountsAsync(taskId);
|
||||
|
||||
result = new RemoveFailedMigrationRecordsResult
|
||||
{
|
||||
DeletedRecordCount = deletedRecords,
|
||||
AlreadyMissingCount = actionable.Count(x => x.Disposition == FailedRecordDisposition.AlreadyMissing),
|
||||
RemovedItemCount = changedItems.Count,
|
||||
AffectedTaskCount = affectedTaskIds.Count,
|
||||
SkippedChangedCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.Changed),
|
||||
SkippedExcludedCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.PermanentlyExcluded),
|
||||
InvalidSnapshotCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.InvalidSnapshot),
|
||||
Message = $"已删除 {deletedRecords} 条视频记录，更新 {changedItems.Count} 条迁移失败项；旧文件已保留，将随系统后续正常同步重新发现。"
|
||||
};
|
||||
});
|
||||
if (!transaction.IsSuccess)
|
||||
throw transaction.ErrorException ?? new InvalidOperationException("删除迁移失败记录失败，数据库操作已回滚。");
|
||||
return result;
|
||||
}
|
||||
|
||||
public Task PauseAsync(string id) => ChangeStatusAsync(id,
|
||||
new[] { StorageMigrationTaskStatus.Queued, StorageMigrationTaskStatus.Running }, StorageMigrationTaskStatus.Paused, null);
|
||||
|
||||
public async Task ResumeAsync(string id)
|
||||
{
|
||||
var task = await RequireTaskAsync(id);
|
||||
if (task.Status != StorageMigrationTaskStatus.Paused) throw new InvalidOperationException("只有暂停中的任务可以恢复。");
|
||||
var fingerprint = StorageConfigurationFingerprint.Create(await _settingsService.GetAsync());
|
||||
if (!string.Equals(fingerprint, task.ConfigurationFingerprint, StringComparison.Ordinal))
|
||||
throw new InvalidOperationException("OpenList 地址、源挂载、基础目录或账号已变化，请重新预检并创建新任务。");
|
||||
await ChangeStatusAsync(id, new[] { StorageMigrationTaskStatus.Paused }, StorageMigrationTaskStatus.Queued, null);
|
||||
}
|
||||
|
||||
public Task CancelAsync(string id) => ChangeStatusAsync(id,
|
||||
new[] { StorageMigrationTaskStatus.Queued, StorageMigrationTaskStatus.Running, StorageMigrationTaskStatus.Paused },
|
||||
StorageMigrationTaskStatus.Cancelled, null);
|
||||
|
||||
public async Task RetryFailedAsync(string id)
|
||||
{
|
||||
var task = await RequireTaskAsync(id);
|
||||
if (task.Status is not (StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled or StorageMigrationTaskStatus.Paused))
|
||||
throw new InvalidOperationException("当前任务没有可重试的失败项。");
|
||||
var fingerprint = StorageConfigurationFingerprint.Create(await _settingsService.GetAsync());
|
||||
if (fingerprint != task.ConfigurationFingerprint) throw new InvalidOperationException("OpenList 配置已变化，请重新预检。");
|
||||
await _db.Updateable<StorageMigrationItem>()
|
||||
.SetColumns(x => new StorageMigrationItem { Stage = StorageMigrationItemStage.Pending, ErrorMessage = null, UpdatedAt = DateTime.Now })
|
||||
.Where(x => x.TaskId == id && x.Stage == StorageMigrationItemStage.Failed).ExecuteCommandAsync();
|
||||
await _db.Updateable<StorageMigrationTask>()
|
||||
.SetColumns(x => new StorageMigrationTask { Status = StorageMigrationTaskStatus.Queued, CompletedAt = null, ErrorMessage = null, UpdatedAt = DateTime.Now })
|
||||
.Where(x => x.Id == id).ExecuteCommandAsync();
|
||||
await RefreshCountsAsync(id);
|
||||
}
|
||||
|
||||
public async Task CleanupAsync(string id, CancellationToken cancellationToken)
|
||||
{
|
||||
var task = await RequireTaskAsync(id);
|
||||
if (task.Status is not (StorageMigrationTaskStatus.Completed or StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled))
|
||||
throw new InvalidOperationException("只有已结束的任务可以清理旧文件。");
|
||||
var targetStorage = await BindTaskTargetAsync(task);
|
||||
await _db.Updateable<StorageMigrationTask>()
|
||||
.SetColumns(x => new StorageMigrationTask { Status = StorageMigrationTaskStatus.Cleaning, ErrorMessage = null, UpdatedAt = DateTime.Now })
|
||||
.Where(x => x.Id == id).ExecuteCommandAsync();
|
||||
|
||||
var items = await _db.Queryable<StorageMigrationItem>().Where(x => x.TaskId == id
|
||||
&& (x.Stage == StorageMigrationItemStage.Succeeded || x.Stage == StorageMigrationItemStage.SucceededWithWarnings)).ToListAsync();
|
||||
var errors = new List<string>();
|
||||
foreach (var item in items)
|
||||
{
|
||||
try { await CleanupItemAsync(item, targetStorage, cancellationToken); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
errors.Add($"{item.VideoId}: {ex.Message}");
|
||||
item.ErrorMessage = "清理失败：" + ex.Message;
|
||||
item.UpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(item).ExecuteCommandAsync();
|
||||
}
|
||||
}
|
||||
|
||||
var remaining = await _db.Queryable<StorageMigrationItem>().Where(x => x.TaskId == id
|
||||
&& (x.Stage == StorageMigrationItemStage.Succeeded || x.Stage == StorageMigrationItemStage.SucceededWithWarnings)).CountAsync();
|
||||
await _db.Updateable<StorageMigrationTask>()
|
||||
.SetColumns(x => new StorageMigrationTask
|
||||
{
|
||||
Status = remaining == 0 ? StorageMigrationTaskStatus.Cleaned : StorageMigrationTaskStatus.PartiallyFailed,
|
||||
CleanedAt = remaining == 0 ? DateTime.Now : null,
|
||||
ErrorMessage = errors.Count == 0 ? null : string.Join("；", errors.Take(20)),
|
||||
UpdatedAt = DateTime.Now
|
||||
}).Where(x => x.Id == id).ExecuteCommandAsync();
|
||||
await RefreshCountsAsync(id);
|
||||
}
|
||||
|
||||
public async Task RollbackAsync(string id, CancellationToken cancellationToken)
|
||||
{
|
||||
var task = await RequireTaskAsync(id);
|
||||
if (task.Status is not (StorageMigrationTaskStatus.Completed or StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled))
|
||||
throw new InvalidOperationException("当前任务状态不允许回滚。");
|
||||
if (task.CleanedAt.HasValue || await _db.Queryable<StorageMigrationItem>().Where(x => x.TaskId == id && x.Stage == StorageMigrationItemStage.Cleaned).AnyAsync())
|
||||
throw new InvalidOperationException("旧文件清理已开始，不能再执行回滚。");
|
||||
var targetStorage = await BindTaskTargetAsync(task);
|
||||
|
||||
var items = await _db.Queryable<StorageMigrationItem>().Where(x => x.TaskId == id
|
||||
&& (x.Stage == StorageMigrationItemStage.Succeeded || x.Stage == StorageMigrationItemStage.SucceededWithWarnings)).ToListAsync();
|
||||
var snapshots = items.ToDictionary(x => x.Id, x => JsonConvert.DeserializeObject<DouyinVideo>(x.OldSnapshotJson));
|
||||
foreach (var pair in snapshots)
|
||||
{
|
||||
var snapshot = pair.Value ?? throw new InvalidOperationException("旧视频快照损坏，无法回滚。");
|
||||
var item = items.First(x => x.Id == pair.Key);
|
||||
var sourceType = item.SourceStorageType ?? snapshot.StorageType;
|
||||
if (sourceType == MediaStorageType.Local)
|
||||
{
|
||||
var cookie = await _db.Queryable<DouyinCookie>().InSingleAsync(snapshot.CookieId);
|
||||
if (!SafeLocalMigrationFile.TryResolve(snapshot.VideoSavePath,
|
||||
StorageMigrationPathPolicy.GetLocalRoots(cookie), out var local, out var error)
|
||||
|| !File.Exists(local) || new FileInfo(local).Length <= 0)
|
||||
throw new InvalidOperationException($"视频 {snapshot.AwemeId} 的旧主媒体不可用：{error ?? "文件不存在或为空"}");
|
||||
}
|
||||
else if (sourceType == MediaStorageType.WebDav)
|
||||
{
|
||||
var length = await targetStorage.GetLengthAsync(snapshot.VideoSavePath, cancellationToken);
|
||||
if (!length.HasValue || length <= 0
|
||||
|| snapshot.FileSize > 0 && length.Value != snapshot.FileSize)
|
||||
throw new InvalidOperationException($"视频 {snapshot.AwemeId} 的原 WebDAV 文件已不可用，无法恢复旧记录。");
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new InvalidOperationException($"视频 {snapshot.AwemeId} 的旧存储类型不支持回滚。");
|
||||
}
|
||||
}
|
||||
|
||||
var transaction = await _db.Ado.UseTranAsync(async () =>
|
||||
{
|
||||
foreach (var item in items)
|
||||
{
|
||||
var snapshot = snapshots[item.Id];
|
||||
var current = await _db.Queryable<DouyinVideo>().InSingleAsync(item.VideoId)
|
||||
?? throw new InvalidOperationException($"视频记录 {item.VideoId} 已不存在");
|
||||
if (current.StorageType != MediaStorageType.OpenList || current.VideoSavePath != item.TargetVideoPath)
|
||||
throw new InvalidOperationException($"视频 {snapshot.AwemeId} 已被其他操作修改，无法回滚");
|
||||
if (await _db.Updateable(snapshot).ExecuteCommandAsync() != 1)
|
||||
throw new InvalidOperationException($"恢复视频 {snapshot.AwemeId} 失败");
|
||||
item.Stage = StorageMigrationItemStage.RolledBack;
|
||||
item.UpdatedAt = DateTime.Now;
|
||||
item.CompletedAt = DateTime.Now;
|
||||
await _db.Updateable(item).ExecuteCommandAsync();
|
||||
}
|
||||
});
|
||||
if (!transaction.IsSuccess) throw transaction.ErrorException ?? new InvalidOperationException("回滚数据库事务失败");
|
||||
|
||||
var warnings = new List<string>();
|
||||
foreach (var item in items)
|
||||
{
|
||||
var snapshot = snapshots[item.Id];
|
||||
if ((item.SourceStorageType ?? snapshot.StorageType) == MediaStorageType.WebDav)
|
||||
continue;
|
||||
foreach (var remotePath in DeserializePaths(item.OwnedRemotePathsJson))
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!await IsRemotePathReferencedAsync(remotePath, cancellationToken)) await targetStorage.DeleteAsync(remotePath, cancellationToken);
|
||||
}
|
||||
catch (Exception ex) { warnings.Add($"{remotePath}: {ex.Message}"); }
|
||||
}
|
||||
}
|
||||
await _db.Updateable<StorageMigrationTask>()
|
||||
.SetColumns(x => new StorageMigrationTask
|
||||
{
|
||||
Status = StorageMigrationTaskStatus.RolledBack,
|
||||
RolledBackAt = DateTime.Now,
|
||||
ErrorMessage = warnings.Count == 0 ? null : "远端独占文件清理警告：" + string.Join("；", warnings.Take(20)),
|
||||
UpdatedAt = DateTime.Now
|
||||
}).Where(x => x.Id == id).ExecuteCommandAsync();
|
||||
await RefreshCountsAsync(id);
|
||||
}
|
||||
|
||||
private async Task CleanupItemAsync(StorageMigrationItem item, IMediaStorage targetStorage, CancellationToken cancellationToken)
|
||||
{
|
||||
var current = await _db.Queryable<DouyinVideo>().InSingleAsync(item.VideoId)
|
||||
?? throw new InvalidOperationException("视频记录不存在");
|
||||
if (current.StorageType != MediaStorageType.OpenList || current.VideoSavePath != item.TargetVideoPath)
|
||||
throw new InvalidOperationException("数据库已不再指向本任务的 OpenList 目标");
|
||||
var remoteLength = await targetStorage.GetLengthAsync(current.VideoSavePath, cancellationToken);
|
||||
if (!remoteLength.HasValue || remoteLength.Value != item.ExpectedLength)
|
||||
throw new InvalidOperationException("远端主媒体不存在或长度已变化");
|
||||
|
||||
var snapshot = JsonConvert.DeserializeObject<DouyinVideo>(item.OldSnapshotJson)
|
||||
?? throw new InvalidOperationException("旧视频快照损坏");
|
||||
if ((item.SourceStorageType ?? snapshot.StorageType) == MediaStorageType.WebDav)
|
||||
{
|
||||
item.Stage = StorageMigrationItemStage.Cleaned;
|
||||
item.ErrorMessage = null;
|
||||
item.WarningMessage = string.IsNullOrWhiteSpace(item.WarningMessage)
|
||||
? "原 WebDAV 记录与 OpenList 指向同一文件，无需删除媒体。"
|
||||
: item.WarningMessage + "；原 WebDAV 记录与 OpenList 指向同一文件，无需删除媒体。";
|
||||
item.UpdatedAt = DateTime.Now;
|
||||
item.CompletedAt = DateTime.Now;
|
||||
await _db.Updateable(item).ExecuteCommandAsync();
|
||||
return;
|
||||
}
|
||||
var cookie = await _db.Queryable<DouyinCookie>().InSingleAsync(snapshot.CookieId);
|
||||
var roots = StorageMigrationPathPolicy.GetLocalRoots(cookie);
|
||||
var candidates = BuildLocalCleanupCandidates(snapshot);
|
||||
var localVideos = await _db.Queryable<DouyinVideo>().Where(x => x.StorageType == MediaStorageType.Local).ToListAsync();
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(candidate.Path) || !File.Exists(candidate.Path)) continue;
|
||||
if (!SafeLocalMigrationFile.TryResolve(candidate.Path, roots, out var safePath, out var error))
|
||||
throw new InvalidOperationException($"拒绝清理不安全路径 {candidate.Path}：{error}");
|
||||
if (IsLocalPathReferenced(candidate.Path, localVideos, snapshot.Id, candidate.Shared)) continue;
|
||||
File.Delete(safePath);
|
||||
}
|
||||
DeleteEmptyParents(snapshot.VideoSavePath, roots);
|
||||
item.Stage = StorageMigrationItemStage.Cleaned;
|
||||
item.ErrorMessage = null;
|
||||
item.UpdatedAt = DateTime.Now;
|
||||
item.CompletedAt = DateTime.Now;
|
||||
await _db.Updateable(item).ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
private static List<CleanupCandidate> BuildLocalCleanupCandidates(DouyinVideo snapshot)
|
||||
{
|
||||
var result = new List<CleanupCandidate>
|
||||
{
|
||||
new(snapshot.VideoSavePath, false),
|
||||
new(snapshot.VideoCoverSavePath, snapshot.ViedoType is VideoTypeEnum.dy_mix or VideoTypeEnum.dy_series),
|
||||
new(snapshot.AuthorAvatar, true)
|
||||
};
|
||||
if (!string.IsNullOrWhiteSpace(snapshot.VideoSavePath))
|
||||
{
|
||||
var directory = Path.GetDirectoryName(snapshot.VideoSavePath);
|
||||
var basename = Path.GetFileNameWithoutExtension(snapshot.VideoSavePath);
|
||||
result.Add(new CleanupCandidate(Path.Combine(directory ?? string.Empty, basename + ".nfo"), false));
|
||||
if (snapshot.ViedoType is VideoTypeEnum.dy_mix or VideoTypeEnum.dy_series)
|
||||
result.Add(new CleanupCandidate(Path.Combine(directory ?? string.Empty, "tvshow.nfo"), true));
|
||||
}
|
||||
if (!string.IsNullOrWhiteSpace(snapshot.DynamicVideos))
|
||||
{
|
||||
try
|
||||
{
|
||||
foreach (var attachment in JsonConvert.DeserializeObject<List<DouyinMergeVideoDto>>(snapshot.DynamicVideos) ?? new())
|
||||
result.Add(new CleanupCandidate(attachment.Path, false));
|
||||
}
|
||||
catch (JsonException) { }
|
||||
}
|
||||
return result.Where(x => !string.IsNullOrWhiteSpace(x.Path)).DistinctBy(x => x.Path).ToList();
|
||||
}
|
||||
|
||||
private static bool IsLocalPathReferenced(string path, IEnumerable<DouyinVideo> localVideos, string excludedVideoId, bool shared)
|
||||
{
|
||||
foreach (var video in localVideos.Where(x => x.Id != excludedVideoId))
|
||||
{
|
||||
if (path == video.VideoSavePath || path == video.VideoCoverSavePath || path == video.AuthorAvatar) return true;
|
||||
if (shared && !string.IsNullOrWhiteSpace(video.VideoSavePath)
|
||||
&& string.Equals(Path.GetDirectoryName(video.VideoSavePath), Path.GetDirectoryName(path), StringComparison.Ordinal)) return true;
|
||||
if (!string.IsNullOrWhiteSpace(video.DynamicVideos) && video.DynamicVideos.Contains(path, StringComparison.Ordinal)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private async Task<bool> IsRemotePathReferencedAsync(string path, CancellationToken cancellationToken)
|
||||
{
|
||||
var videos = await _db.Queryable<DouyinVideo>().Where(x => x.StorageType.IsRemote()).ToListAsync();
|
||||
foreach (var video in videos)
|
||||
{
|
||||
if (video.VideoSavePath == path || video.VideoCoverSavePath == path || video.AuthorAvatar == path) return true;
|
||||
if (!string.IsNullOrWhiteSpace(video.DynamicVideos) && video.DynamicVideos.Contains(path, StringComparison.Ordinal)) return true;
|
||||
if (NfoContentBuilder.Build(video).Keys.Contains(path)) return true;
|
||||
if (Path.GetFileName(path).Equals("tvshow.nfo", StringComparison.OrdinalIgnoreCase)
|
||||
&& StoragePath.DirectoryName(video.VideoSavePath) == StoragePath.DirectoryName(path)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static void DeleteEmptyParents(string filePath, IReadOnlyList<string> roots)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(filePath)) return;
|
||||
var directory = Path.GetDirectoryName(Path.GetFullPath(filePath));
|
||||
var root = roots.Where(x => SafeLocalMigrationFile.IsWithin(filePath, x)).OrderByDescending(x => x.Length).FirstOrDefault();
|
||||
while (!string.IsNullOrWhiteSpace(directory) && !string.Equals(directory, root, StringComparison.Ordinal)
|
||||
&& Directory.Exists(directory) && !Directory.EnumerateFileSystemEntries(directory).Any())
|
||||
{
|
||||
Directory.Delete(directory, false);
|
||||
directory = Path.GetDirectoryName(directory);
|
||||
}
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> DeserializePaths(string json)
|
||||
{
|
||||
try { return JsonConvert.DeserializeObject<List<string>>(json) ?? new(); }
|
||||
catch { return Array.Empty<string>(); }
|
||||
}
|
||||
|
||||
public async Task RefreshCountsAsync(string taskId)
|
||||
{
|
||||
var items = await _db.Queryable<StorageMigrationItem>().Where(x => x.TaskId == taskId).ToListAsync();
|
||||
var pending = items.Count(i => i.Stage == StorageMigrationItemStage.Pending || i.Stage == StorageMigrationItemStage.Uploading
|
||||
|| i.Stage == StorageMigrationItemStage.Verifying || i.Stage == StorageMigrationItemStage.Committing);
|
||||
var success = items.Count(i => i.Stage == StorageMigrationItemStage.Succeeded || i.Stage == StorageMigrationItemStage.SucceededWithWarnings
|
||||
|| i.Stage == StorageMigrationItemStage.Cleaned);
|
||||
var warning = items.Count(i => i.Stage == StorageMigrationItemStage.SucceededWithWarnings);
|
||||
var failed = items.Count(i => i.Stage == StorageMigrationItemStage.Failed);
|
||||
var removed = items.Count(i => i.Stage == StorageMigrationItemStage.RecordRemoved);
|
||||
var now = DateTime.Now;
|
||||
await _db.Updateable<StorageMigrationTask>()
|
||||
.SetColumns(x => new StorageMigrationTask
|
||||
{
|
||||
PendingCount = pending,
|
||||
SuccessCount = success,
|
||||
WarningCount = warning,
|
||||
FailedCount = failed,
|
||||
RemovedCount = removed,
|
||||
UpdatedAt = now
|
||||
}).Where(x => x.Id == taskId).ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
private async Task<FailedRecordRemovalPlan> BuildFailedRecordRemovalPlanAsync()
|
||||
{
|
||||
var activeMigrationCount = await _db.Queryable<StorageMigrationTask>().Where(x =>
|
||||
x.Status == StorageMigrationTaskStatus.Queued || x.Status == StorageMigrationTaskStatus.Running
|
||||
|| x.Status == StorageMigrationTaskStatus.Paused || x.Status == StorageMigrationTaskStatus.Cleaning).CountAsync();
|
||||
var terminalTaskIds = await _db.Queryable<StorageMigrationTask>().Where(x =>
|
||||
x.Status == StorageMigrationTaskStatus.Completed || x.Status == StorageMigrationTaskStatus.PartiallyFailed
|
||||
|| x.Status == StorageMigrationTaskStatus.Cancelled || x.Status == StorageMigrationTaskStatus.Cleaned
|
||||
|| x.Status == StorageMigrationTaskStatus.RolledBack).Select(x => x.Id).ToListAsync();
|
||||
var failedItems = new List<StorageMigrationItem>();
|
||||
foreach (var taskIdBatch in terminalTaskIds.Chunk(500))
|
||||
failedItems.AddRange(await _db.Queryable<StorageMigrationItem>().Where(x => taskIdBatch.Contains(x.TaskId)
|
||||
&& x.Stage == StorageMigrationItemStage.Failed).ToListAsync());
|
||||
var groups = failedItems.GroupBy(x => x.VideoId).OrderBy(x => x.Key, StringComparer.Ordinal)
|
||||
.Select(x => new FailedRecordGroup { VideoId = x.Key, Items = x.OrderBy(i => i.Id, StringComparer.Ordinal).ToList() })
|
||||
.ToList();
|
||||
|
||||
foreach (var group in groups)
|
||||
{
|
||||
foreach (var item in group.Items)
|
||||
{
|
||||
try
|
||||
{
|
||||
var snapshot = JsonConvert.DeserializeObject<DouyinVideo>(item.OldSnapshotJson);
|
||||
if (snapshot != null) group.Snapshots.Add(snapshot);
|
||||
}
|
||||
catch (JsonException) { }
|
||||
}
|
||||
}
|
||||
|
||||
var videoIds = groups.Select(x => x.VideoId).Where(x => !string.IsNullOrWhiteSpace(x)).Distinct().ToList();
|
||||
var currentVideosById = new List<DouyinVideo>();
|
||||
foreach (var videoIdBatch in videoIds.Chunk(500))
|
||||
currentVideosById.AddRange(await _db.Queryable<DouyinVideo>().Where(x => videoIdBatch.Contains(x.Id)).ToListAsync());
|
||||
var currentById = currentVideosById.ToDictionary(x => x.Id, StringComparer.Ordinal);
|
||||
var snapshotAwemeIds = groups.SelectMany(x => x.Snapshots).Select(x => x.AwemeId)
|
||||
.Where(x => !string.IsNullOrWhiteSpace(x)).Distinct().ToList();
|
||||
var currentVideosByAweme = new List<DouyinVideo>();
|
||||
var excludedAwemeIds = new HashSet<string>(StringComparer.Ordinal);
|
||||
foreach (var awemeIdBatch in snapshotAwemeIds.Chunk(500))
|
||||
{
|
||||
currentVideosByAweme.AddRange(await _db.Queryable<DouyinVideo>()
|
||||
.Where(x => awemeIdBatch.Contains(x.AwemeId)).ToListAsync());
|
||||
excludedAwemeIds.UnionWith(await _db.Queryable<DouyinVideoDelete>()
|
||||
.Where(x => awemeIdBatch.Contains(x.ViedoId)).Select(x => x.ViedoId).ToListAsync());
|
||||
}
|
||||
var currentByAweme = currentVideosByAweme.GroupBy(x => x.AwemeId)
|
||||
.ToDictionary(x => x.Key, x => x.ToList(), StringComparer.Ordinal);
|
||||
|
||||
foreach (var group in groups)
|
||||
{
|
||||
var awemeIds = group.Snapshots.Select(x => x.AwemeId).Where(x => !string.IsNullOrWhiteSpace(x))
|
||||
.Distinct(StringComparer.Ordinal).ToList();
|
||||
if (group.Snapshots.Count == 0 || awemeIds.Count != 1
|
||||
|| group.Snapshots.All(x => !string.Equals(x.Id, group.VideoId, StringComparison.Ordinal)))
|
||||
{
|
||||
group.Disposition = FailedRecordDisposition.InvalidSnapshot;
|
||||
continue;
|
||||
}
|
||||
|
||||
group.AwemeId = awemeIds[0];
|
||||
if (excludedAwemeIds.Contains(group.AwemeId))
|
||||
{
|
||||
group.Disposition = FailedRecordDisposition.PermanentlyExcluded;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!currentById.TryGetValue(group.VideoId, out var current))
|
||||
{
|
||||
group.Disposition = currentByAweme.TryGetValue(group.AwemeId, out var replacements)
|
||||
&& replacements.Any(x => x.Id != group.VideoId)
|
||||
? FailedRecordDisposition.Changed
|
||||
: FailedRecordDisposition.AlreadyMissing;
|
||||
continue;
|
||||
}
|
||||
|
||||
group.Current = current;
|
||||
var matchesSnapshot = group.Snapshots.Any(snapshot => current.StorageType == snapshot.StorageType
|
||||
&& string.Equals(current.AwemeId, group.AwemeId, StringComparison.Ordinal)
|
||||
&& string.Equals(snapshot.Id, current.Id, StringComparison.Ordinal)
|
||||
&& string.Equals(snapshot.AwemeId, current.AwemeId, StringComparison.Ordinal)
|
||||
&& string.Equals(snapshot.VideoSavePath, current.VideoSavePath, StringComparison.Ordinal));
|
||||
group.Disposition = matchesSnapshot ? FailedRecordDisposition.Eligible : FailedRecordDisposition.Changed;
|
||||
}
|
||||
|
||||
var tokenSource = string.Join("\n", groups.SelectMany(group => group.Items.Select(item =>
|
||||
$"{item.Id}|{item.TaskId}|{item.VideoId}|{item.UpdatedAt.Ticks}|{(int)group.Disposition}|{group.AwemeId}|{group.Current?.Id}|{(int?)(group.Current?.StorageType)}|{group.Current?.VideoSavePath}")));
|
||||
var token = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(tokenSource))).ToLowerInvariant();
|
||||
return new FailedRecordRemovalPlan
|
||||
{
|
||||
ActiveMigrationCount = activeMigrationCount,
|
||||
Groups = groups,
|
||||
ConfirmationToken = token
|
||||
};
|
||||
}
|
||||
|
||||
private async Task ChangeStatusAsync(string id, StorageMigrationTaskStatus[] allowed, StorageMigrationTaskStatus status, string error)
|
||||
{
|
||||
var task = await RequireTaskAsync(id);
|
||||
if (!allowed.Contains(task.Status)) throw new InvalidOperationException($"任务状态 {task.Status} 不允许此操作。");
|
||||
await _db.Updateable<StorageMigrationTask>()
|
||||
.SetColumns(x => new StorageMigrationTask { Status = status, ErrorMessage = error, UpdatedAt = DateTime.Now })
|
||||
.Where(x => x.Id == id).ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
private async Task<StorageMigrationTask> RequireTaskAsync(string id) =>
|
||||
await _db.Queryable<StorageMigrationTask>().InSingleAsync(id) ?? throw new KeyNotFoundException("迁移任务不存在");
|
||||
|
||||
private async Task<IMediaStorage> BindTaskTargetAsync(StorageMigrationTask task)
|
||||
{
|
||||
if ((task.TargetStorageType ?? MediaStorageType.WebDav) != MediaStorageType.OpenList)
|
||||
throw new InvalidOperationException("旧 WebDAV 迁移任务只保留审计信息，不能再执行写入、清理或回滚。");
|
||||
var settings = await _settingsService.GetAsync();
|
||||
if (!string.Equals(StorageConfigurationFingerprint.Create(settings), task.ConfigurationFingerprint, StringComparison.Ordinal))
|
||||
throw new InvalidOperationException("OpenList 地址、源挂载、基础目录或账号已变化，请恢复任务原目标后再操作。");
|
||||
return _openList.Bind(settings);
|
||||
}
|
||||
|
||||
private async Task<List<PlanningEntry>> BuildPlanningAsync()
|
||||
{
|
||||
var videos = await _db.Queryable<DouyinVideo>()
|
||||
.Where(x => x.StorageType != MediaStorageType.OpenList)
|
||||
.OrderBy(x => x.CreateTime).ToListAsync();
|
||||
var cookies = (await _db.Queryable<DouyinCookie>().ToListAsync()).ToDictionary(x => x.Id ?? string.Empty, StringComparer.Ordinal);
|
||||
var categories = (await _db.Queryable<DouyinCollectCate>().ToListAsync()).ToDictionary(x => x.Id ?? string.Empty, StringComparer.Ordinal);
|
||||
var followed = await _db.Queryable<DouyinFollowed>().ToListAsync();
|
||||
var config = _commonService.GetConfig() ?? new AppConfig();
|
||||
var episodes = videos.Where(x => x.ViedoType is VideoTypeEnum.dy_mix or VideoTypeEnum.dy_series)
|
||||
.GroupBy(x => x.CateId ?? x.CateXId ?? string.Empty)
|
||||
.SelectMany(group => group.OrderBy(x => x.CreateTime).ThenBy(x => x.Id).Select((video, index) => new { video.Id, Episode = index + 1 }))
|
||||
.ToDictionary(x => x.Id, x => x.Episode);
|
||||
var result = new List<PlanningEntry>(videos.Count);
|
||||
var openListSettings = await _settingsService.GetAsync();
|
||||
var targetStorage = _openList.Bind(openListSettings);
|
||||
foreach (var video in videos)
|
||||
{
|
||||
cookies.TryGetValue(video.CookieId ?? string.Empty, out var cookie);
|
||||
categories.TryGetValue(video.CateId ?? string.Empty, out var category);
|
||||
var follow = followed.FirstOrDefault(x => x.UperId == video.AuthorId);
|
||||
StorageMigrationPathPlan plan = null;
|
||||
string planError = null;
|
||||
try
|
||||
{
|
||||
plan = video.StorageType == MediaStorageType.WebDav
|
||||
? new StorageMigrationPathPlan
|
||||
{
|
||||
VideoPath = StoragePath.NormalizeRemote(video.VideoSavePath),
|
||||
CoverPath = string.IsNullOrWhiteSpace(video.VideoCoverSavePath)
|
||||
? string.Empty : StoragePath.NormalizeRemote(video.VideoCoverSavePath),
|
||||
AvatarPath = string.IsNullOrWhiteSpace(video.AuthorAvatar)
|
||||
? string.Empty : StoragePath.NormalizeRemote(video.AuthorAvatar)
|
||||
}
|
||||
: StorageMigrationPathPolicy.Build(video, cookie, category, follow, config,
|
||||
episodes.GetValueOrDefault(video.Id, 1));
|
||||
}
|
||||
catch (Exception ex) { planError = $"视频 {video.AwemeId ?? video.Id}：{ex.Message}"; }
|
||||
|
||||
var roots = StorageMigrationPathPolicy.GetLocalRoots(cookie);
|
||||
var readable = SafeLocalMigrationFile.TryResolve(video.VideoSavePath, roots, out var fullPath, out _)
|
||||
&& new FileInfo(fullPath).Length > 0;
|
||||
long? existingTargetLength = null;
|
||||
if (plan != null && video.StorageType == MediaStorageType.WebDav)
|
||||
{
|
||||
try { existingTargetLength = await targetStorage.GetLengthAsync(plan.VideoPath); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
planError = $"视频 {video.AwemeId ?? video.Id} OpenList 接管检测失败：{ex.GetBaseException().Message}";
|
||||
}
|
||||
}
|
||||
result.Add(new PlanningEntry
|
||||
{
|
||||
Video = video,
|
||||
Cookie = cookie,
|
||||
Category = category,
|
||||
Followed = follow,
|
||||
Plan = plan,
|
||||
PlanError = planError,
|
||||
LocalRoots = roots,
|
||||
LocalReadable = readable,
|
||||
LocalLength = readable ? new FileInfo(fullPath).Length : 0,
|
||||
AdoptExistingTarget = existingTargetLength is > 0
|
||||
&& (video.FileSize <= 0 || existingTargetLength.Value == video.FileSize)
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static List<MigrationAttachmentPlan> BuildAttachments(PlanningEntry entry)
|
||||
{
|
||||
var attachments = new List<MigrationAttachmentPlan>();
|
||||
Add(attachments, "cover", entry.Video.VideoCoverSavePath, entry.Video.VideoCoverUrl, entry.Plan.CoverPath, false);
|
||||
Add(attachments, "avatar", entry.Video.AuthorAvatar, entry.Video.AuthorAvatarUrl, entry.Plan.AvatarPath, true);
|
||||
if (!string.IsNullOrWhiteSpace(entry.Video.DynamicVideos))
|
||||
{
|
||||
try
|
||||
{
|
||||
var dynamicItems = JsonConvert.DeserializeObject<List<DouyinMergeVideoDto>>(entry.Video.DynamicVideos) ?? new();
|
||||
var index = 0;
|
||||
foreach (var item in dynamicItems)
|
||||
{
|
||||
index++;
|
||||
var oldName = Path.GetFileName(item.Path);
|
||||
var name = string.IsNullOrWhiteSpace(oldName) ? $"attachment-{index:D3}.bin" : oldName;
|
||||
Add(attachments, "dynamic", item.Path, Uri.IsWellFormedUriString(item.Path, UriKind.Absolute) ? item.Path : null,
|
||||
StoragePath.CombineRemote(entry.Plan.DirectoryPath, name), false, item.Height, item.Width);
|
||||
}
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
Serilog.Log.Warning(ex, "解析迁移附件失败：{VideoId}", entry.Video.Id);
|
||||
}
|
||||
}
|
||||
return attachments;
|
||||
}
|
||||
|
||||
private static void Add(List<MigrationAttachmentPlan> list, string kind, string source, string sourceUrl, string target, bool shared, int height = 0, int width = 0)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(target)) return;
|
||||
list.Add(new MigrationAttachmentPlan { Kind = kind, SourcePath = source, SourceUrl = sourceUrl, TargetPath = target, Shared = shared, Height = height, Width = width });
|
||||
}
|
||||
|
||||
private static StorageMigrationTaskDetail ToDetail(StorageMigrationTask task)
|
||||
{
|
||||
var isOpenListTask = task.TargetStorageType == MediaStorageType.OpenList;
|
||||
return new StorageMigrationTaskDetail
|
||||
{
|
||||
Task = task,
|
||||
StatusText = task.Status switch
|
||||
{
|
||||
StorageMigrationTaskStatus.Queued => "排队",
|
||||
StorageMigrationTaskStatus.Running => "运行中",
|
||||
StorageMigrationTaskStatus.Paused => "已暂停",
|
||||
StorageMigrationTaskStatus.Completed => "已完成",
|
||||
StorageMigrationTaskStatus.PartiallyFailed => "部分失败",
|
||||
StorageMigrationTaskStatus.Cancelled => "已取消",
|
||||
StorageMigrationTaskStatus.Cleaning => "清理中",
|
||||
StorageMigrationTaskStatus.Cleaned => "已清理",
|
||||
StorageMigrationTaskStatus.RolledBack => "已回滚",
|
||||
_ => task.Status.ToString()
|
||||
},
|
||||
CurrentFile = task.CurrentFile,
|
||||
CanPause = isOpenListTask && task.Status is (StorageMigrationTaskStatus.Queued or StorageMigrationTaskStatus.Running),
|
||||
CanResume = isOpenListTask && task.Status == StorageMigrationTaskStatus.Paused,
|
||||
CanCancel = isOpenListTask && task.Status is (StorageMigrationTaskStatus.Queued or StorageMigrationTaskStatus.Running or StorageMigrationTaskStatus.Paused),
|
||||
CanRetryFailed = isOpenListTask && task.FailedCount > 0
|
||||
&& task.Status is (StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled or StorageMigrationTaskStatus.Paused),
|
||||
CanCleanup = isOpenListTask && task.SuccessCount > 0
|
||||
&& task.Status is (StorageMigrationTaskStatus.Completed or StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled),
|
||||
CanRollback = isOpenListTask && task.SuccessCount > 0 && !task.CleanedAt.HasValue
|
||||
&& task.Status is (StorageMigrationTaskStatus.Completed or StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled),
|
||||
CanArchive = CanArchive(task)
|
||||
};
|
||||
}
|
||||
|
||||
private static bool CanArchive(StorageMigrationTask task) => task != null && !task.IsArchived
|
||||
&& IsTerminal(task.Status)
|
||||
&& (task.TargetStorageType != MediaStorageType.OpenList
|
||||
|| task.Status is StorageMigrationTaskStatus.Cleaned or StorageMigrationTaskStatus.RolledBack
|
||||
|| task.SuccessCount == 0);
|
||||
|
||||
private static bool IsTerminal(StorageMigrationTaskStatus status) => status is
|
||||
StorageMigrationTaskStatus.Completed or StorageMigrationTaskStatus.PartiallyFailed
|
||||
or StorageMigrationTaskStatus.Cancelled or StorageMigrationTaskStatus.Cleaned
|
||||
or StorageMigrationTaskStatus.RolledBack;
|
||||
|
||||
private static string FormatBytes(long value)
|
||||
{
|
||||
string[] units = { "B", "KB", "MB", "GB", "TB" };
|
||||
var size = (double)value;
|
||||
var unit = 0;
|
||||
while (size >= 1024 && unit < units.Length - 1) { size /= 1024; unit++; }
|
||||
return $"{size:0.##} {units[unit]}";
|
||||
}
|
||||
|
||||
private sealed class PlanningEntry
|
||||
{
|
||||
public DouyinVideo Video { get; set; }
|
||||
public DouyinCookie Cookie { get; set; }
|
||||
public DouyinCollectCate Category { get; set; }
|
||||
public DouyinFollowed Followed { get; set; }
|
||||
public StorageMigrationPathPlan Plan { get; set; }
|
||||
public string PlanError { get; set; }
|
||||
public IReadOnlyList<string> LocalRoots { get; set; }
|
||||
public bool LocalReadable { get; set; }
|
||||
public long LocalLength { get; set; }
|
||||
public bool AdoptExistingTarget { get; set; }
|
||||
}
|
||||
|
||||
internal sealed class MigrationAttachmentPlan
|
||||
{
|
||||
public string Kind { get; set; }
|
||||
public string SourcePath { get; set; }
|
||||
public string SourceUrl { get; set; }
|
||||
public string TargetPath { get; set; }
|
||||
public bool Shared { get; set; }
|
||||
public int Height { get; set; }
|
||||
public int Width { get; set; }
|
||||
}
|
||||
|
||||
private enum FailedRecordDisposition
|
||||
{
|
||||
Eligible,
|
||||
AlreadyMissing,
|
||||
Changed,
|
||||
PermanentlyExcluded,
|
||||
InvalidSnapshot
|
||||
}
|
||||
|
||||
private sealed class FailedRecordRemovalPlan
|
||||
{
|
||||
public int ActiveMigrationCount { get; set; }
|
||||
public List<FailedRecordGroup> Groups { get; set; } = new();
|
||||
public string ConfirmationToken { get; set; }
|
||||
}
|
||||
|
||||
private sealed class FailedRecordGroup
|
||||
{
|
||||
public string VideoId { get; set; }
|
||||
public string AwemeId { get; set; }
|
||||
public DouyinVideo Current { get; set; }
|
||||
public List<DouyinVideo> Snapshots { get; set; } = new();
|
||||
public List<StorageMigrationItem> Items { get; set; } = new();
|
||||
public FailedRecordDisposition Disposition { get; set; }
|
||||
}
|
||||
|
||||
private sealed record CleanupCandidate(string Path, bool Shared);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user