370 lines
19 KiB
C#
370 lines
19 KiB
C#
using dy.net.model.dto;
|
|
using dy.net.model.entity;
|
|
using dy.net.storage;
|
|
using SqlSugar;
|
|
using MediaStorageType = dy.net.model.dto.StorageType;
|
|
|
|
namespace dy.net.service
|
|
{
|
|
public sealed class OpenListDirectoryRepairWorker : BackgroundService
|
|
{
|
|
private const int InspectionBatchSize = 4;
|
|
private readonly IServiceScopeFactory _scopeFactory;
|
|
private readonly ILogger<OpenListDirectoryRepairWorker> _logger;
|
|
|
|
public OpenListDirectoryRepairWorker(
|
|
IServiceScopeFactory scopeFactory,
|
|
ILogger<OpenListDirectoryRepairWorker> logger)
|
|
{
|
|
_scopeFactory = scopeFactory;
|
|
_logger = logger;
|
|
}
|
|
|
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
|
{
|
|
await RecoverInterruptedAsync();
|
|
while (!stoppingToken.IsCancellationRequested)
|
|
{
|
|
try
|
|
{
|
|
if (!await RunOneAsync(stoppingToken))
|
|
await Task.Delay(TimeSpan.FromSeconds(2), stoppingToken);
|
|
}
|
|
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { }
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "OpenList 目录修复后台服务异常");
|
|
await Task.Delay(TimeSpan.FromSeconds(3), stoppingToken);
|
|
}
|
|
}
|
|
}
|
|
|
|
private async Task RecoverInterruptedAsync()
|
|
{
|
|
using var scope = _scopeFactory.CreateScope();
|
|
var db = scope.ServiceProvider.GetRequiredService<ISqlSugarClient>();
|
|
var now = DateTime.Now;
|
|
await db.Updateable<OpenListDirectoryRepairItem>()
|
|
.SetColumns(x => new OpenListDirectoryRepairItem
|
|
{
|
|
Status = OpenListDirectoryRepairItemStatus.Pending,
|
|
ErrorMessage = "应用重启，目录将在恢复后重新检查。",
|
|
UpdatedAt = now
|
|
})
|
|
.Where(x => x.Status == OpenListDirectoryRepairItemStatus.Inspecting).ExecuteCommandAsync();
|
|
await db.Updateable<OpenListDirectoryRepairItem>()
|
|
.SetColumns(x => new OpenListDirectoryRepairItem
|
|
{
|
|
Status = OpenListDirectoryRepairItemStatus.EmptyConfirmed,
|
|
ErrorMessage = "应用在删除确认后重启，需再次确认清理。",
|
|
UpdatedAt = now
|
|
})
|
|
.Where(x => x.Status == OpenListDirectoryRepairItemStatus.Deleting).ExecuteCommandAsync();
|
|
var interrupted = await db.Queryable<OpenListDirectoryRepairTask>().Where(x =>
|
|
x.Status == OpenListDirectoryRepairStatus.Scanning
|
|
|| x.Status == OpenListDirectoryRepairStatus.Cleaning).ToListAsync();
|
|
foreach (var task in interrupted)
|
|
{
|
|
task.Status = task.Status == OpenListDirectoryRepairStatus.Cleaning
|
|
? OpenListDirectoryRepairStatus.AwaitingConfirmation
|
|
: OpenListDirectoryRepairStatus.Queued;
|
|
task.CurrentDirectory = null;
|
|
task.ErrorMessage = task.Status == OpenListDirectoryRepairStatus.AwaitingConfirmation
|
|
? "应用重启后已停止删除，请重新确认。"
|
|
: "应用重启后已恢复扫描。";
|
|
task.UpdatedAt = now;
|
|
await db.Updateable(task).ExecuteCommandAsync();
|
|
await scope.ServiceProvider.GetRequiredService<OpenListDirectoryRepairService>()
|
|
.RefreshCountsAsync(task.Id);
|
|
}
|
|
}
|
|
|
|
private async Task<bool> RunOneAsync(CancellationToken cancellationToken)
|
|
{
|
|
using var scope = _scopeFactory.CreateScope();
|
|
var db = scope.ServiceProvider.GetRequiredService<ISqlSugarClient>();
|
|
var service = scope.ServiceProvider.GetRequiredService<OpenListDirectoryRepairService>();
|
|
var task = await db.Queryable<OpenListDirectoryRepairTask>().Where(x =>
|
|
x.Status == OpenListDirectoryRepairStatus.Queued
|
|
|| x.Status == OpenListDirectoryRepairStatus.Scanning
|
|
|| x.Status == OpenListDirectoryRepairStatus.Cleaning)
|
|
.OrderBy(x => x.CreatedAt).FirstAsync();
|
|
if (task == null) return false;
|
|
try
|
|
{
|
|
await service.EnsureFingerprintAsync(task);
|
|
}
|
|
catch (InvalidOperationException ex)
|
|
{
|
|
task.Status = OpenListDirectoryRepairStatus.Paused;
|
|
task.ErrorMessage = ex.Message;
|
|
task.CurrentDirectory = null;
|
|
task.UpdatedAt = DateTime.Now;
|
|
await db.Updateable(task).ExecuteCommandAsync();
|
|
return true;
|
|
}
|
|
|
|
if (task.Status == OpenListDirectoryRepairStatus.Queued)
|
|
{
|
|
task.Status = OpenListDirectoryRepairStatus.Scanning;
|
|
task.StartedAt ??= DateTime.Now;
|
|
task.CompletedAt = null;
|
|
task.UpdatedAt = DateTime.Now;
|
|
await db.Updateable(task).ExecuteCommandAsync();
|
|
}
|
|
if (task.Status == OpenListDirectoryRepairStatus.Scanning)
|
|
return await InspectOneAsync(scope.ServiceProvider, task, cancellationToken);
|
|
return await DeleteOneAsync(scope.ServiceProvider, task, cancellationToken);
|
|
}
|
|
|
|
private static async Task<bool> InspectOneAsync(
|
|
IServiceProvider services,
|
|
OpenListDirectoryRepairTask task,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var db = services.GetRequiredService<ISqlSugarClient>();
|
|
var service = services.GetRequiredService<OpenListDirectoryRepairService>();
|
|
var client = services.GetRequiredService<OpenListClient>();
|
|
var items = await db.Queryable<OpenListDirectoryRepairItem>().Where(x =>
|
|
x.TaskId == task.Id && x.Status == OpenListDirectoryRepairItemStatus.Pending)
|
|
.OrderBy(x => x.CreatedAt).Take(InspectionBatchSize).ToListAsync();
|
|
if (items.Count == 0)
|
|
{
|
|
await CompleteScanAsync(services, task.Id);
|
|
return true;
|
|
}
|
|
|
|
var startedAt = DateTime.Now;
|
|
foreach (var item in items)
|
|
{
|
|
item.Status = OpenListDirectoryRepairItemStatus.Inspecting;
|
|
item.Attempts++;
|
|
item.ErrorMessage = null;
|
|
item.UpdatedAt = startedAt;
|
|
}
|
|
task.CurrentDirectory = items[0].ActualPath;
|
|
task.UpdatedAt = startedAt;
|
|
await db.Updateable(items).ExecuteCommandAsync();
|
|
await db.Updateable(task).ExecuteCommandAsync();
|
|
var (settings, password) = await service.GetConnectionAsync();
|
|
await Task.WhenAll(items.Select(async item =>
|
|
{
|
|
try
|
|
{
|
|
var inspection = await client.InspectKnownDirectoryAsync(
|
|
settings, password, item.ActualPath, true, cancellationToken);
|
|
item.EntryCount = inspection.EntryCount;
|
|
item.Status = !inspection.Exists
|
|
? OpenListDirectoryRepairItemStatus.Missing
|
|
: inspection.EntryCount == 0
|
|
? OpenListDirectoryRepairItemStatus.EmptyConfirmed
|
|
: OpenListDirectoryRepairItemStatus.SkippedNonEmpty;
|
|
item.ErrorMessage = inspection.Exists && inspection.EntryCount > 0
|
|
? $"目录包含 {inspection.EntryCount} 个对象，已安全跳过。" : null;
|
|
}
|
|
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
|
|
catch (Exception ex)
|
|
{
|
|
item.Status = OpenListDirectoryRepairItemStatus.Failed;
|
|
item.ErrorMessage = ex.GetBaseException().Message;
|
|
}
|
|
item.CompletedAt = DateTime.Now;
|
|
item.UpdatedAt = item.CompletedAt.Value;
|
|
}));
|
|
await db.Updateable(items).ExecuteCommandAsync();
|
|
await service.RefreshCountsAsync(task.Id);
|
|
return true;
|
|
}
|
|
|
|
private static async Task CompleteScanAsync(IServiceProvider services, string taskId)
|
|
{
|
|
var db = services.GetRequiredService<ISqlSugarClient>();
|
|
var service = services.GetRequiredService<OpenListDirectoryRepairService>();
|
|
await service.RefreshCountsAsync(taskId);
|
|
var task = await db.Queryable<OpenListDirectoryRepairTask>().InSingleAsync(taskId);
|
|
if (task?.Status != OpenListDirectoryRepairStatus.Scanning) return;
|
|
task.Status = task.EmptyCount > 0
|
|
? OpenListDirectoryRepairStatus.AwaitingConfirmation
|
|
: task.FailedCount > 0
|
|
? OpenListDirectoryRepairStatus.PartiallyFailed
|
|
: OpenListDirectoryRepairStatus.Completed;
|
|
task.CurrentDirectory = null;
|
|
task.ScanCompletedAt = DateTime.Now;
|
|
task.CompletedAt = task.Status is OpenListDirectoryRepairStatus.Completed
|
|
or OpenListDirectoryRepairStatus.PartiallyFailed ? DateTime.Now : null;
|
|
task.ErrorMessage = task.FailedCount > 0 ? $"{task.FailedCount} 个目录检查失败，未执行删除。" : null;
|
|
task.UpdatedAt = DateTime.Now;
|
|
await db.Updateable(task).ExecuteCommandAsync();
|
|
}
|
|
|
|
private static async Task<bool> DeleteOneAsync(
|
|
IServiceProvider services,
|
|
OpenListDirectoryRepairTask task,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var db = services.GetRequiredService<ISqlSugarClient>();
|
|
var service = services.GetRequiredService<OpenListDirectoryRepairService>();
|
|
var client = services.GetRequiredService<OpenListClient>();
|
|
var item = await db.Queryable<OpenListDirectoryRepairItem>().Where(x =>
|
|
x.TaskId == task.Id && x.Status == OpenListDirectoryRepairItemStatus.EmptyConfirmed)
|
|
.OrderBy(x => x.CreatedAt).FirstAsync();
|
|
if (item == null)
|
|
{
|
|
await CompleteCleanupAsync(services, task.Id, cancellationToken);
|
|
return true;
|
|
}
|
|
|
|
item.Status = OpenListDirectoryRepairItemStatus.Deleting;
|
|
item.Attempts++;
|
|
item.ErrorMessage = null;
|
|
item.UpdatedAt = DateTime.Now;
|
|
task.CurrentDirectory = item.ActualPath;
|
|
task.UpdatedAt = item.UpdatedAt;
|
|
await db.Updateable(item).ExecuteCommandAsync();
|
|
await db.Updateable(task).ExecuteCommandAsync();
|
|
try
|
|
{
|
|
var (settings, password) = await service.GetConnectionAsync();
|
|
var before = await client.InspectKnownDirectoryAsync(settings, password, item.ActualPath, true, cancellationToken);
|
|
if (!before.Exists)
|
|
{
|
|
item.Status = OpenListDirectoryRepairItemStatus.Missing;
|
|
}
|
|
else if (before.EntryCount > 0)
|
|
{
|
|
item.EntryCount = before.EntryCount;
|
|
item.Status = OpenListDirectoryRepairItemStatus.SkippedNonEmpty;
|
|
item.ErrorMessage = $"删除前发现目录已有 {before.EntryCount} 个对象，已安全跳过。";
|
|
}
|
|
else
|
|
{
|
|
await client.DeleteKnownObjectAsync(settings, password, item.ActualPath, cancellationToken);
|
|
var after = await client.InspectKnownDirectoryAsync(settings, password, item.ActualPath, true, cancellationToken);
|
|
if (after.Exists) throw new IOException("OpenList 返回删除成功，但目录仍然存在。");
|
|
item.Status = OpenListDirectoryRepairItemStatus.Deleted;
|
|
}
|
|
}
|
|
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
|
|
catch (Exception ex)
|
|
{
|
|
item.Status = OpenListDirectoryRepairItemStatus.Failed;
|
|
item.ErrorMessage = ex.GetBaseException().Message;
|
|
}
|
|
item.CompletedAt = DateTime.Now;
|
|
item.UpdatedAt = item.CompletedAt.Value;
|
|
await db.Updateable(item).ExecuteCommandAsync();
|
|
await service.RefreshCountsAsync(task.Id);
|
|
return true;
|
|
}
|
|
|
|
private static async Task CompleteCleanupAsync(
|
|
IServiceProvider services,
|
|
string taskId,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var db = services.GetRequiredService<ISqlSugarClient>();
|
|
var service = services.GetRequiredService<OpenListDirectoryRepairService>();
|
|
await service.RefreshCountsAsync(taskId);
|
|
var task = await db.Queryable<OpenListDirectoryRepairTask>().InSingleAsync(taskId);
|
|
if (task?.Status != OpenListDirectoryRepairStatus.Cleaning) return;
|
|
task.Status = task.FailedCount > 0
|
|
? OpenListDirectoryRepairStatus.PartiallyFailed
|
|
: OpenListDirectoryRepairStatus.Completed;
|
|
task.CurrentDirectory = null;
|
|
task.CompletedAt = DateTime.Now;
|
|
task.ErrorMessage = task.FailedCount > 0 ? $"{task.FailedCount} 个目录清理失败，可重试。" : null;
|
|
task.UpdatedAt = DateTime.Now;
|
|
await db.Updateable(task).ExecuteCommandAsync();
|
|
await RecoverAffectedTasksAsync(services, task, cancellationToken);
|
|
}
|
|
|
|
private static async Task RecoverAffectedTasksAsync(
|
|
IServiceProvider services,
|
|
OpenListDirectoryRepairTask repair,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var db = services.GetRequiredService<ISqlSugarClient>();
|
|
var videos = await db.Queryable<DouyinVideo>().Where(x => x.StorageType == MediaStorageType.OpenList
|
|
&& x.VideoSavePath.StartsWith(repair.RequestedPath + "/")).ToListAsync();
|
|
var storage = services.GetRequiredService<OpenListMediaStorage>();
|
|
foreach (var video in videos)
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
var canonicalMain = ReplacePrefix(video.VideoSavePath, repair.RequestedPath, repair.CanonicalPath);
|
|
var length = await storage.GetLengthAsync(canonicalMain, cancellationToken);
|
|
if (!length.HasValue || length <= 0 || video.FileSize > 0 && length.Value != video.FileSize) continue;
|
|
video.VideoSavePath = canonicalMain;
|
|
video.VideoCoverSavePath = ReplacePrefix(video.VideoCoverSavePath, repair.RequestedPath, repair.CanonicalPath);
|
|
video.AuthorAvatar = ReplacePrefix(video.AuthorAvatar, repair.RequestedPath, repair.CanonicalPath);
|
|
if (!string.IsNullOrWhiteSpace(video.DynamicVideos))
|
|
video.DynamicVideos = video.DynamicVideos.Replace(repair.RequestedPath + "/",
|
|
repair.CanonicalPath + "/", StringComparison.Ordinal);
|
|
await db.Updateable(video).ExecuteCommandAsync();
|
|
}
|
|
|
|
var failedItems = await db.Queryable<VideoDownloadTaskItem>().Where(x =>
|
|
(x.Stage == VideoTaskItemStage.Failed || x.Stage == VideoTaskItemStage.WaitingForStorage)
|
|
&& x.RetrySnapshotJson != null
|
|
&& (x.ErrorType == VideoTaskErrorType.StorageUnavailable
|
|
|| x.ErrorType == VideoTaskErrorType.IntegrityCheckFailed)).ToListAsync();
|
|
var taskIds = new HashSet<string>(StringComparer.Ordinal);
|
|
foreach (var item in failedItems)
|
|
{
|
|
if (!(item.TargetPath?.StartsWith(repair.RequestedPath + "/", StringComparison.Ordinal) == true
|
|
|| item.ErrorMessage?.Contains("Operation canceled", StringComparison.OrdinalIgnoreCase) == true
|
|
|| item.ErrorMessage?.Contains("超时", StringComparison.Ordinal) == true
|
|
|| item.ErrorType == VideoTaskErrorType.IntegrityCheckFailed)) continue;
|
|
item.TargetPath = ReplacePrefix(item.TargetPath, repair.RequestedPath, repair.CanonicalPath);
|
|
item.RetrySnapshotJson = item.RetrySnapshotJson.Replace(repair.RequestedPath + "/",
|
|
repair.CanonicalPath + "/", StringComparison.Ordinal);
|
|
item.Stage = VideoTaskItemStage.Pending;
|
|
item.ErrorType = VideoTaskErrorType.None;
|
|
item.ErrorMessage = null;
|
|
item.CompletedAt = null;
|
|
item.WorkerManaged = true;
|
|
item.UpdatedAt = DateTime.Now;
|
|
taskIds.Add(item.TaskId);
|
|
}
|
|
if (failedItems.Count > 0) await db.Updateable(failedItems).ExecuteCommandAsync();
|
|
foreach (var taskId in taskIds)
|
|
{
|
|
await db.Updateable<VideoDownloadTask>()
|
|
.SetColumns(x => new VideoDownloadTask
|
|
{
|
|
Status = VideoTaskStatus.Queued,
|
|
CompletedAt = null,
|
|
ErrorMessage = null,
|
|
UpdatedAt = DateTime.Now
|
|
}).Where(x => x.Id == taskId).ExecuteCommandAsync();
|
|
await services.GetRequiredService<VideoTaskService>().RefreshCountsAsync(taskId);
|
|
}
|
|
|
|
try { _ = await services.GetRequiredService<VideoTaskService>().ProbeStorageAsync(cancellationToken); }
|
|
catch (Exception ex) { Serilog.Log.Warning(ex, "目录修复后 OpenList 健康检测失败"); }
|
|
try
|
|
{
|
|
var migrations = await db.Queryable<StorageMigrationTask>().Where(x =>
|
|
x.TargetStorageType == MediaStorageType.OpenList
|
|
&& x.Status == StorageMigrationTaskStatus.PartiallyFailed).ToListAsync();
|
|
foreach (var migration in migrations)
|
|
await services.GetRequiredService<StorageMigrationService>().RetryFailedAsync(migration.Id);
|
|
}
|
|
catch (Exception ex) { Serilog.Log.Warning(ex, "目录修复后迁移失败项自动重试未能提交"); }
|
|
try
|
|
{
|
|
_ = await services.GetRequiredService<DouyinQuartzJobService>()
|
|
.TriggerVideoJobsNowAsync(VideoTypeEnum.dy_follows);
|
|
}
|
|
catch (Exception ex) { Serilog.Log.Warning(ex, "目录修复后关注同步未能自动提交"); }
|
|
}
|
|
|
|
private static string ReplacePrefix(string value, string oldPrefix, string newPrefix)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(value)) return value;
|
|
return value.StartsWith(oldPrefix + "/", StringComparison.Ordinal)
|
|
? newPrefix + value[oldPrefix.Length..]
|
|
: value;
|
|
}
|
|
}
|
|
}
|