1153 lines
58 KiB
C#
1153 lines
58 KiB
C#
using dy.net.model.dto;
|
|
using dy.net.model.entity;
|
|
using dy.net.repository;
|
|
using dy.net.service;
|
|
using dy.net.storage;
|
|
using dy.net.Tests.TestInfrastructure;
|
|
using Microsoft.AspNetCore.DataProtection;
|
|
using Newtonsoft.Json;
|
|
using SqlSugar;
|
|
using MediaStorageType = dy.net.model.dto.StorageType;
|
|
|
|
namespace dy.net.Tests;
|
|
|
|
public class VideoTaskServiceTests
|
|
{
|
|
[Fact]
|
|
public async Task TaskLifecycle_RefreshesStageCountsAndCompletesPartiallyFailed()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
var task = await host.Tasks.CreateTaskAsync(VideoTaskType.Sync, VideoTaskTrigger.Manual, "手动同步");
|
|
await host.Tasks.StartTaskAsync(task.Id);
|
|
var succeeded = await host.Tasks.AddItemAsync(task.Id, "a", "cookie", "账号", VideoTypeEnum.dy_favorite, "成功", "作者");
|
|
var failed = await host.Tasks.AddItemAsync(task.Id, "b", "cookie", "账号", VideoTypeEnum.dy_favorite, "失败", "作者");
|
|
|
|
await host.Tasks.MarkSucceededAsync(succeeded.Id, 128, "video-a");
|
|
await host.Tasks.MarkFailedAsync(failed.Id, new InvalidOperationException("作品已下架"), VideoTaskErrorType.SourceUnavailable);
|
|
await host.Tasks.MarkSkippedAsync(task.Id, "c", "cookie", "账号", VideoTypeEnum.dy_favorite,
|
|
"跳过", "作者", VideoTaskSkipReason.AlreadyExists, "目标已存在");
|
|
await host.Tasks.CompleteTaskAsync(task.Id);
|
|
|
|
var detail = await host.Tasks.GetTaskAsync(VideoTaskType.Sync, task.Id);
|
|
Assert.Equal(VideoTaskStatus.PartiallyFailed, detail.Status);
|
|
Assert.Equal(3, detail.TotalCount);
|
|
Assert.Equal(1, detail.SuccessCount);
|
|
Assert.Equal(1, detail.FailedCount);
|
|
Assert.Equal(1, detail.SkippedCount);
|
|
Assert.Equal(0, detail.PendingCount);
|
|
Assert.Equal(0, detail.RunningCount);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task FailedImageItemWithoutSnapshot_DoesNotExposeBrokenRetry()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
var task = await host.Tasks.CreateTaskAsync(VideoTaskType.Sync, VideoTaskTrigger.Manual, "图文同步");
|
|
var item = await host.Tasks.AddItemAsync(task.Id, "image-aweme", "cookie", "账号",
|
|
VideoTypeEnum.dy_follows, "图文作品", "作者");
|
|
await host.Tasks.MarkFailedAsync(item.Id, new IOException("图文合成失败"), VideoTaskErrorType.SourceUnavailable);
|
|
|
|
var page = await host.Tasks.GetItemsAsync(VideoTaskType.Sync, task.Id,
|
|
new VideoTaskItemPageRequest { PageIndex = 1, PageSize = 20, Stage = VideoTaskItemStage.Failed });
|
|
|
|
Assert.False(Assert.Single(page.Items).CanRetry);
|
|
var error = await Assert.ThrowsAsync<InvalidOperationException>(() =>
|
|
host.Tasks.RetryItemAsync(VideoTaskType.Sync, task.Id, item.Id));
|
|
Assert.Contains("重新执行对应的同步任务", error.Message);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task RetryFailed_RetriesOnlyItemsWithRecoverySnapshots()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
var task = await host.Tasks.CreateTaskAsync(VideoTaskType.Sync, VideoTaskTrigger.Manual, "混合失败");
|
|
var image = await host.Tasks.AddItemAsync(task.Id, "image", "cookie", "账号",
|
|
VideoTypeEnum.dy_follows, "图文", "作者");
|
|
var video = await host.Tasks.AddItemAsync(task.Id, "video", "cookie", "账号",
|
|
VideoTypeEnum.dy_follows, "视频", "作者", "/follow/video.mp4",
|
|
new[] { "https://media.test/video.mp4" },
|
|
new DouyinVideo
|
|
{
|
|
Id = "video-record",
|
|
AwemeId = "video",
|
|
CookieId = "cookie",
|
|
VideoUrl = "https://media.test/video.mp4",
|
|
VideoSavePath = "/follow/video.mp4",
|
|
StorageType = MediaStorageType.WebDav
|
|
});
|
|
await host.Tasks.MarkFailedAsync(image.Id, new IOException("图文失败"), VideoTaskErrorType.SourceUnavailable);
|
|
await host.Tasks.MarkFailedAsync(video.Id, new IOException("写入失败"), VideoTaskErrorType.StorageUnavailable);
|
|
|
|
await host.Tasks.RetryFailedAsync(VideoTaskType.Sync, task.Id);
|
|
|
|
image = await host.Database.Queryable<VideoDownloadTaskItem>().InSingleAsync(image.Id);
|
|
video = await host.Database.Queryable<VideoDownloadTaskItem>().InSingleAsync(video.Id);
|
|
Assert.Equal(VideoTaskItemStage.Failed, image.Stage);
|
|
Assert.False(image.WorkerManaged);
|
|
Assert.Equal(VideoTaskItemStage.Pending, video.Stage);
|
|
Assert.True(video.WorkerManaged);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task StorageFailure_PreservesSafeUnderlyingVerificationReason()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
var task = await host.Tasks.CreateTaskAsync(VideoTaskType.Sync, VideoTaskTrigger.Manual, "存储错误详情");
|
|
var item = await host.Tasks.AddItemAsync(task.Id, "aweme", "cookie", "账号", null, "作品", "作者");
|
|
var error = new MediaStorageException("WebDav 媒体写入失败：/follow/video.mp4",
|
|
new IOException("WebDAV MOVE 后等待约 46.2 秒，最终文件仍返回旧长度或长度不一致：期望 20，实际 10"));
|
|
|
|
await host.Tasks.MarkFailedAsync(item.Id, error);
|
|
|
|
item = await host.Database.Queryable<VideoDownloadTaskItem>().InSingleAsync(item.Id);
|
|
Assert.Equal(VideoTaskErrorType.StorageUnavailable, item.ErrorType);
|
|
Assert.Contains("媒体写入失败", item.ErrorMessage);
|
|
Assert.Contains("等待约 46.2 秒", item.ErrorMessage);
|
|
Assert.Contains("期望 20，实际 10", item.ErrorMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task CompleteTask_WithUnhandledTaskError_IsPersistedAsFailed()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
var task = await host.Tasks.CreateTaskAsync(VideoTaskType.Sync, VideoTaskTrigger.Scheduled, "定时同步");
|
|
await host.Tasks.StartTaskAsync(task.Id);
|
|
|
|
await host.Tasks.CompleteTaskAsync(task.Id, "调度执行失败");
|
|
|
|
var detail = await host.Tasks.GetTaskAsync(VideoTaskType.Sync, task.Id);
|
|
Assert.Equal(VideoTaskStatus.Failed, detail.Status);
|
|
Assert.Equal("调度执行失败", detail.ErrorMessage);
|
|
Assert.NotNull(detail.CompletedAt);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task CompleteTask_WithSuccessAndUnhandledTaskError_IsPersistedAsPartiallyFailed()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
var task = await host.Tasks.CreateTaskAsync(VideoTaskType.Sync, VideoTaskTrigger.Scheduled, "定时同步");
|
|
await host.Tasks.StartTaskAsync(task.Id);
|
|
var item = await host.Tasks.AddItemAsync(task.Id, "aweme", "cookie", "账号", null, "作品", "作者");
|
|
await host.Tasks.MarkSucceededAsync(item.Id, 128, "video");
|
|
|
|
await host.Tasks.CompleteTaskAsync(task.Id, "后续处理失败");
|
|
|
|
var detail = await host.Tasks.GetTaskAsync(VideoTaskType.Sync, task.Id);
|
|
Assert.Equal(VideoTaskStatus.PartiallyFailed, detail.Status);
|
|
Assert.Equal("后续处理失败", detail.ErrorMessage);
|
|
Assert.Equal(1, detail.SuccessCount);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ScheduledEmptyStorageWait_ReusesSameTaskWithoutAffectingManualRuns()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
var first = await host.Tasks.CreateTaskAsync(
|
|
VideoTaskType.Sync,
|
|
VideoTaskTrigger.Scheduled,
|
|
"关注同步",
|
|
VideoTypeEnum.dy_follows);
|
|
await host.Tasks.BlockTaskForStorageAsync(first.Id, "存储不可用");
|
|
|
|
var reused = await host.Tasks.CreateTaskAsync(
|
|
VideoTaskType.Sync,
|
|
VideoTaskTrigger.Scheduled,
|
|
"关注同步",
|
|
VideoTypeEnum.dy_follows);
|
|
var otherType = await host.Tasks.CreateTaskAsync(
|
|
VideoTaskType.Sync,
|
|
VideoTaskTrigger.Scheduled,
|
|
"收藏同步",
|
|
VideoTypeEnum.dy_collects);
|
|
var manual = await host.Tasks.CreateTaskAsync(
|
|
VideoTaskType.Sync,
|
|
VideoTaskTrigger.Manual,
|
|
"关注手动同步",
|
|
VideoTypeEnum.dy_follows);
|
|
|
|
Assert.Equal(first.Id, reused.Id);
|
|
Assert.Equal(VideoTaskStatus.Queued, reused.Status);
|
|
Assert.NotEqual(first.Id, otherType.Id);
|
|
Assert.NotEqual(first.Id, manual.Id);
|
|
Assert.Equal(3, await host.Database.Queryable<VideoDownloadTask>().CountAsync());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task TaskSummary_CountsAllTasksBeyondFirstHundred()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
var now = DateTime.Now;
|
|
var completed = Enumerable.Range(0, 105).Select(index => new VideoDownloadTask
|
|
{
|
|
Id = $"completed-{index}",
|
|
Type = VideoTaskType.Sync,
|
|
Trigger = VideoTaskTrigger.Scheduled,
|
|
Status = VideoTaskStatus.Completed,
|
|
StorageType = MediaStorageType.Local,
|
|
StorageFingerprint = "local",
|
|
Title = "历史完成任务",
|
|
CreatedAt = now.AddMinutes(-index),
|
|
UpdatedAt = now.AddMinutes(-index),
|
|
CompletedAt = now.AddMinutes(-index)
|
|
}).ToList();
|
|
await host.Database.Insertable(completed).ExecuteCommandAsync();
|
|
var waiting = await host.Tasks.CreateTaskAsync(
|
|
VideoTaskType.Sync,
|
|
VideoTaskTrigger.Scheduled,
|
|
"等待存储",
|
|
VideoTypeEnum.dy_follows);
|
|
await host.Tasks.BlockTaskForStorageAsync(waiting.Id, "存储不可用");
|
|
|
|
var summary = await host.Tasks.GetSummaryAsync();
|
|
|
|
Assert.Equal(105, summary.Completed);
|
|
Assert.Equal(1, summary.WaitingForStorage);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task SuccessfulStorageProbe_ClosesEmptyWaitsAndRequeuesRecoverableItems()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
await host.Database.Insertable(new DouyinCookie
|
|
{
|
|
Id = "storage-cookie",
|
|
UserName = "账号",
|
|
SavePath = host.Temporary.Path
|
|
}).ExecuteCommandAsync();
|
|
var empty = await host.Tasks.CreateTaskAsync(
|
|
VideoTaskType.Sync,
|
|
VideoTaskTrigger.Scheduled,
|
|
"空等待",
|
|
VideoTypeEnum.dy_follows);
|
|
await host.Tasks.BlockTaskForStorageAsync(empty.Id, "存储不可用");
|
|
var recoverable = await host.Tasks.CreateTaskAsync(
|
|
VideoTaskType.Sync,
|
|
VideoTaskTrigger.Manual,
|
|
"有条目等待",
|
|
VideoTypeEnum.dy_favorite);
|
|
var item = await host.Tasks.AddItemAsync(
|
|
recoverable.Id,
|
|
"waiting-item",
|
|
"storage-cookie",
|
|
"账号",
|
|
VideoTypeEnum.dy_favorite,
|
|
"等待作品",
|
|
"作者",
|
|
workerManaged: true);
|
|
item.Stage = VideoTaskItemStage.WaitingForStorage;
|
|
item.ErrorType = VideoTaskErrorType.StorageUnavailable;
|
|
item.ErrorMessage = "存储不可用";
|
|
await host.Database.Updateable(item).ExecuteCommandAsync();
|
|
await host.Tasks.BlockTaskForStorageAsync(recoverable.Id, "存储不可用");
|
|
|
|
var probe = await host.Tasks.ProbeStorageAsync(CancellationToken.None);
|
|
|
|
Assert.True(probe.Success, probe.Message);
|
|
empty = await host.Database.Queryable<VideoDownloadTask>().InSingleAsync(empty.Id);
|
|
recoverable = await host.Database.Queryable<VideoDownloadTask>().InSingleAsync(recoverable.Id);
|
|
item = await host.Database.Queryable<VideoDownloadTaskItem>().InSingleAsync(item.Id);
|
|
Assert.Equal(VideoTaskStatus.Cancelled, empty.Status);
|
|
Assert.Contains("无条目的等待任务", empty.ErrorMessage);
|
|
Assert.Equal(VideoTaskStatus.Queued, recoverable.Status);
|
|
Assert.Equal(VideoTaskItemStage.Pending, item.Stage);
|
|
Assert.Equal(VideoTaskErrorType.None, item.ErrorType);
|
|
Assert.Null(item.ErrorMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ThreeConsecutiveStorageFailures_OpenCircuitForTasksOnSameTarget()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
var first = await host.Tasks.CreateTaskAsync(VideoTaskType.Sync, VideoTaskTrigger.Manual, "任务一");
|
|
var second = await host.Tasks.CreateTaskAsync(VideoTaskType.Sync, VideoTaskTrigger.Manual, "任务二");
|
|
await host.Tasks.StartTaskAsync(first.Id);
|
|
await host.Tasks.StartTaskAsync(second.Id);
|
|
var failures = new List<VideoDownloadTaskItem>();
|
|
for (var index = 0; index < 3; index++)
|
|
failures.Add(await host.Tasks.AddItemAsync(first.Id, $"failed-{index}", "cookie", "账号", null, "失败", "作者"));
|
|
var waiting = await host.Tasks.AddItemAsync(second.Id, "waiting", "cookie", "账号", null, "等待", "作者");
|
|
|
|
foreach (var item in failures)
|
|
await host.Tasks.MarkFailedAsync(item.Id, new IOException("存储写入失败"), VideoTaskErrorType.StorageUnavailable);
|
|
|
|
var health = await host.Tasks.GetStorageHealthAsync();
|
|
Assert.Equal(MediaStorageHealthStatus.Unavailable, health.Status);
|
|
Assert.Equal(3, health.ConsecutiveFailures);
|
|
Assert.False(await host.Tasks.IsStorageAvailableAsync());
|
|
Assert.Equal(VideoTaskStatus.WaitingForStorage,
|
|
(await host.Database.Queryable<VideoDownloadTask>().InSingleAsync(second.Id)).Status);
|
|
Assert.Equal(VideoTaskItemStage.WaitingForStorage,
|
|
(await host.Database.Queryable<VideoDownloadTaskItem>().InSingleAsync(waiting.Id)).Stage);
|
|
|
|
await host.Database.Insertable(new DouyinCookie { Id = "changed", UserName = "新目录", SavePath = host.Temporary.Path }).ExecuteCommandAsync();
|
|
Assert.True(await host.Tasks.IsStorageAvailableAsync());
|
|
health = await host.Tasks.GetStorageHealthAsync();
|
|
Assert.Equal(MediaStorageHealthStatus.Healthy, health.Status);
|
|
Assert.Equal(0, health.ConsecutiveFailures);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task StorageHealth_IsIsolatedByTaskTarget()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
var webDavTask = await host.Tasks.CreateTaskAsync(VideoTaskType.Redownload,
|
|
VideoTaskTrigger.UserAction, "WebDAV 历史任务", storageType: MediaStorageType.WebDav);
|
|
for (var index = 0; index < 3; index++)
|
|
{
|
|
var item = await host.Tasks.AddItemAsync(webDavTask.Id, $"webdav-{index}", null, null, null, "失败", null);
|
|
await host.Tasks.MarkFailedAsync(item.Id, new IOException("WebDAV 写入失败"), VideoTaskErrorType.StorageUnavailable);
|
|
}
|
|
|
|
var currentLocalHealth = await host.Tasks.GetStorageHealthAsync();
|
|
var webDavHealth = await host.Database.Queryable<MediaStorageHealth>()
|
|
.Where(x => x.StorageType == MediaStorageType.WebDav).FirstAsync();
|
|
Assert.Equal(MediaStorageHealthStatus.Healthy, currentLocalHealth.Status);
|
|
Assert.Equal(MediaStorageHealthStatus.Unavailable, webDavHealth.Status);
|
|
Assert.Equal(3, webDavHealth.ConsecutiveFailures);
|
|
|
|
var localTask = await host.Tasks.CreateTaskAsync(VideoTaskType.Sync, VideoTaskTrigger.Manual, "本地任务");
|
|
var localItem = await host.Tasks.AddItemAsync(localTask.Id, "local-success", null, null, null, "成功", null);
|
|
await host.Tasks.MarkSucceededAsync(localItem.Id, 10);
|
|
|
|
webDavHealth = await host.Database.Queryable<MediaStorageHealth>()
|
|
.Where(x => x.StorageType == MediaStorageType.WebDav).FirstAsync();
|
|
Assert.Equal(MediaStorageHealthStatus.Unavailable, webDavHealth.Status);
|
|
Assert.Equal(3, webDavHealth.ConsecutiveFailures);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task SourceForbidden_DoesNotOpenStorageCircuit_AndThirdWorkStartsCooldown()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
await host.Database.Insertable(new DouyinCookie { Id = "cookie", UserName = "账号", Cookies = "sessionid=test" })
|
|
.ExecuteCommandAsync();
|
|
var task = await host.Tasks.CreateTaskAsync(VideoTaskType.Sync, VideoTaskTrigger.Manual, "403 测试");
|
|
var items = new List<VideoDownloadTaskItem>();
|
|
for (var index = 0; index < 3; index++)
|
|
{
|
|
var item = await host.Tasks.AddItemAsync(task.Id, $"aweme-{index}", "cookie", "账号", null, $"作品{index}", "作者");
|
|
items.Add(item);
|
|
await host.Tasks.MarkDownloadFailedAsync(item.Id, "cookie", Forbidden());
|
|
}
|
|
|
|
var storage = await host.Tasks.GetStorageHealthAsync();
|
|
var cookie = await host.Database.Queryable<DouyinCookie>().InSingleAsync("cookie");
|
|
var third = await host.Database.Queryable<VideoDownloadTaskItem>().InSingleAsync(items[2].Id);
|
|
Assert.Equal(MediaStorageHealthStatus.Healthy, storage.Status);
|
|
Assert.Equal(0, storage.ConsecutiveFailures);
|
|
Assert.Equal(3, cookie.ConsecutiveSourceForbidden);
|
|
Assert.True(cookie.SourceCooldownUntil > DateTime.UtcNow);
|
|
Assert.True(cookie.SourceProbePending);
|
|
Assert.Equal(VideoTaskItemStage.WaitingForSource, third.Stage);
|
|
Assert.Equal(VideoTaskStatus.WaitingForSource,
|
|
(await host.Database.Queryable<VideoDownloadTask>().InSingleAsync(task.Id)).Status);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Cooldown_AllowsOnlyOldestProbe_AndSecondForbiddenRequiresAuthorization()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
await host.Database.Insertable(new DouyinCookie { Id = "cookie", UserName = "账号", Cookies = "sessionid=test" })
|
|
.ExecuteCommandAsync();
|
|
var task = await host.Tasks.CreateTaskAsync(VideoTaskType.ExclusionRestore, VideoTaskTrigger.UserAction, "来源恢复");
|
|
var items = new List<VideoDownloadTaskItem>();
|
|
for (var index = 0; index < 3; index++)
|
|
{
|
|
var item = await host.Tasks.AddItemAsync(task.Id, $"aweme-{index}", "cookie", "账号", null, $"作品{index}", "作者", workerManaged: true);
|
|
items.Add(item);
|
|
await host.Tasks.MarkDownloadFailedAsync(item.Id, "cookie", Forbidden());
|
|
}
|
|
var additional = await host.Tasks.AddItemAsync(task.Id, "waiting-later", "cookie", "账号", null, "稍后作品", "作者", workerManaged: true);
|
|
await host.Tasks.WaitItemForSourceAsync(additional.Id, "cookie", "等待探测");
|
|
|
|
var cookie = await host.Database.Queryable<DouyinCookie>().InSingleAsync("cookie");
|
|
cookie.SourceCooldownUntil = DateTime.UtcNow.AddSeconds(-1);
|
|
await host.Database.Updateable(cookie).ExecuteCommandAsync();
|
|
|
|
var laterDecision = await host.Tasks.TryAcquireSourceAsync("cookie", additional.Id);
|
|
var oldestDecision = await host.Tasks.TryAcquireSourceAsync("cookie", items[0].Id);
|
|
var concurrentDecision = await host.Tasks.TryAcquireSourceAsync("cookie", items[0].Id);
|
|
Assert.False(laterDecision.Allowed);
|
|
Assert.True(oldestDecision.Allowed);
|
|
Assert.True(oldestDecision.IsProbe);
|
|
Assert.False(concurrentDecision.Allowed);
|
|
|
|
var disposition = await host.Tasks.MarkDownloadFailedAsync(items[0].Id, "cookie", Forbidden());
|
|
cookie = await host.Database.Queryable<DouyinCookie>().InSingleAsync("cookie");
|
|
Assert.Equal(SourceFailureDisposition.RequiresAuthorization, disposition);
|
|
Assert.True(cookie.SourceRequiresAuthorization);
|
|
Assert.False(cookie.SourceProbePending);
|
|
Assert.Null(cookie.SourceCooldownUntil);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ProbeSuccess_ResetsPersistedHealthAndRequeuesWaitingItems()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
await host.Database.Insertable(new DouyinCookie
|
|
{
|
|
Id = "cookie",
|
|
UserName = "账号",
|
|
Cookies = "sessionid=test",
|
|
ConsecutiveSourceForbidden = 3,
|
|
SourceProbePending = true,
|
|
SourceProbeInProgress = true,
|
|
SourceCooldownUntil = DateTime.UtcNow.AddSeconds(-1),
|
|
LastSourceStatusCode = 403,
|
|
LastSourceError = "全部地址 403"
|
|
}).ExecuteCommandAsync();
|
|
var task = await host.Tasks.CreateTaskAsync(VideoTaskType.ExclusionRestore, VideoTaskTrigger.UserAction, "恢复");
|
|
var item = await host.Tasks.AddItemAsync(task.Id, "aweme", "cookie", "账号", null, "作品", "作者", workerManaged: true);
|
|
await host.Tasks.WaitItemForSourceAsync(item.Id, "cookie", "等待探测");
|
|
|
|
await host.Tasks.RecordSourceSuccessAsync("cookie");
|
|
|
|
var cookie = await host.Database.Queryable<DouyinCookie>().InSingleAsync("cookie");
|
|
item = await host.Database.Queryable<VideoDownloadTaskItem>().InSingleAsync(item.Id);
|
|
Assert.Equal(0, cookie.ConsecutiveSourceForbidden);
|
|
Assert.False(cookie.SourceProbePending);
|
|
Assert.False(cookie.SourceProbeInProgress);
|
|
Assert.False(cookie.SourceRequiresAuthorization);
|
|
Assert.Null(cookie.SourceCooldownUntil);
|
|
Assert.Equal(VideoTaskItemStage.Pending, item.Stage);
|
|
Assert.Equal(VideoTaskStatus.Queued,
|
|
(await host.Database.Queryable<VideoDownloadTask>().InSingleAsync(task.Id)).Status);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Unauthorized_ImmediatelyRequiresAuthorization()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
await host.Database.Insertable(new DouyinCookie { Id = "cookie", UserName = "账号", Cookies = "old" }).ExecuteCommandAsync();
|
|
var task = await host.Tasks.CreateTaskAsync(VideoTaskType.Sync, VideoTaskTrigger.Manual, "401");
|
|
var item = await host.Tasks.AddItemAsync(task.Id, "aweme", "cookie", "账号", null, "作品", "作者");
|
|
var result = Forbidden();
|
|
result = new MediaDownloadResult
|
|
{
|
|
Success = false,
|
|
FailureKind = MediaDownloadFailureKind.SourceUnauthorized,
|
|
HttpStatusCode = 401,
|
|
SourceHost = result.SourceHost,
|
|
AttemptedUrlCount = 1,
|
|
Message = "需要重新授权"
|
|
};
|
|
|
|
var disposition = await host.Tasks.MarkDownloadFailedAsync(item.Id, "cookie", result);
|
|
|
|
var cookie = await host.Database.Queryable<DouyinCookie>().InSingleAsync("cookie");
|
|
item = await host.Database.Queryable<VideoDownloadTaskItem>().InSingleAsync(item.Id);
|
|
Assert.Equal(SourceFailureDisposition.RequiresAuthorization, disposition);
|
|
Assert.True(cookie.SourceRequiresAuthorization);
|
|
Assert.Equal(VideoTaskItemStage.WaitingForSource, item.Stage);
|
|
Assert.Equal(VideoTaskErrorType.CookieInvalid, item.ErrorType);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task NotFound_FailsOnlyItemWithoutBlockingAuthorization()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
await host.Database.Insertable(new DouyinCookie { Id = "cookie", UserName = "账号", Cookies = "valid" }).ExecuteCommandAsync();
|
|
var task = await host.Tasks.CreateTaskAsync(VideoTaskType.Sync, VideoTaskTrigger.Manual, "404");
|
|
var item = await host.Tasks.AddItemAsync(task.Id, "aweme", "cookie", "账号", null, "作品", "作者");
|
|
var result = new MediaDownloadResult
|
|
{
|
|
Success = false,
|
|
FailureKind = MediaDownloadFailureKind.SourceNotFound,
|
|
HttpStatusCode = 404,
|
|
SourceHost = "v3.douyinvod.com",
|
|
AttemptedUrlCount = 2,
|
|
Message = "作品已下架"
|
|
};
|
|
|
|
var disposition = await host.Tasks.MarkDownloadFailedAsync(item.Id, "cookie", result);
|
|
|
|
var cookie = await host.Database.Queryable<DouyinCookie>().InSingleAsync("cookie");
|
|
item = await host.Database.Queryable<VideoDownloadTaskItem>().InSingleAsync(item.Id);
|
|
Assert.Equal(SourceFailureDisposition.Continue, disposition);
|
|
Assert.False(cookie.SourceRequiresAuthorization);
|
|
Assert.False(cookie.SourceProbePending);
|
|
Assert.Equal(VideoTaskItemStage.Failed, item.Stage);
|
|
Assert.Equal(VideoTaskErrorType.SourceUnavailable, item.ErrorType);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task RateLimited_UsesRetryAfterWithoutRequiringAuthorization()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
await host.Database.Insertable(new DouyinCookie { Id = "cookie", UserName = "账号", Cookies = "valid" }).ExecuteCommandAsync();
|
|
var task = await host.Tasks.CreateTaskAsync(VideoTaskType.Sync, VideoTaskTrigger.Manual, "429");
|
|
var item = await host.Tasks.AddItemAsync(task.Id, "aweme", "cookie", "账号", null, "作品", "作者");
|
|
var retryAt = DateTime.UtcNow.AddMinutes(3);
|
|
var result = new MediaDownloadResult
|
|
{
|
|
Success = false,
|
|
FailureKind = MediaDownloadFailureKind.SourceRateLimited,
|
|
HttpStatusCode = 429,
|
|
SourceHost = "v3.douyinvod.com",
|
|
AttemptedUrlCount = 1,
|
|
RetryAfter = retryAt,
|
|
Message = "请求过于频繁"
|
|
};
|
|
|
|
var disposition = await host.Tasks.MarkDownloadFailedAsync(item.Id, "cookie", result);
|
|
|
|
var cookie = await host.Database.Queryable<DouyinCookie>().InSingleAsync("cookie");
|
|
item = await host.Database.Queryable<VideoDownloadTaskItem>().InSingleAsync(item.Id);
|
|
Assert.Equal(SourceFailureDisposition.WaitForSource, disposition);
|
|
Assert.False(cookie.SourceRequiresAuthorization);
|
|
Assert.True(cookie.SourceProbePending);
|
|
Assert.True(cookie.SourceCooldownUntil >= retryAt.AddSeconds(-1));
|
|
Assert.Equal(VideoTaskErrorType.SourceRateLimited, item.ErrorType);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Redownload_MixedStorageSelectionCreatesOneParentPerStorageTarget()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
var local = Video("local-video", "local-aweme", MediaStorageType.Local,
|
|
Path.Combine(host.Temporary.Path, "local.mp4"));
|
|
var remote = Video("remote-video", "remote-aweme", MediaStorageType.WebDav, "关注/remote.mp4");
|
|
await host.Database.Insertable(new[] { local, remote }).ExecuteCommandAsync();
|
|
|
|
var result = await host.CreateVideoService().ReDownloadViedoAsync(new ReDownViedoDto
|
|
{
|
|
Ids = new() { local.Id, remote.Id }
|
|
});
|
|
|
|
Assert.True(result);
|
|
var tasks = await host.Database.Queryable<VideoDownloadTask>().OrderBy(x => x.StorageType).ToListAsync();
|
|
var jobs = await host.Database.Queryable<DouyinReDownload>().ToListAsync();
|
|
Assert.Equal(2, tasks.Count);
|
|
Assert.Equal(2, jobs.Count);
|
|
Assert.Contains(tasks, x => x.StorageType == MediaStorageType.Local && jobs.Any(j => j.TaskId == x.Id && j.VideoRecordId == local.Id));
|
|
Assert.Contains(tasks, x => x.StorageType == MediaStorageType.WebDav && jobs.Any(j => j.TaskId == x.Id && j.VideoRecordId == remote.Id));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task RestartRecovery_RequeuesSnapshotItemsAndLeavesUnsafeItemsFailed()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
var task = await host.Tasks.CreateTaskAsync(VideoTaskType.Sync, VideoTaskTrigger.Scheduled, "重启任务");
|
|
await host.Tasks.StartTaskAsync(task.Id);
|
|
var unsafePending = await host.Tasks.AddItemAsync(task.Id, "pending", "cookie", "账号", null, "等待", "作者");
|
|
var unsafeActive = await host.Tasks.AddItemAsync(task.Id, "unsafe-active", "cookie", "账号", null, "无快照", "作者");
|
|
await host.Tasks.SetItemStageAsync(unsafeActive.Id, VideoTaskItemStage.Downloading);
|
|
var snapshot = Video("recoverable-record", "recoverable", MediaStorageType.Local,
|
|
Path.Combine(host.Temporary.Path, "recoverable.mp4"));
|
|
var recoverable = await host.Tasks.AddItemAsync(task.Id, snapshot.AwemeId, "cookie", "账号", null,
|
|
"可恢复", "作者", snapshot.VideoSavePath, new[] { snapshot.VideoUrl }, snapshot);
|
|
await host.Tasks.SetItemStageAsync(recoverable.Id, VideoTaskItemStage.Verifying);
|
|
|
|
await host.Tasks.RecoverInterruptedAsync();
|
|
|
|
var recoveredTask = await host.Database.Queryable<VideoDownloadTask>().InSingleAsync(task.Id);
|
|
unsafePending = await host.Database.Queryable<VideoDownloadTaskItem>().InSingleAsync(unsafePending.Id);
|
|
unsafeActive = await host.Database.Queryable<VideoDownloadTaskItem>().InSingleAsync(unsafeActive.Id);
|
|
recoverable = await host.Database.Queryable<VideoDownloadTaskItem>().InSingleAsync(recoverable.Id);
|
|
Assert.Equal(VideoTaskStatus.Queued, recoveredTask.Status);
|
|
Assert.Contains("自动恢复 1 个条目", recoveredTask.ErrorMessage);
|
|
Assert.Equal(VideoTaskItemStage.Pending, recoverable.Stage);
|
|
Assert.True(recoverable.WorkerManaged);
|
|
Assert.Equal(VideoTaskErrorType.None, recoverable.ErrorType);
|
|
Assert.Null(recoverable.CompletedAt);
|
|
Assert.All(new[] { unsafePending, unsafeActive }, item =>
|
|
{
|
|
Assert.Equal(VideoTaskItemStage.Failed, item.Stage);
|
|
Assert.Equal(VideoTaskErrorType.Interrupted, item.ErrorType);
|
|
Assert.Contains("下一次同步", item.ErrorMessage);
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public async Task RestartRecovery_ClosesStaleEmptyQueueButKeepsQueuedTaskWithItems()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
var empty = await host.Tasks.CreateTaskAsync(VideoTaskType.Sync, VideoTaskTrigger.Manual, "空排队任务");
|
|
var withItem = await host.Tasks.CreateTaskAsync(VideoTaskType.Sync, VideoTaskTrigger.Manual, "有效排队任务");
|
|
await host.Tasks.AddItemAsync(withItem.Id, "pending", "cookie", "账号", null, "等待", "作者", workerManaged: true);
|
|
var staleTime = DateTime.Now.AddMinutes(-30);
|
|
empty.UpdatedAt = staleTime;
|
|
withItem.UpdatedAt = staleTime;
|
|
await host.Database.Updateable(new[] { empty, withItem }).ExecuteCommandAsync();
|
|
|
|
await host.Tasks.RecoverInterruptedAsync();
|
|
|
|
empty = await host.Database.Queryable<VideoDownloadTask>().InSingleAsync(empty.Id);
|
|
withItem = await host.Database.Queryable<VideoDownloadTask>().InSingleAsync(withItem.Id);
|
|
Assert.Equal(VideoTaskStatus.Cancelled, empty.Status);
|
|
Assert.Contains("空任务", empty.ErrorMessage);
|
|
Assert.NotNull(empty.CompletedAt);
|
|
Assert.Equal(VideoTaskStatus.Queued, withItem.Status);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task RestartRecovery_EndsStaleWebDavSyncTasksAfterOpenListSwitch()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
var config = await host.Database.Queryable<AppConfig>().FirstAsync();
|
|
config.StorageType = MediaStorageType.OpenList;
|
|
await host.Database.Updateable(config).ExecuteCommandAsync();
|
|
var queued = await host.Tasks.CreateTaskAsync(VideoTaskType.Sync, VideoTaskTrigger.Scheduled,
|
|
"旧 WebDAV 收藏同步", VideoTypeEnum.dy_favorite, MediaStorageType.WebDav);
|
|
var item = await host.Tasks.AddItemAsync(queued.Id, "aweme", "cookie", "账号",
|
|
VideoTypeEnum.dy_favorite, "作品", "作者", "/collect/old.mp4");
|
|
var waiting = await host.Tasks.CreateTaskAsync(VideoTaskType.Sync, VideoTaskTrigger.Scheduled,
|
|
"旧 WebDAV 关注同步", VideoTypeEnum.dy_follows, MediaStorageType.WebDav);
|
|
await host.Tasks.BlockTaskForStorageAsync(waiting.Id, "WebDAV 不可用");
|
|
var staleTime = DateTime.Now.AddMinutes(-30);
|
|
queued.UpdatedAt = staleTime;
|
|
waiting.UpdatedAt = staleTime;
|
|
await host.Database.Updateable(new[] { queued, waiting }).ExecuteCommandAsync();
|
|
|
|
await host.Tasks.RecoverInterruptedAsync();
|
|
|
|
queued = await host.Database.Queryable<VideoDownloadTask>().InSingleAsync(queued.Id);
|
|
waiting = await host.Database.Queryable<VideoDownloadTask>().InSingleAsync(waiting.Id);
|
|
item = await host.Database.Queryable<VideoDownloadTaskItem>().InSingleAsync(item.Id);
|
|
Assert.Equal(VideoTaskStatus.Cancelled, queued.Status);
|
|
Assert.Equal(VideoTaskStatus.Cancelled, waiting.Status);
|
|
Assert.True(queued.IsArchived);
|
|
Assert.True(waiting.IsArchived);
|
|
Assert.Equal(VideoTaskItemStage.Cancelled, item.Stage);
|
|
Assert.Equal(VideoTaskErrorType.ConfigurationChanged, item.ErrorType);
|
|
Assert.Contains("OpenList", queued.ErrorMessage);
|
|
Assert.Equal(0, await host.Database.Queryable<VideoDownloadTask>()
|
|
.Where(x => x.Status == VideoTaskStatus.Queued).CountAsync());
|
|
Assert.Equal(0, await host.Database.Queryable<VideoDownloadTask>()
|
|
.Where(x => x.Status == VideoTaskStatus.WaitingForStorage).CountAsync());
|
|
var visible = await host.Tasks.GetTasksAsync(new VideoTaskPageRequest { PageIndex = 1, PageSize = 20 });
|
|
Assert.DoesNotContain(visible.Items, x => x.Id == queued.Id || x.Id == waiting.Id);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task RestartRecovery_ReleasesInterruptedSourceProbeForOldestWaitingItem()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
await host.Database.Insertable(new DouyinCookie
|
|
{
|
|
Id = "cookie",
|
|
UserName = "账号",
|
|
Cookies = "valid",
|
|
SourceProbePending = true,
|
|
SourceProbeInProgress = true,
|
|
SourceCooldownUntil = DateTime.UtcNow.AddMinutes(-1)
|
|
}).ExecuteCommandAsync();
|
|
var task = await host.Tasks.CreateTaskAsync(VideoTaskType.ExclusionRestore, VideoTaskTrigger.UserAction, "等待来源");
|
|
var item = await host.Tasks.AddItemAsync(task.Id, "aweme", "cookie", "账号", null, "作品", "作者", workerManaged: true);
|
|
await host.Tasks.WaitItemForSourceAsync(item.Id, "cookie", "等待来源");
|
|
|
|
await host.Tasks.RecoverInterruptedAsync();
|
|
var decision = await host.Tasks.TryAcquireSourceAsync("cookie", item.Id);
|
|
|
|
Assert.True(decision.Allowed);
|
|
Assert.True(decision.IsProbe);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task UnexcludeOnly_RemovesAllDuplicateRowsWithoutCreatingTask()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
await host.Database.Insertable(new[]
|
|
{
|
|
Exclusion("one", "same-aweme"),
|
|
Exclusion("duplicate", "same-aweme")
|
|
}).ExecuteCommandAsync();
|
|
var service = new VideoExclusionService(host.Database, host.Tasks);
|
|
|
|
var result = await service.UnexcludeAsync(new UnexcludeVideosRequest { Ids = new() { "one" } });
|
|
|
|
Assert.Equal(1, result.ReleasedCount);
|
|
Assert.Null(result.TaskId);
|
|
Assert.Equal(0, await host.Database.Queryable<DouyinVideoDelete>().CountAsync());
|
|
Assert.Equal(0, await host.Database.Queryable<VideoDownloadTask>().CountAsync());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task UnexcludeAndRestore_QueuesValidSnapshotsAndReportsOldIncompleteRows()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
var snapshot = new DouyinVideo
|
|
{
|
|
Id = "video-one",
|
|
AwemeId = "good-aweme",
|
|
CookieId = "cookie",
|
|
ViedoType = VideoTypeEnum.dy_favorite,
|
|
VideoTitle = "可恢复",
|
|
VideoUrl = "https://example.test/video.mp4",
|
|
VideoSavePath = Path.Combine(host.Temporary.Path, "good.mp4"),
|
|
StorageType = MediaStorageType.Local
|
|
};
|
|
var good = Exclusion("good", snapshot.AwemeId);
|
|
good.RestoreSnapshotJson = JsonConvert.SerializeObject(snapshot);
|
|
var old = Exclusion("old", "old-aweme");
|
|
await host.Database.Insertable(new[] { good, old }).ExecuteCommandAsync();
|
|
var service = new VideoExclusionService(host.Database, host.Tasks);
|
|
|
|
var result = await service.UnexcludeAsync(new UnexcludeVideosRequest
|
|
{
|
|
Ids = new() { good.Id, old.Id },
|
|
CreateDownloadTask = true
|
|
});
|
|
|
|
Assert.Equal(2, result.ReleasedCount);
|
|
Assert.Equal(1, result.QueuedCount);
|
|
Assert.Contains("old-aweme", result.CannotQueueIds);
|
|
Assert.NotNull(result.TaskId);
|
|
Assert.Equal(VideoTaskType.ExclusionRestore,
|
|
(await host.Database.Queryable<VideoDownloadTask>().InSingleAsync(result.TaskId)).Type);
|
|
Assert.Single(await host.Database.Queryable<VideoDownloadTaskItem>().Where(x => x.TaskId == result.TaskId).ToListAsync());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task CleanupHistory_UsesThirtyAndNinetyDayWindowsAndKeepsRollbackData()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
var task = await host.Tasks.CreateTaskAsync(VideoTaskType.Sync, VideoTaskTrigger.Manual, "历史");
|
|
var success = await host.Tasks.AddItemAsync(task.Id, "success", null, null, null, "成功", null);
|
|
var failed = await host.Tasks.AddItemAsync(task.Id, "failed", null, null, null, "失败", null);
|
|
success.Stage = VideoTaskItemStage.Succeeded;
|
|
success.CompletedAt = DateTime.Now.AddDays(-31);
|
|
failed.Stage = VideoTaskItemStage.Failed;
|
|
failed.CompletedAt = DateTime.Now.AddDays(-89);
|
|
await host.Database.Updateable(new[] { success, failed }).ExecuteCommandAsync();
|
|
var migration = new StorageMigrationTask
|
|
{
|
|
Id = "rollback-still-available",
|
|
Status = StorageMigrationTaskStatus.Completed,
|
|
Concurrency = 1,
|
|
ConfigurationFingerprint = "fingerprint",
|
|
CreatedAt = DateTime.Now.AddDays(-100),
|
|
UpdatedAt = DateTime.Now.AddDays(-100)
|
|
};
|
|
var migrationItem = new StorageMigrationItem
|
|
{
|
|
Id = "migration-item",
|
|
TaskId = migration.Id,
|
|
VideoId = "video",
|
|
Stage = StorageMigrationItemStage.Succeeded,
|
|
OldSnapshotJson = "{}",
|
|
CreatedAt = migration.CreatedAt,
|
|
UpdatedAt = migration.UpdatedAt
|
|
};
|
|
await host.Database.Insertable(migration).ExecuteCommandAsync();
|
|
await host.Database.Insertable(migrationItem).ExecuteCommandAsync();
|
|
|
|
await host.Tasks.CleanupHistoryAsync();
|
|
|
|
Assert.Null(await host.Database.Queryable<VideoDownloadTaskItem>().InSingleAsync(success.Id));
|
|
Assert.NotNull(await host.Database.Queryable<VideoDownloadTaskItem>().InSingleAsync(failed.Id));
|
|
Assert.NotNull(await host.Database.Queryable<StorageMigrationItem>().InSingleAsync(migrationItem.Id));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task RemoveFailedMigrationRecords_DeduplicatesHistoryAndKeepsFilesAndExclusions()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
var oldFile = Path.Combine(host.Temporary.Path, "eligible-old.mp4");
|
|
await File.WriteAllTextAsync(oldFile, "keep this local fallback");
|
|
var eligible = Video("eligible", "aweme-eligible", MediaStorageType.Local, oldFile);
|
|
var changed = Video("changed", "aweme-changed", MediaStorageType.Local, "/current/changed.mp4");
|
|
var excluded = Video("excluded", "aweme-excluded", MediaStorageType.Local, "/old/excluded.mp4");
|
|
var replacement = Video("replacement-new", "aweme-replacement", MediaStorageType.Local, "/current/replacement.mp4");
|
|
await host.Database.Insertable(new[] { eligible, changed, excluded, replacement }).ExecuteCommandAsync();
|
|
await host.Database.Insertable(Exclusion("excluded-entry", excluded.AwemeId)).ExecuteCommandAsync();
|
|
|
|
var first = MigrationTask("migration-first");
|
|
var second = MigrationTask("migration-second");
|
|
await host.Database.Insertable(new[] { first, second }).ExecuteCommandAsync();
|
|
var items = new[]
|
|
{
|
|
MigrationItem("eligible-first", first.Id, eligible, eligible.VideoSavePath),
|
|
MigrationItem("eligible-second", second.Id, eligible, eligible.VideoSavePath),
|
|
MigrationItem("already-missing", first.Id,
|
|
Video("missing", "aweme-missing", MediaStorageType.Local, "/old/missing.mp4"), "/old/missing.mp4"),
|
|
MigrationItem("changed", first.Id, changed, "/old/changed.mp4"),
|
|
MigrationItem("replacement", first.Id,
|
|
Video("replacement-old", replacement.AwemeId, MediaStorageType.Local, "/old/replacement.mp4"), "/old/replacement.mp4"),
|
|
MigrationItem("excluded", first.Id, excluded, excluded.VideoSavePath),
|
|
new StorageMigrationItem
|
|
{
|
|
Id = "invalid", TaskId = first.Id, VideoId = "invalid-video", Stage = StorageMigrationItemStage.Failed,
|
|
OldSnapshotJson = "{}", CreatedAt = DateTime.Now, UpdatedAt = DateTime.Now
|
|
}
|
|
};
|
|
await host.Database.Insertable(items).ExecuteCommandAsync();
|
|
|
|
var preview = await host.Migration.PreviewFailedRecordRemovalAsync();
|
|
|
|
Assert.True(preview.CanExecute);
|
|
Assert.Equal(7, preview.FailedItemCount);
|
|
Assert.Equal(6, preview.DistinctVideoCount);
|
|
Assert.Equal(1, preview.EligibleRecordCount);
|
|
Assert.Equal(1, preview.AlreadyMissingCount);
|
|
Assert.Equal(2, preview.ChangedRecordCount);
|
|
Assert.Equal(1, preview.PermanentlyExcludedCount);
|
|
Assert.Equal(1, preview.InvalidSnapshotCount);
|
|
|
|
var result = await host.Migration.RemoveFailedRecordsAsync(new RemoveFailedMigrationRecordsRequest
|
|
{
|
|
ConfirmationToken = preview.ConfirmationToken
|
|
});
|
|
|
|
Assert.Equal(1, result.DeletedRecordCount);
|
|
Assert.Equal(1, result.AlreadyMissingCount);
|
|
Assert.Equal(3, result.RemovedItemCount);
|
|
Assert.Equal(2, result.AffectedTaskCount);
|
|
Assert.Null(await host.Database.Queryable<DouyinVideo>().InSingleAsync(eligible.Id));
|
|
Assert.NotNull(await host.Database.Queryable<DouyinVideo>().InSingleAsync(changed.Id));
|
|
Assert.NotNull(await host.Database.Queryable<DouyinVideo>().InSingleAsync(replacement.Id));
|
|
Assert.NotNull(await host.Database.Queryable<DouyinVideo>().InSingleAsync(excluded.Id));
|
|
Assert.True(File.Exists(oldFile));
|
|
Assert.Single(await host.Database.Queryable<DouyinVideoDelete>().ToListAsync());
|
|
Assert.All(await host.Database.Queryable<StorageMigrationItem>()
|
|
.Where(x => x.Id == "eligible-first" || x.Id == "eligible-second" || x.Id == "already-missing").ToListAsync(),
|
|
item => Assert.Equal(StorageMigrationItemStage.RecordRemoved, item.Stage));
|
|
Assert.Equal(StorageMigrationItemStage.Failed,
|
|
(await host.Database.Queryable<StorageMigrationItem>().InSingleAsync("changed")).Stage);
|
|
Assert.Equal(2, (await host.Database.Queryable<StorageMigrationTask>().InSingleAsync(first.Id)).RemovedCount);
|
|
Assert.Equal(1, (await host.Database.Queryable<StorageMigrationTask>().InSingleAsync(second.Id)).RemovedCount);
|
|
Assert.Null(await host.CreateVideoService().GetByAwemeId(eligible.AwemeId));
|
|
await Assert.ThrowsAsync<InvalidOperationException>(() => host.Migration.RemoveFailedRecordsAsync(
|
|
new RemoveFailedMigrationRecordsRequest { ConfirmationToken = preview.ConfirmationToken }));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task RemoveFailedMigrationRecords_IsBlockedWhileMigrationIsActive()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
var video = Video("eligible", "aweme", MediaStorageType.Local, "/old/video.mp4");
|
|
await host.Database.Insertable(video).ExecuteCommandAsync();
|
|
var historical = MigrationTask("historical");
|
|
var active = MigrationTask("active");
|
|
active.Status = StorageMigrationTaskStatus.Running;
|
|
await host.Database.Insertable(new[] { historical, active }).ExecuteCommandAsync();
|
|
await host.Database.Insertable(MigrationItem("failed", historical.Id, video, video.VideoSavePath)).ExecuteCommandAsync();
|
|
|
|
var preview = await host.Migration.PreviewFailedRecordRemovalAsync();
|
|
|
|
Assert.False(preview.CanExecute);
|
|
Assert.Equal(1, preview.ActiveMigrationCount);
|
|
Assert.Null(preview.ConfirmationToken);
|
|
Assert.Contains(preview.Errors, x => x.Contains("未结束") || x.Contains("运行"));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task FailedMigrationRecordPreview_BatchesLargeVideoSets()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
var task = MigrationTask("large-history");
|
|
await host.Database.Insertable(task).ExecuteCommandAsync();
|
|
var videos = Enumerable.Range(0, 1005).Select(index =>
|
|
Video($"video-{index}", $"aweme-{index}", MediaStorageType.Local, $"/old/{index}.mp4")).ToList();
|
|
var items = videos.Select((video, index) => MigrationItem($"item-{index}", task.Id, video, video.VideoSavePath)).ToList();
|
|
await host.Database.Insertable(videos).ExecuteCommandAsync();
|
|
await host.Database.Insertable(items).ExecuteCommandAsync();
|
|
|
|
var preview = await host.Migration.PreviewFailedRecordRemovalAsync();
|
|
|
|
Assert.True(preview.CanExecute);
|
|
Assert.Equal(1005, preview.FailedItemCount);
|
|
Assert.Equal(1005, preview.EligibleRecordCount);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task StorageInventory_ReportsRecordsOutsideCurrentStorage()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
var local = Video("local", "aweme-local", MediaStorageType.Local, "/old/local.mp4");
|
|
local.FileSize = 10;
|
|
var remote = Video("remote", "aweme-remote", MediaStorageType.WebDav, "/new/remote.mp4");
|
|
remote.FileSize = 20;
|
|
await host.Database.Insertable(new[] { local, remote }).ExecuteCommandAsync();
|
|
|
|
var inventory = await host.CreateVideoService().GetStorageInventoryAsync(MediaStorageType.WebDav);
|
|
|
|
Assert.Equal(2, inventory.TotalRecordCount);
|
|
Assert.Equal(1, inventory.LocalRecordCount);
|
|
Assert.Equal(1, inventory.WebDavRecordCount);
|
|
Assert.Equal(1, inventory.OtherStorageRecordCount);
|
|
Assert.Equal(10, inventory.LocalDeclaredBytes);
|
|
Assert.Equal(20, inventory.WebDavDeclaredBytes);
|
|
Assert.False(inventory.AllRecordsOnCurrentStorage);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ArchiveHistory_HidesSettledFailuresButKeepsTasksNeedingCleanup()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
var failedOnly = MigrationTask("failed-only");
|
|
var needsCleanup = MigrationTask("needs-cleanup");
|
|
failedOnly.TargetStorageType = MediaStorageType.OpenList;
|
|
needsCleanup.TargetStorageType = MediaStorageType.OpenList;
|
|
needsCleanup.Status = StorageMigrationTaskStatus.Completed;
|
|
needsCleanup.SuccessCount = 2;
|
|
await host.Database.Insertable(new[] { failedOnly, needsCleanup }).ExecuteCommandAsync();
|
|
|
|
var result = await host.Migration.ArchiveAllSettledAsync();
|
|
|
|
Assert.Equal(1, result.ArchivedCount);
|
|
Assert.Equal(1, result.RequiresCleanupCount);
|
|
Assert.True((await host.Database.Queryable<StorageMigrationTask>().InSingleAsync(failedOnly.Id)).IsArchived);
|
|
Assert.False((await host.Database.Queryable<StorageMigrationTask>().InSingleAsync(needsCleanup.Id)).IsArchived);
|
|
var visible = await host.Tasks.GetTasksAsync(new VideoTaskPageRequest { PageIndex = 1, PageSize = 20 });
|
|
Assert.DoesNotContain(visible.Items, x => x.Id == failedOnly.Id);
|
|
Assert.Contains(visible.Items, x => x.Id == needsCleanup.Id);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task StorageReplacementCleanup_DeletesOnlyUnreferencedLocalArtifactsAfterRemoteCommit()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
var root = Path.Combine(host.Temporary.Path, "media");
|
|
var folder = Path.Combine(root, "author", "video");
|
|
Directory.CreateDirectory(folder);
|
|
var main = Path.Combine(folder, "video.mp4");
|
|
var cover = Path.Combine(folder, "video-poster.jpg");
|
|
var nfo = Path.Combine(folder, "video.nfo");
|
|
var avatar = Path.Combine(root, "author", "avatar.jpg");
|
|
await File.WriteAllTextAsync(main, "old-main");
|
|
await File.WriteAllTextAsync(cover, "old-cover");
|
|
await File.WriteAllTextAsync(nfo, "old-nfo");
|
|
await File.WriteAllTextAsync(avatar, "shared-avatar");
|
|
var cookie = new DouyinCookie { Id = "cookie", UserName = "账号", SavePath = root };
|
|
await host.Database.Insertable(cookie).ExecuteCommandAsync();
|
|
var snapshot = Video("replaced", "aweme-replaced", MediaStorageType.Local, main);
|
|
snapshot.VideoCoverSavePath = cover;
|
|
snapshot.AuthorAvatar = avatar;
|
|
var current = Video(snapshot.Id, snapshot.AwemeId, MediaStorageType.WebDav, "/target/video.mp4");
|
|
current.VideoCoverSavePath = "/target/video-poster.jpg";
|
|
var shared = Video("shared", "aweme-shared", MediaStorageType.Local, Path.Combine(root, "other.mp4"));
|
|
shared.AuthorAvatar = avatar;
|
|
await host.Database.Insertable(new[] { current, shared }).ExecuteCommandAsync();
|
|
var settings = await host.Settings.BuildCandidateAsync(new WebDavTestRequest
|
|
{
|
|
Endpoint = "http://storage.test/dav",
|
|
BasePath = "/dysync",
|
|
UserName = "user",
|
|
Password = "password"
|
|
});
|
|
await host.Settings.SaveAsync(settings, true, "ok");
|
|
var remote = System.Text.Encoding.UTF8.GetBytes("new-main");
|
|
await using (var content = new MemoryStream(remote, false))
|
|
await host.WebDav.WriteAsync(current.VideoSavePath, content, remote.Length, "video/mp4");
|
|
|
|
var deleted = await host.CreateVideoService().CleanupSupersededLocalArtifactsAsync(snapshot, current.VideoSavePath);
|
|
|
|
Assert.Equal(3, deleted);
|
|
Assert.False(File.Exists(main));
|
|
Assert.False(File.Exists(cover));
|
|
Assert.False(File.Exists(nfo));
|
|
Assert.True(File.Exists(avatar));
|
|
Assert.True(await host.WebDav.ExistsAsync(current.VideoSavePath));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task CleanupRetry_RevalidatesRemoteAndClearsCleanupWarning()
|
|
{
|
|
using var host = VideoTaskTestHost.Create();
|
|
var root = Path.Combine(host.Temporary.Path, "cleanup-retry");
|
|
Directory.CreateDirectory(root);
|
|
var main = Path.Combine(root, "video.mp4");
|
|
await File.WriteAllTextAsync(main, "old-main");
|
|
await host.Database.Insertable(new DouyinCookie { Id = "cookie", UserName = "账号", SavePath = root })
|
|
.ExecuteCommandAsync();
|
|
var snapshot = Video("replaced", "aweme-cleanup-retry", MediaStorageType.Local, main);
|
|
var current = Video(snapshot.Id, snapshot.AwemeId, MediaStorageType.WebDav, "/target/retry.mp4");
|
|
await host.Database.Insertable(current).ExecuteCommandAsync();
|
|
var settings = await host.Settings.BuildCandidateAsync(new WebDavTestRequest
|
|
{
|
|
Endpoint = "http://storage.test/dav",
|
|
BasePath = "/dysync",
|
|
UserName = "user",
|
|
Password = "password"
|
|
});
|
|
await host.Settings.SaveAsync(settings, true, "ok");
|
|
var remote = System.Text.Encoding.UTF8.GetBytes("new-main");
|
|
await using (var content = new MemoryStream(remote, false))
|
|
await host.WebDav.WriteAsync(current.VideoSavePath, content, remote.Length, "video/mp4");
|
|
current.SupersededStorageSnapshot = snapshot;
|
|
var task = await host.Tasks.CreateTaskAsync(VideoTaskType.Sync, VideoTaskTrigger.Manual,
|
|
"清理重试", storageType: MediaStorageType.WebDav);
|
|
var item = await host.Tasks.AddItemAsync(task.Id, current.AwemeId, "cookie", "账号",
|
|
current.ViedoType, current.VideoTitle, current.Author, current.VideoSavePath,
|
|
new[] { current.VideoUrl }, current, remote.Length);
|
|
await host.Tasks.MarkSucceededAsync(item.Id, remote.Length, current.Id,
|
|
"已切换到 WebDav，但旧本地文件清理失败，可稍后重试清理。");
|
|
var legacyDto = Assert.Single((await host.Tasks.GetItemsAsync(VideoTaskType.Sync, task.Id,
|
|
new VideoTaskItemPageRequest { PageIndex = 1, PageSize = 20 })).Items);
|
|
Assert.True(legacyDto.CanRetryCleanup);
|
|
|
|
var deleted = await host.CreateVideoService().RetryTaskItemLocalCleanupAsync(task.Id, item.Id);
|
|
|
|
item = await host.Database.Queryable<VideoDownloadTaskItem>().InSingleAsync(item.Id);
|
|
Assert.Equal(1, deleted);
|
|
Assert.False(File.Exists(main));
|
|
Assert.Equal(VideoTaskItemStage.Succeeded, item.Stage);
|
|
Assert.False(item.CleanupPending);
|
|
Assert.Null(item.CleanupError);
|
|
Assert.Null(item.WarningMessage);
|
|
var dto = Assert.Single((await host.Tasks.GetItemsAsync(VideoTaskType.Sync, task.Id,
|
|
new VideoTaskItemPageRequest { PageIndex = 1, PageSize = 20 })).Items);
|
|
Assert.False(dto.CanRetryCleanup);
|
|
}
|
|
|
|
private static DouyinVideoDelete Exclusion(string id, string awemeId) => new()
|
|
{
|
|
Id = id,
|
|
ViedoId = awemeId,
|
|
VideoTitle = awemeId,
|
|
DeleteTime = DateTime.Now
|
|
};
|
|
|
|
private static StorageMigrationTask MigrationTask(string id) => new()
|
|
{
|
|
Id = id,
|
|
Status = StorageMigrationTaskStatus.PartiallyFailed,
|
|
Concurrency = 1,
|
|
ConfigurationFingerprint = "fingerprint",
|
|
CreatedAt = DateTime.Now,
|
|
UpdatedAt = DateTime.Now
|
|
};
|
|
|
|
private static StorageMigrationItem MigrationItem(string id, string taskId, DouyinVideo snapshot, string oldPath) => new()
|
|
{
|
|
Id = id,
|
|
TaskId = taskId,
|
|
VideoId = snapshot.Id,
|
|
Stage = StorageMigrationItemStage.Failed,
|
|
OldVideoPath = oldPath,
|
|
OldSnapshotJson = JsonConvert.SerializeObject(new DouyinVideo
|
|
{
|
|
Id = snapshot.Id,
|
|
AwemeId = snapshot.AwemeId,
|
|
StorageType = MediaStorageType.Local,
|
|
VideoSavePath = oldPath,
|
|
CookieId = snapshot.CookieId,
|
|
ViedoType = snapshot.ViedoType
|
|
}),
|
|
CreatedAt = DateTime.Now,
|
|
UpdatedAt = DateTime.Now
|
|
};
|
|
|
|
private static DouyinVideo Video(string id, string awemeId, MediaStorageType storageType, string path) => new()
|
|
{
|
|
Id = id,
|
|
AwemeId = awemeId,
|
|
CookieId = "cookie",
|
|
ViedoType = VideoTypeEnum.dy_follows,
|
|
VideoTitle = id,
|
|
Author = "同名博主",
|
|
AuthorId = id + "-author",
|
|
VideoUrl = "https://example.test/video.mp4",
|
|
VideoSavePath = path,
|
|
StorageType = storageType,
|
|
CreateTime = DateTime.Now,
|
|
SyncTime = DateTime.Now
|
|
};
|
|
|
|
private static MediaDownloadResult Forbidden() => new()
|
|
{
|
|
Success = false,
|
|
ActualSavePath = "/target/video.mp4",
|
|
FailureKind = MediaDownloadFailureKind.SourceForbidden,
|
|
HttpStatusCode = 403,
|
|
SourceHost = "v3.douyinvod.com",
|
|
AttemptedUrlCount = 4,
|
|
Message = "全部 4 个候选媒体地址均返回 403。"
|
|
};
|
|
|
|
private sealed class VideoTaskTestHost : IDisposable
|
|
{
|
|
private readonly StubHttpClientFactory _clientFactory;
|
|
|
|
private VideoTaskTestHost(TemporaryDirectory temporary, SqlSugarClient database,
|
|
StubHttpClientFactory clientFactory, WebDavSettingsService settings, WebDavMediaStorage webDav, StorageMigrationService migration,
|
|
VideoTaskService tasks)
|
|
{
|
|
Temporary = temporary;
|
|
Database = database;
|
|
_clientFactory = clientFactory;
|
|
Settings = settings;
|
|
WebDav = webDav;
|
|
Migration = migration;
|
|
Tasks = tasks;
|
|
}
|
|
|
|
public TemporaryDirectory Temporary { get; }
|
|
public SqlSugarClient Database { get; }
|
|
public WebDavMediaStorage WebDav { get; }
|
|
public OpenListMediaStorage OpenList { get; private set; }
|
|
public WebDavSettingsService Settings { get; }
|
|
public StorageMigrationService Migration { get; }
|
|
public VideoTaskService Tasks { get; }
|
|
|
|
public DouyinVideoService CreateVideoService() => new(
|
|
new DouyinVideoRepository(Database),
|
|
new DouyinCookieRepository(Database),
|
|
Database,
|
|
new MediaStorageRouter(new LocalMediaStorage(), WebDav, OpenList),
|
|
Tasks);
|
|
|
|
public static VideoTaskTestHost Create()
|
|
{
|
|
var temporary = new TemporaryDirectory();
|
|
try
|
|
{
|
|
var database = new SqlSugarClient(new ConnectionConfig
|
|
{
|
|
ConnectionString = $"DataSource={Path.Combine(temporary.Path, "tasks.sqlite")}",
|
|
DbType = DbType.Sqlite,
|
|
InitKeyType = InitKeyType.Attribute,
|
|
IsAutoCloseConnection = true
|
|
});
|
|
database.CodeFirst.InitTables(new[]
|
|
{
|
|
typeof(AppConfig), typeof(DouyinCookie), typeof(WebDavSettings), typeof(VideoDownloadTask),
|
|
typeof(VideoDownloadTaskItem), typeof(MediaStorageHealth), typeof(StorageMigrationTask),
|
|
typeof(StorageMigrationItem), typeof(DouyinVideoDelete), typeof(DouyinVideo), typeof(DouyinReDownload),
|
|
typeof(OpenListSettings), typeof(OpenListTransferJob), typeof(OpenListDirectoryRepairTask),
|
|
typeof(OpenListDirectoryRepairItem)
|
|
});
|
|
database.Insertable(new AppConfig
|
|
{
|
|
Id = "config",
|
|
StorageType = MediaStorageType.Local,
|
|
FollowedTitleTemplate = string.Empty,
|
|
FollowedTitleSeparator = string.Empty,
|
|
FullFollowedTitleTemplate = string.Empty
|
|
}).ExecuteCommand();
|
|
var provider = DataProtectionProvider.Create(new DirectoryInfo(Path.Combine(temporary.Path, "keys")),
|
|
builder => builder.SetApplicationName("dysync.net"));
|
|
var clientFactory = new StubHttpClientFactory(new InMemoryWebDavHandler("user", "password"));
|
|
var settings = new WebDavSettingsService(database, provider);
|
|
var webDav = new WebDavMediaStorage(clientFactory, settings);
|
|
var common = new DouyinCommonService(database);
|
|
var openListSettings = new OpenListSettingsService(database, provider, settings);
|
|
var openListClient = new OpenListClient(clientFactory);
|
|
var transfers = new OpenListTransferService(database, openListSettings, openListClient);
|
|
var openList = new OpenListMediaStorage(openListSettings, openListClient, transfers);
|
|
var migration = new StorageMigrationService(database, common, openListSettings, openList);
|
|
var repair = new OpenListDirectoryRepairService(database, common, openListSettings, openListClient);
|
|
var tasks = new VideoTaskService(database, common, settings, webDav, openListSettings, openList, migration, repair);
|
|
var host = new VideoTaskTestHost(temporary, database, clientFactory, settings, webDav, migration, tasks)
|
|
{
|
|
OpenList = openList
|
|
};
|
|
return host;
|
|
}
|
|
catch
|
|
{
|
|
temporary.Dispose();
|
|
throw;
|
|
}
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
Database.Dispose();
|
|
_clientFactory.Dispose();
|
|
Temporary.Dispose();
|
|
}
|
|
}
|
|
}
|