feat: add fnOS packaging, storage workflows and release pipeline
This commit is contained in:
@@ -0,0 +1,77 @@
|
||||
using System.Reflection;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
public sealed class DatabaseUpgradeBackup
|
||||
{
|
||||
private readonly string _databaseRoot;
|
||||
private readonly string _version;
|
||||
private readonly string _markerPath;
|
||||
|
||||
private DatabaseUpgradeBackup(string databaseRoot, string version)
|
||||
{
|
||||
_databaseRoot = databaseRoot;
|
||||
_version = version;
|
||||
_markerPath = Path.Combine(databaseRoot, ".schema-version");
|
||||
}
|
||||
|
||||
public static DatabaseUpgradeBackup Prepare(string dbPath)
|
||||
{
|
||||
var root = string.IsNullOrWhiteSpace(dbPath)
|
||||
? Path.Combine(Environment.CurrentDirectory, "db")
|
||||
: Path.Combine(dbPath, "db");
|
||||
Directory.CreateDirectory(root);
|
||||
var version = Assembly.GetExecutingAssembly().GetName().Version?.ToString(3) ?? "0.2.24";
|
||||
var backup = new DatabaseUpgradeBackup(root, version);
|
||||
backup.CreateIfNeeded();
|
||||
return backup;
|
||||
}
|
||||
|
||||
public void MarkSchemaReady() => File.WriteAllText(_markerPath, _version);
|
||||
|
||||
private void CreateIfNeeded()
|
||||
{
|
||||
var database = Path.Combine(_databaseRoot, "dy.sqlite");
|
||||
if (!File.Exists(database)) return;
|
||||
if (File.Exists(_markerPath) && string.Equals(File.ReadAllText(_markerPath).Trim(), _version, StringComparison.Ordinal)) return;
|
||||
|
||||
var backupRoot = Path.Combine(_databaseRoot, "upgrade-backups");
|
||||
Directory.CreateDirectory(backupRoot);
|
||||
var versionPrefix = $"pre-{_version}-";
|
||||
if (!Directory.EnumerateDirectories(backupRoot, versionPrefix + "*").Any())
|
||||
{
|
||||
var destination = Path.Combine(backupRoot, versionPrefix + DateTime.Now.ToString("yyyyMMdd-HHmmss"));
|
||||
Directory.CreateDirectory(destination);
|
||||
CopyIfPresent(database, Path.Combine(destination, "dy.sqlite"));
|
||||
CopyIfPresent(database + "-wal", Path.Combine(destination, "dy.sqlite-wal"));
|
||||
CopyIfPresent(database + "-shm", Path.Combine(destination, "dy.sqlite-shm"));
|
||||
CopyDirectory(Path.Combine(_databaseRoot, "keys"), Path.Combine(destination, "keys"));
|
||||
}
|
||||
|
||||
foreach (var old in new DirectoryInfo(backupRoot).GetDirectories()
|
||||
.OrderByDescending(x => x.CreationTimeUtc).Skip(3))
|
||||
{
|
||||
try { old.Delete(true); }
|
||||
catch (Exception ex) { Serilog.Log.Warning(ex, "清理旧升级备份失败:{Path}", old.FullName); }
|
||||
}
|
||||
}
|
||||
|
||||
private static void CopyIfPresent(string source, string destination)
|
||||
{
|
||||
if (File.Exists(source)) File.Copy(source, destination, false);
|
||||
}
|
||||
|
||||
private static void CopyDirectory(string source, string destination)
|
||||
{
|
||||
if (!Directory.Exists(source)) return;
|
||||
Directory.CreateDirectory(destination);
|
||||
foreach (var file in Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories))
|
||||
{
|
||||
var relative = Path.GetRelativePath(source, file);
|
||||
var target = Path.Combine(destination, relative);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(target)!);
|
||||
File.Copy(file, target, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -135,6 +135,13 @@ namespace dy.net.service
|
||||
return count > 0;
|
||||
}
|
||||
|
||||
public async Task<DouyinVideoDelete> GetDeleteVideoAsync(string videoId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(videoId)) return null;
|
||||
return await sqlSugarClient.Queryable<DouyinVideoDelete>()
|
||||
.Where(x => x.ViedoId == videoId).OrderByDescending(x => x.DeleteTime).FirstAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 新增要删除的视频
|
||||
/// </summary>
|
||||
|
||||
@@ -48,10 +48,15 @@ namespace dy.net.service
|
||||
var d= await _cookieRepository.FastResetCookie(id, cookie);
|
||||
if (d)
|
||||
{
|
||||
var cookies =await _cookieRepository.GetByIdAsync(id);
|
||||
cookies.StatusCode = 0;
|
||||
cookies.StatusMsg = "正常";
|
||||
await _cookieRepository.UpdateAsync(cookies);
|
||||
var cookies = string.IsNullOrWhiteSpace(id)
|
||||
? await _cookieRepository.GetAllAsync()
|
||||
: new List<DouyinCookie> { await _cookieRepository.GetByIdAsync(id) };
|
||||
foreach (var item in cookies.Where(x => x != null))
|
||||
{
|
||||
item.StatusCode = 0;
|
||||
item.StatusMsg = "正常";
|
||||
await _cookieRepository.UpdateAsync(item);
|
||||
}
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
@@ -7,6 +7,13 @@ using SqlSugar;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
public enum AddFollowResult
|
||||
{
|
||||
Added,
|
||||
AlreadyExists,
|
||||
Failed
|
||||
}
|
||||
|
||||
public class DouyinFollowService
|
||||
{
|
||||
|
||||
@@ -30,16 +37,23 @@ namespace dy.net.service
|
||||
/// <param name="followed"></param>
|
||||
/// <returns></returns>
|
||||
public async Task<bool> AddAsync(DouyinFollowed followed)
|
||||
{
|
||||
return await TryAddAsync(followed) == AddFollowResult.Added;
|
||||
}
|
||||
|
||||
public async Task<AddFollowResult> TryAddAsync(DouyinFollowed followed)
|
||||
{
|
||||
var foll = await _followRepository.GetFirstAsync(x => x.SecUid == followed.SecUid && x.mySelfId == followed.mySelfId);
|
||||
if (foll != null)
|
||||
{
|
||||
return false;
|
||||
return AddFollowResult.AlreadyExists;
|
||||
}
|
||||
followed.Id = IdGener.GetLong().ToString();
|
||||
followed.LastSyncTime = DateTime.UtcNow;
|
||||
followed.IsNoFollowed = true;
|
||||
return await _followRepository.InsertAsync(followed);
|
||||
return await _followRepository.InsertAsync(followed)
|
||||
? AddFollowResult.Added
|
||||
: AddFollowResult.Failed;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
+457
-162
@@ -11,12 +11,17 @@ using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using dy.net.storage;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
public class DouyinHttpClientService : IDisposable
|
||||
{
|
||||
private readonly IHttpClientFactory _clientFactory;
|
||||
private readonly MediaStorageRouter _storageRouter;
|
||||
private readonly Func<TimeSpan, CancellationToken, Task> _delayAsync;
|
||||
private const int TransientRequestRetryCount = 2;
|
||||
private const int MediaNetworkRetryCount = 2;
|
||||
private readonly JsonSerializerSettings _jsonSettings=new JsonSerializerSettings
|
||||
{
|
||||
NullValueHandling = NullValueHandling.Ignore,
|
||||
@@ -24,9 +29,19 @@ namespace dy.net.service
|
||||
};
|
||||
private bool _disposedValue;
|
||||
|
||||
public DouyinHttpClientService(IHttpClientFactory clientFactory)
|
||||
public DouyinHttpClientService(IHttpClientFactory clientFactory, MediaStorageRouter storageRouter)
|
||||
: this(clientFactory, storageRouter, Task.Delay)
|
||||
{
|
||||
}
|
||||
|
||||
internal DouyinHttpClientService(
|
||||
IHttpClientFactory clientFactory,
|
||||
MediaStorageRouter storageRouter,
|
||||
Func<TimeSpan, CancellationToken, Task> delayAsync)
|
||||
{
|
||||
_clientFactory = clientFactory;
|
||||
_storageRouter = storageRouter;
|
||||
_delayAsync = delayAsync ?? Task.Delay;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -51,21 +66,46 @@ namespace dy.net.service
|
||||
);
|
||||
}
|
||||
|
||||
using var requestMessage = new HttpRequestMessage(httpMethod, fullUrl);
|
||||
|
||||
if (!string.IsNullOrEmpty(refererValue) && Uri.IsWellFormedUriString(refererValue, UriKind.Absolute))
|
||||
{
|
||||
requestMessage.Headers.Referrer = new Uri(refererValue);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(cookie))
|
||||
{
|
||||
requestMessage.Headers.TryAddWithoutValidation("Cookie", cookie);
|
||||
}
|
||||
|
||||
// 优化:移除using,由IHttpClientFactory管理生命周期
|
||||
var httpClient = _clientFactory.CreateClient(DouyinRequestParamManager.DY_HTTP_CLIENT);
|
||||
return await httpClient.SendAsync(requestMessage);
|
||||
for (var attempt = 0; ; attempt++)
|
||||
{
|
||||
using var requestMessage = new HttpRequestMessage(httpMethod, fullUrl);
|
||||
if (!string.IsNullOrEmpty(refererValue) && Uri.IsWellFormedUriString(refererValue, UriKind.Absolute))
|
||||
requestMessage.Headers.Referrer = new Uri(refererValue);
|
||||
if (!string.IsNullOrEmpty(cookie))
|
||||
requestMessage.Headers.TryAddWithoutValidation("Cookie", cookie);
|
||||
|
||||
try
|
||||
{
|
||||
var response = await httpClient.SendAsync(requestMessage);
|
||||
if (!IsTransientStatus(response.StatusCode) || attempt >= TransientRequestRetryCount)
|
||||
return response;
|
||||
var status = (int)response.StatusCode;
|
||||
response.Dispose();
|
||||
await DelayTransientRetryAsync(attempt + 1, CancellationToken.None);
|
||||
Log.Warning("抖音接口暂时返回 HTTP {Status},正在进行第 {Retry}/{MaxRetry} 次有限重试",
|
||||
status, attempt + 1, TransientRequestRetryCount);
|
||||
}
|
||||
catch (Exception ex) when (IsTransientRequestFailure(ex) && attempt < TransientRequestRetryCount)
|
||||
{
|
||||
Log.Warning("抖音接口连接失败,正在进行第 {Retry}/{MaxRetry} 次有限重试:{Reason}",
|
||||
attempt + 1, TransientRequestRetryCount, ex.GetBaseException().Message);
|
||||
await DelayTransientRetryAsync(attempt + 1, CancellationToken.None);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsTransientStatus(HttpStatusCode status) =>
|
||||
status == HttpStatusCode.RequestTimeout || (int)status >= 500;
|
||||
|
||||
private static bool IsTransientRequestFailure(Exception error) =>
|
||||
error is HttpRequestException or TimeoutException or TaskCanceledException;
|
||||
|
||||
private Task DelayTransientRetryAsync(int retryNumber, CancellationToken cancellationToken)
|
||||
{
|
||||
var exponential = retryNumber <= 1 ? 1 : 3;
|
||||
var jitter = Random.Shared.Next(0, 401);
|
||||
return _delayAsync(TimeSpan.FromMilliseconds(exponential * 1000 + jitter), cancellationToken);
|
||||
}
|
||||
|
||||
#region 收藏夹相关方法(保留原有逻辑,仅优化资源释放)
|
||||
@@ -450,19 +490,21 @@ namespace dy.net.service
|
||||
using var jsonReader = new JsonTextReader(reader);
|
||||
var model = JsonSerializer.Create(_jsonSettings).Deserialize<DouyinVideoInfoResponse>(jsonReader);
|
||||
if (model == null)
|
||||
Log.Error($"SyncUpderPostVideos fail, data is null");
|
||||
throw new InvalidDataException("获取博主作品列表失败:响应内容为空或无法解析");
|
||||
return model;
|
||||
}
|
||||
else
|
||||
{
|
||||
Log.Error($"SyncUpderPostVideos StatusCode fail: {response.StatusCode}");
|
||||
return null;
|
||||
throw new HttpRequestException(
|
||||
$"获取博主作品列表失败:HTTP {(int)response.StatusCode} {response.StatusCode}",
|
||||
null,
|
||||
response.StatusCode);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Error($"SyncUpderPostVideos error: {ex.Message}", ex);
|
||||
return null;
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -538,185 +580,438 @@ namespace dy.net.service
|
||||
}
|
||||
#endregion
|
||||
|
||||
#region 下载相关(优化内存管理)
|
||||
public async Task<(bool Success, string ActualSavePath)> DownloadAsync(
|
||||
#region 下载相关
|
||||
public Task<MediaDownloadResult> DownloadAsync(
|
||||
string videoUrl,
|
||||
string savePath,
|
||||
string cookie,
|
||||
List<string> otherUrls = null,
|
||||
CancellationToken cancellationToken = default,
|
||||
TimeSpan? streamTimeout = null,
|
||||
int maxRetryCount = 3,
|
||||
int maxRetryCount = 12,
|
||||
TimeSpan? initialRetryDelay = null)
|
||||
{
|
||||
// 优化1:隔离外部列表,避免引用泄漏
|
||||
var retryUrls = new List<string> { videoUrl };
|
||||
if (otherUrls != null && otherUrls.Any())
|
||||
{
|
||||
retryUrls.AddRange(otherUrls);
|
||||
maxRetryCount = Math.Min(maxRetryCount, retryUrls.Count); // 防止溢出
|
||||
}
|
||||
|
||||
int retryCount = 0;
|
||||
var retryDelay = initialRetryDelay ?? TimeSpan.FromSeconds(1);
|
||||
streamTimeout ??= TimeSpan.FromSeconds(60);
|
||||
|
||||
while (retryCount < maxRetryCount)
|
||||
{
|
||||
try
|
||||
var urls = BuildCandidateUrls(videoUrl, otherUrls, maxRetryCount);
|
||||
return DownloadCandidatesAsync(
|
||||
urls,
|
||||
savePath,
|
||||
cookie,
|
||||
cancellationToken,
|
||||
streamTimeout ?? TimeSpan.FromSeconds(60),
|
||||
initialRetryDelay ?? TimeSpan.FromSeconds(1),
|
||||
async (response, actualPath, timeout, token) =>
|
||||
{
|
||||
string currentUrl = retryUrls[retryCount];
|
||||
return await TryDownloadOnceAsync(currentUrl, savePath, cookie, cancellationToken, streamTimeout.Value);
|
||||
}
|
||||
catch (Exception ex) when (IsRetryableException(ex) && retryCount < maxRetryCount - 1)
|
||||
{
|
||||
retryCount++;
|
||||
var delay = TimeSpan.FromMilliseconds(retryDelay.TotalMilliseconds * Math.Pow(2, retryCount - 1));
|
||||
Log.Warning(ex, $"下载失败(第{retryCount}/{maxRetryCount}次重试):{videoUrl},将在{delay.TotalSeconds:F1}秒后重试");
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
Log.Information($"重试等待被取消:{videoUrl}");
|
||||
return (false, savePath);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
Log.Information($"下载被取消:{videoUrl}");
|
||||
return (false, savePath);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Error(ex, $"下载失败(不可重试):{videoUrl}");
|
||||
CleanupIncompleteFile(savePath);
|
||||
return (false, savePath);
|
||||
}
|
||||
}
|
||||
|
||||
Log.Error($"下载失败:已达最大重试次数 {maxRetryCount},URL={videoUrl}");
|
||||
return (false, savePath);
|
||||
await WriteLocalAsync(response, actualPath, timeout, token);
|
||||
});
|
||||
}
|
||||
|
||||
private async Task<(bool Success, string ActualSavePath)> TryDownloadOnceAsync(
|
||||
string videoUrl,
|
||||
/// <summary>
|
||||
/// 将抖音媒体写入指定存储。远端存储负责安全中转和最终提交,
|
||||
/// 不会先落入永久本地媒体目录。
|
||||
/// </summary>
|
||||
public async Task<MediaDownloadResult> DownloadToStorageAsync(
|
||||
StorageType storageType,
|
||||
string mediaUrl,
|
||||
string savePath,
|
||||
string cookie,
|
||||
List<string> otherUrls = null,
|
||||
CancellationToken cancellationToken = default,
|
||||
int maxRetryCount = 12,
|
||||
IMediaStorage storageOverride = null)
|
||||
{
|
||||
if (storageType == StorageType.Local)
|
||||
return await DownloadAsync(mediaUrl, savePath, cookie, otherUrls, cancellationToken, maxRetryCount: maxRetryCount);
|
||||
var storage = storageOverride ?? _storageRouter.Resolve(storageType);
|
||||
if (storage.StorageType != storageType)
|
||||
throw new InvalidOperationException("指定的媒体存储与目标存储类型不一致");
|
||||
if (storage is ICanonicalMediaStorage canonicalStorage)
|
||||
savePath = await canonicalStorage.CanonicalizePathAsync(savePath, false, cancellationToken);
|
||||
var urls = BuildCandidateUrls(mediaUrl, otherUrls, maxRetryCount);
|
||||
return await DownloadCandidatesAsync(
|
||||
urls,
|
||||
savePath,
|
||||
cookie,
|
||||
cancellationToken,
|
||||
TimeSpan.FromSeconds(60),
|
||||
TimeSpan.FromSeconds(1),
|
||||
async (response, actualPath, _, token) =>
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var responseStream = await response.Content.ReadAsStreamAsync(token);
|
||||
await storage.WriteAsync(
|
||||
actualPath,
|
||||
responseStream,
|
||||
response.Content.Headers.ContentLength,
|
||||
response.Content.Headers.ContentType?.MediaType,
|
||||
token);
|
||||
}
|
||||
catch (OperationCanceledException) when (token.IsCancellationRequested) { throw; }
|
||||
catch (MediaIntegrityException) { throw; }
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new MediaStorageException($"{storageType} 媒体写入失败:{actualPath}", ex);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private async Task<MediaDownloadResult> DownloadCandidatesAsync(
|
||||
IReadOnlyList<Uri> urls,
|
||||
string savePath,
|
||||
string cookie,
|
||||
CancellationToken cancellationToken,
|
||||
TimeSpan streamTimeout)
|
||||
TimeSpan streamTimeout,
|
||||
TimeSpan retryDelay,
|
||||
Func<HttpResponseMessage, string, TimeSpan, CancellationToken, Task> writeAsync)
|
||||
{
|
||||
DateTime lastStreamActivity = DateTime.UtcNow;
|
||||
string actualSavePath = savePath;
|
||||
string detectedExtension = string.Empty;
|
||||
if (urls.Count == 0)
|
||||
return Failure(MediaDownloadFailureKind.SourceUnavailable, savePath, null, null, 0, null, "没有有效的媒体下载地址。");
|
||||
|
||||
// 确保目录存在
|
||||
var directory = Path.GetDirectoryName(savePath);
|
||||
if (!Directory.Exists(directory))
|
||||
var client = _clientFactory.CreateClient(DouyinRequestParamManager.DY_HTTP_CLIENT_DOWN);
|
||||
MediaDownloadResult lastFailure = null;
|
||||
var forbiddenCount = 0;
|
||||
var attempted = 0;
|
||||
|
||||
foreach (var source in urls)
|
||||
{
|
||||
Directory.CreateDirectory(directory);
|
||||
attempted++;
|
||||
var networkRetry = 0;
|
||||
var rateLimitRetry = 0;
|
||||
while (true)
|
||||
{
|
||||
HttpResponseMessage response = null;
|
||||
Uri finalUri = source;
|
||||
try
|
||||
{
|
||||
(response, finalUri) = await SendWithSafeRedirectsAsync(client, source, cookie, cancellationToken);
|
||||
var status = (int)response.StatusCode;
|
||||
var host = SafeHost(finalUri);
|
||||
|
||||
if (response.StatusCode == HttpStatusCode.Unauthorized)
|
||||
return Failure(MediaDownloadFailureKind.SourceUnauthorized, savePath, status, host, attempted, null,
|
||||
"抖音媒体请求返回 401,需要重新授权。");
|
||||
|
||||
if (response.StatusCode == HttpStatusCode.Forbidden)
|
||||
{
|
||||
forbiddenCount++;
|
||||
lastFailure = Failure(MediaDownloadFailureKind.SourceForbidden, savePath, status, host, attempted, null,
|
||||
"抖音媒体来源拒绝访问(403)。");
|
||||
break;
|
||||
}
|
||||
|
||||
if (response.StatusCode is HttpStatusCode.NotFound or HttpStatusCode.Gone)
|
||||
{
|
||||
lastFailure = Failure(MediaDownloadFailureKind.SourceNotFound, savePath, status, host, attempted, null,
|
||||
$"媒体来源返回 {status},作品可能已下架或链接已失效。");
|
||||
break;
|
||||
}
|
||||
|
||||
if ((int)response.StatusCode == 429)
|
||||
{
|
||||
var retryAt = GetRetryAfter(response);
|
||||
if (rateLimitRetry++ == 0)
|
||||
{
|
||||
var wait = retryAt.HasValue ? retryAt.Value - DateTime.UtcNow : TimeSpan.FromSeconds(5);
|
||||
if (wait < TimeSpan.Zero) wait = TimeSpan.Zero;
|
||||
if (wait > TimeSpan.FromSeconds(60)) wait = TimeSpan.FromSeconds(60);
|
||||
await _delayAsync(wait, cancellationToken);
|
||||
continue;
|
||||
}
|
||||
return Failure(MediaDownloadFailureKind.SourceRateLimited, savePath, status, host, attempted, retryAt,
|
||||
"抖音媒体来源请求过于频繁(429),已进入冷却等待。");
|
||||
}
|
||||
|
||||
if ((int)response.StatusCode >= 500)
|
||||
{
|
||||
if (networkRetry < MediaNetworkRetryCount)
|
||||
{
|
||||
networkRetry++;
|
||||
await DelayMediaRetryAsync(retryDelay, networkRetry, cancellationToken);
|
||||
continue;
|
||||
}
|
||||
lastFailure = Failure(MediaDownloadFailureKind.SourceUnavailable, savePath, status, host, attempted, null,
|
||||
$"媒体来源暂时不可用({status})。");
|
||||
break;
|
||||
}
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
lastFailure = Failure(MediaDownloadFailureKind.SourceUnavailable, savePath, status, host, attempted, null,
|
||||
$"媒体来源请求失败({status})。");
|
||||
break;
|
||||
}
|
||||
|
||||
var mediaType = response.Content.Headers.ContentType?.MediaType;
|
||||
if (IsObviousNonMedia(mediaType))
|
||||
{
|
||||
lastFailure = Failure(MediaDownloadFailureKind.SourceInvalidContent, savePath, status, host, attempted, null,
|
||||
$"媒体来源返回了非媒体内容({mediaType})。");
|
||||
break;
|
||||
}
|
||||
if (response.Content.Headers.ContentLength == 0)
|
||||
{
|
||||
lastFailure = Failure(MediaDownloadFailureKind.IntegrityCheckFailed, savePath, status, host, attempted, null,
|
||||
"媒体来源返回空内容。");
|
||||
break;
|
||||
}
|
||||
|
||||
var actualPath = DetectActualSavePath(savePath, mediaType);
|
||||
await writeAsync(response, actualPath, streamTimeout, cancellationToken);
|
||||
return MediaDownloadResult.Succeeded(actualPath, host, attempted);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
CleanupIncompleteFile(savePath);
|
||||
return Failure(MediaDownloadFailureKind.Cancelled, savePath, null, SafeHost(finalUri), attempted, null, "媒体下载已取消。");
|
||||
}
|
||||
catch (MediaStorageException ex)
|
||||
{
|
||||
CleanupIncompleteFile(savePath);
|
||||
Log.Error(ex, "媒体存储写入失败:{Path}", savePath);
|
||||
return Failure(MediaDownloadFailureKind.StorageUnavailable, savePath, null, SafeHost(finalUri), attempted, null,
|
||||
DescribeStorageFailure(ex));
|
||||
}
|
||||
catch (MediaIntegrityException ex)
|
||||
{
|
||||
CleanupIncompleteFile(savePath);
|
||||
lastFailure = Failure(MediaDownloadFailureKind.IntegrityCheckFailed, savePath, null, SafeHost(finalUri), attempted, null, ex.Message);
|
||||
break;
|
||||
}
|
||||
catch (MediaSourceException ex)
|
||||
{
|
||||
lastFailure = Failure(ex.FailureKind, savePath, ex.StatusCode, ex.SourceHost, attempted, ex.RetryAfter, ex.Message);
|
||||
break;
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or TimeoutException or TaskCanceledException or IOException)
|
||||
{
|
||||
if (networkRetry < MediaNetworkRetryCount)
|
||||
{
|
||||
networkRetry++;
|
||||
await DelayMediaRetryAsync(retryDelay, networkRetry, cancellationToken);
|
||||
continue;
|
||||
}
|
||||
lastFailure = Failure(MediaDownloadFailureKind.SourceUnavailable, savePath, null, SafeHost(finalUri), attempted, null,
|
||||
"连接媒体来源失败。");
|
||||
break;
|
||||
}
|
||||
finally
|
||||
{
|
||||
response?.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 清理已存在的文件
|
||||
CleanupIncompleteFile(savePath);
|
||||
if (forbiddenCount == attempted && attempted > 0)
|
||||
lastFailure = Failure(MediaDownloadFailureKind.SourceForbidden, savePath, 403, lastFailure?.SourceHost,
|
||||
attempted, null, $"全部 {attempted} 个候选媒体地址均返回 403。");
|
||||
else if (lastFailure?.FailureKind == MediaDownloadFailureKind.SourceForbidden)
|
||||
lastFailure = Failure(MediaDownloadFailureKind.SourceUnavailable, savePath, null, lastFailure.SourceHost,
|
||||
attempted, null, "候选媒体地址返回了多种来源错误。");
|
||||
|
||||
// 优化2:使用using包裹下载专用HttpClient(工厂创建的Client仍可using,释放内部handler)
|
||||
using var httpClient = _clientFactory.CreateClient(DouyinRequestParamManager.DY_HTTP_CLIENT_DOWN);
|
||||
lastFailure ??= Failure(MediaDownloadFailureKind.SourceUnavailable, savePath, null, null, attempted, null, "所有候选媒体地址均不可用。");
|
||||
if (lastFailure.FailureKind == MediaDownloadFailureKind.SourceForbidden)
|
||||
Log.Warning("抖音媒体来源被拒绝:Host={SourceHost}, Status={Status}, Candidates={Candidates}",
|
||||
lastFailure.SourceHost, lastFailure.HttpStatusCode, lastFailure.AttemptedUrlCount);
|
||||
else
|
||||
Log.Warning("媒体来源下载失败:Host={SourceHost}, Status={Status}, Kind={Kind}, Candidates={Candidates}",
|
||||
lastFailure.SourceHost, lastFailure.HttpStatusCode, lastFailure.FailureKind, lastFailure.AttemptedUrlCount);
|
||||
return lastFailure;
|
||||
}
|
||||
|
||||
private Task DelayMediaRetryAsync(TimeSpan baseDelay, int retryNumber, CancellationToken cancellationToken)
|
||||
{
|
||||
var multiplier = Math.Pow(2, Math.Max(0, retryNumber - 1));
|
||||
var milliseconds = Math.Min(5000, Math.Max(100, baseDelay.TotalMilliseconds * multiplier));
|
||||
milliseconds += Random.Shared.Next(0, 251);
|
||||
return _delayAsync(TimeSpan.FromMilliseconds(milliseconds), cancellationToken);
|
||||
}
|
||||
|
||||
private static List<Uri> BuildCandidateUrls(string primary, IEnumerable<string> alternatives, int limit)
|
||||
{
|
||||
var result = new List<Uri>();
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
foreach (var value in new[] { primary }.Concat(alternatives ?? Enumerable.Empty<string>()))
|
||||
{
|
||||
if (result.Count >= Math.Clamp(limit, 1, 12)) break;
|
||||
if (!Uri.TryCreate(value, UriKind.Absolute, out var uri) || (uri.Scheme != Uri.UriSchemeHttp && uri.Scheme != Uri.UriSchemeHttps)) continue;
|
||||
if (seen.Add(uri.AbsoluteUri)) result.Add(uri);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
internal static string DescribeStorageFailure(MediaStorageException error)
|
||||
{
|
||||
if (error == null) return "媒体存储写入失败。";
|
||||
var message = error.Message?.Trim();
|
||||
var reason = error.GetBaseException()?.Message?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(message)) message = "媒体存储写入失败。";
|
||||
if (string.IsNullOrWhiteSpace(reason) || string.Equals(message, reason, StringComparison.Ordinal))
|
||||
return message;
|
||||
return $"{message};原因:{reason}";
|
||||
}
|
||||
|
||||
private static async Task<(HttpResponseMessage Response, Uri FinalUri)> SendWithSafeRedirectsAsync(
|
||||
HttpClient client,
|
||||
Uri initialUri,
|
||||
string cookie,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var current = initialUri;
|
||||
for (var redirect = 0; redirect <= 5; redirect++)
|
||||
{
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, current);
|
||||
if (!string.IsNullOrWhiteSpace(cookie) && IsDouyinHost(current.Host))
|
||||
request.Headers.TryAddWithoutValidation("Cookie", cookie);
|
||||
var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
|
||||
if (!IsRedirect(response.StatusCode)) return (response, current);
|
||||
|
||||
var location = response.Headers.Location;
|
||||
response.Dispose();
|
||||
if (location == null)
|
||||
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable, "媒体来源返回了无目标地址的跳转。",
|
||||
(int)response.StatusCode, SafeHost(current));
|
||||
if (redirect == 5)
|
||||
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable, "媒体来源跳转次数超过 5 次。",
|
||||
(int)response.StatusCode, SafeHost(current));
|
||||
current = location.IsAbsoluteUri ? location : new Uri(current, location);
|
||||
if (current.Scheme != Uri.UriSchemeHttp && current.Scheme != Uri.UriSchemeHttps)
|
||||
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable, "媒体来源返回了不安全的跳转协议。",
|
||||
sourceHost: SafeHost(current));
|
||||
}
|
||||
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable, "媒体来源跳转失败。", sourceHost: SafeHost(current));
|
||||
}
|
||||
|
||||
private static async Task WriteLocalAsync(
|
||||
HttpResponseMessage response,
|
||||
string actualSavePath,
|
||||
TimeSpan streamTimeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var directory = Path.GetDirectoryName(actualSavePath);
|
||||
FileStream fileStream = null;
|
||||
try
|
||||
{
|
||||
// 配置请求头
|
||||
httpClient.DefaultRequestHeaders.Remove("Cookie"); // 先移除再添加,避免重复
|
||||
httpClient.DefaultRequestHeaders.Add("Cookie", cookie);
|
||||
httpClient.Timeout = TimeSpan.FromMinutes(5);
|
||||
|
||||
// 优化3:使用HttpCompletionOption.ResponseHeadersRead,不缓存整个响应
|
||||
using var response = await httpClient.GetAsync(
|
||||
videoUrl,
|
||||
HttpCompletionOption.ResponseHeadersRead,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
long? totalBytes = response.Content.Headers.ContentLength;
|
||||
|
||||
// 检测文件类型
|
||||
var ext = Path.GetExtension(savePath);
|
||||
if (ext == ".mp3")
|
||||
{
|
||||
var contentType = response.Content.Headers.ContentType?.MediaType;
|
||||
if (!string.IsNullOrEmpty(contentType))
|
||||
{
|
||||
if (contentType.Contains("audio/mp4") || contentType.Contains("audio/m4a"))
|
||||
detectedExtension = "m4a";
|
||||
else if (contentType.Contains("audio/mpeg") || contentType.Contains("audio/mp3"))
|
||||
detectedExtension = "mp3";
|
||||
else if (contentType.Contains("video/mp4"))
|
||||
detectedExtension = "mp4";
|
||||
}
|
||||
}
|
||||
|
||||
// 修正保存路径
|
||||
if (!string.IsNullOrEmpty(detectedExtension))
|
||||
{
|
||||
actualSavePath = Path.ChangeExtension(actualSavePath, detectedExtension.Trim('.'));
|
||||
CleanupIncompleteFile(actualSavePath);
|
||||
}
|
||||
|
||||
// 优化4:流式写入文件,且使用ArrayPool复用缓冲区(减少大对象分配)
|
||||
using var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
using var fileStream = new FileStream(
|
||||
actualSavePath,
|
||||
FileMode.CreateNew,
|
||||
FileAccess.Write,
|
||||
FileShare.None,
|
||||
bufferSize: 8192,
|
||||
FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
|
||||
var buffer = System.Buffers.ArrayPool<byte>.Shared.Rent(8192); // 复用缓冲区
|
||||
try
|
||||
{
|
||||
int bytesRead;
|
||||
long totalRead = 0;
|
||||
if (!string.IsNullOrWhiteSpace(directory)) Directory.CreateDirectory(directory);
|
||||
CleanupIncompleteFile(actualSavePath);
|
||||
fileStream = new FileStream(actualSavePath, FileMode.CreateNew, FileAccess.Write, FileShare.None,
|
||||
81920, FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
}
|
||||
catch (Exception ex) when (ex is UnauthorizedAccessException or IOException)
|
||||
{
|
||||
throw new MediaStorageException($"本地媒体写入失败:{actualSavePath}", ex);
|
||||
}
|
||||
|
||||
while ((bytesRead = await responseStream.ReadAsync(buffer, 0, buffer.Length, cancellationToken).ConfigureAwait(false)) > 0)
|
||||
Stream responseStream;
|
||||
try { responseStream = await response.Content.ReadAsStreamAsync(cancellationToken); }
|
||||
catch (Exception ex) when (ex is HttpRequestException or IOException)
|
||||
{
|
||||
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable, "读取媒体来源响应失败。",
|
||||
sourceHost: SafeHost(response.RequestMessage?.RequestUri), innerException: ex);
|
||||
}
|
||||
await using var ownedResponseStream = responseStream;
|
||||
var buffer = System.Buffers.ArrayPool<byte>.Shared.Rent(81920);
|
||||
long totalRead = 0;
|
||||
var lastActivity = DateTime.UtcNow;
|
||||
try
|
||||
{
|
||||
int read;
|
||||
while (true)
|
||||
{
|
||||
if (DateTime.UtcNow - lastStreamActivity > streamTimeout)
|
||||
throw new TimeoutException($"流读取超时({streamTimeout.TotalSeconds}秒无数据)");
|
||||
|
||||
await fileStream.WriteAsync(buffer, 0, bytesRead, cancellationToken).ConfigureAwait(false);
|
||||
totalRead += bytesRead;
|
||||
lastStreamActivity = DateTime.UtcNow;
|
||||
try { read = await responseStream.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken); }
|
||||
catch (Exception ex) when (ex is HttpRequestException or IOException)
|
||||
{
|
||||
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable, "读取媒体来源响应失败。",
|
||||
sourceHost: SafeHost(response.RequestMessage?.RequestUri), innerException: ex);
|
||||
}
|
||||
if (read <= 0) break;
|
||||
if (DateTime.UtcNow - lastActivity > streamTimeout)
|
||||
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable,
|
||||
$"媒体流超过 {streamTimeout.TotalSeconds:0} 秒没有数据。");
|
||||
try { await fileStream.WriteAsync(buffer.AsMemory(0, read), cancellationToken); }
|
||||
catch (Exception ex) when (ex is UnauthorizedAccessException or IOException)
|
||||
{
|
||||
throw new MediaStorageException($"本地媒体写入失败:{actualSavePath}", ex);
|
||||
}
|
||||
totalRead += read;
|
||||
lastActivity = DateTime.UtcNow;
|
||||
}
|
||||
try { await fileStream.FlushAsync(cancellationToken); }
|
||||
catch (Exception ex) when (ex is UnauthorizedAccessException or IOException)
|
||||
{
|
||||
throw new MediaStorageException($"本地媒体写入失败:{actualSavePath}", ex);
|
||||
}
|
||||
|
||||
await fileStream.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
System.Buffers.ArrayPool<byte>.Shared.Return(buffer); // 归还缓冲区
|
||||
System.Buffers.ArrayPool<byte>.Shared.Return(buffer);
|
||||
}
|
||||
|
||||
return (true, actualSavePath);
|
||||
}
|
||||
catch
|
||||
{
|
||||
CleanupIncompleteFile(actualSavePath);
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
//httpClient.DefaultRequestHeaders.Clear(); // 清空请求头,帮助GC
|
||||
if (totalRead <= 0) throw new MediaIntegrityException("下载内容为空。");
|
||||
var expected = response.Content.Headers.ContentLength;
|
||||
if (expected.HasValue && expected.Value != totalRead)
|
||||
throw new MediaIntegrityException($"下载长度不一致:期望 {expected.Value},实际 {totalRead}。");
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
|
||||
catch (MediaSourceException) { CleanupIncompleteFile(actualSavePath); throw; }
|
||||
catch (MediaIntegrityException) { CleanupIncompleteFile(actualSavePath); throw; }
|
||||
catch (MediaStorageException) { CleanupIncompleteFile(actualSavePath); throw; }
|
||||
finally { if (fileStream != null) await fileStream.DisposeAsync(); }
|
||||
}
|
||||
|
||||
private bool IsRetryableException(Exception ex)
|
||||
private static string DetectActualSavePath(string savePath, string mediaType)
|
||||
{
|
||||
return ex is HttpRequestException
|
||||
|| ex is TimeoutException
|
||||
|| ex is IOException
|
||||
|| (ex is AggregateException aggEx && aggEx.InnerExceptions.Any(IsRetryableException));
|
||||
if (!Path.GetExtension(savePath).Equals(".mp3", StringComparison.OrdinalIgnoreCase)) return savePath;
|
||||
if (mediaType?.Contains("audio/mp4", StringComparison.OrdinalIgnoreCase) == true
|
||||
|| mediaType?.Contains("audio/m4a", StringComparison.OrdinalIgnoreCase) == true)
|
||||
return Path.ChangeExtension(savePath, ".m4a");
|
||||
if (mediaType?.Contains("video/mp4", StringComparison.OrdinalIgnoreCase) == true)
|
||||
return Path.ChangeExtension(savePath, ".mp4");
|
||||
return savePath;
|
||||
}
|
||||
|
||||
private static bool IsObviousNonMedia(string mediaType) =>
|
||||
!string.IsNullOrWhiteSpace(mediaType)
|
||||
&& (mediaType.StartsWith("text/html", StringComparison.OrdinalIgnoreCase)
|
||||
|| mediaType.Contains("json", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
private static bool IsDouyinHost(string host) =>
|
||||
string.Equals(host, "douyin.com", StringComparison.OrdinalIgnoreCase)
|
||||
|| host.EndsWith(".douyin.com", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static bool IsRedirect(HttpStatusCode code) => code is
|
||||
HttpStatusCode.MovedPermanently or HttpStatusCode.Redirect or HttpStatusCode.RedirectMethod
|
||||
or HttpStatusCode.TemporaryRedirect or HttpStatusCode.PermanentRedirect;
|
||||
|
||||
private static string SafeHost(Uri uri) => uri?.IsAbsoluteUri == true ? uri.IdnHost : null;
|
||||
|
||||
private static DateTime? GetRetryAfter(HttpResponseMessage response)
|
||||
{
|
||||
var value = response.Headers.RetryAfter;
|
||||
if (value?.Date.HasValue == true) return value.Date.Value.UtcDateTime;
|
||||
if (value?.Delta.HasValue == true) return DateTime.UtcNow.Add(value.Delta.Value);
|
||||
return DateTime.UtcNow.AddMinutes(15);
|
||||
}
|
||||
|
||||
private static MediaDownloadResult Failure(
|
||||
MediaDownloadFailureKind kind,
|
||||
string savePath,
|
||||
int? status,
|
||||
string host,
|
||||
int attempted,
|
||||
DateTime? retryAfter,
|
||||
string message) => new()
|
||||
{
|
||||
Success = false,
|
||||
ActualSavePath = savePath,
|
||||
FailureKind = kind,
|
||||
HttpStatusCode = status,
|
||||
SourceHost = host,
|
||||
AttemptedUrlCount = attempted,
|
||||
RetryAfter = retryAfter,
|
||||
Message = message
|
||||
};
|
||||
|
||||
private static void CleanupIncompleteFile(string savePath)
|
||||
{
|
||||
if (File.Exists(savePath))
|
||||
@@ -757,4 +1052,4 @@ namespace dy.net.service
|
||||
}
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
using System.Net;
|
||||
using dy.net.model.dto;
|
||||
using dy.net.utils;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
public interface IDouyinLiveStatusClient
|
||||
{
|
||||
Task<DouyinLiveStatusProbe> ProbeAsync(
|
||||
string secUid,
|
||||
string cookie,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
public sealed class DouyinLiveStatusRequestException : InvalidOperationException
|
||||
{
|
||||
public DouyinLiveStatusRequestException(
|
||||
string message,
|
||||
HttpStatusCode? statusCode = null,
|
||||
bool requiresAccountCooldown = false,
|
||||
Exception innerException = null)
|
||||
: base(message, innerException)
|
||||
{
|
||||
StatusCode = statusCode;
|
||||
RequiresAccountCooldown = requiresAccountCooldown;
|
||||
}
|
||||
|
||||
public HttpStatusCode? StatusCode { get; }
|
||||
public bool RequiresAccountCooldown { get; }
|
||||
}
|
||||
|
||||
public sealed class DouyinLiveStatusClient : IDouyinLiveStatusClient
|
||||
{
|
||||
private const string ProfilePath = "/aweme/v1/web/user/profile/other/";
|
||||
private readonly IHttpClientFactory _clientFactory;
|
||||
private readonly DouyinABogusSigner _signer = new();
|
||||
|
||||
public DouyinLiveStatusClient(IHttpClientFactory clientFactory) => _clientFactory = clientFactory;
|
||||
|
||||
public async Task<DouyinLiveStatusProbe> ProbeAsync(
|
||||
string secUid,
|
||||
string cookie,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(secUid))
|
||||
throw new InvalidOperationException("博主 SecUid 为空,无法检查直播状态");
|
||||
if (string.IsNullOrWhiteSpace(cookie))
|
||||
throw new InvalidOperationException("授权 Cookie 为空,无法检查直播状态");
|
||||
|
||||
var parameters = BuildParameters(secUid.Trim(), cookie);
|
||||
var unsignedQuery = BuildQueryString(parameters);
|
||||
var requestUri = ProfilePath + "?" + unsignedQuery + "&a_bogus=" + Encode(_signer.Sign(unsignedQuery));
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
|
||||
request.Headers.Referrer = new Uri("https://www.douyin.com/user/" + Encode(secUid.Trim()));
|
||||
request.Headers.TryAddWithoutValidation("Accept", "application/json, text/plain, */*");
|
||||
request.Headers.TryAddWithoutValidation("Cookie", cookie);
|
||||
|
||||
var client = _clientFactory.CreateClient(DouyinRequestParamManager.DY_HTTP_CLIENT);
|
||||
try
|
||||
{
|
||||
using var response = await client.SendAsync(
|
||||
request,
|
||||
HttpCompletionOption.ResponseHeadersRead,
|
||||
cancellationToken);
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
var shouldCooldown = response.StatusCode is HttpStatusCode.Forbidden
|
||||
or HttpStatusCode.TooManyRequests
|
||||
or HttpStatusCode.Unauthorized;
|
||||
throw new DouyinLiveStatusRequestException(
|
||||
$"抖音直播状态请求失败(HTTP {(int)response.StatusCode})",
|
||||
response.StatusCode,
|
||||
shouldCooldown);
|
||||
}
|
||||
|
||||
var content = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
UserProfileResponse payload;
|
||||
try
|
||||
{
|
||||
payload = JsonConvert.DeserializeObject<UserProfileResponse>(content);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
throw new DouyinLiveStatusRequestException(
|
||||
"抖音返回了无法识别的直播状态响应,可能需要验证授权",
|
||||
requiresAccountCooldown: true,
|
||||
innerException: ex);
|
||||
}
|
||||
|
||||
if (payload?.User == null)
|
||||
throw new DouyinLiveStatusRequestException(
|
||||
"抖音没有返回博主资料,可能需要验证授权",
|
||||
requiresAccountCooldown: true);
|
||||
if (payload.StatusCode != 0)
|
||||
throw new DouyinLiveStatusRequestException(
|
||||
$"抖音拒绝了直播状态查询(状态码 {payload.StatusCode})",
|
||||
requiresAccountCooldown: true);
|
||||
|
||||
return DouyinLiveStatusParser.Parse(
|
||||
payload.User.LiveStatus,
|
||||
payload.User.RoomId,
|
||||
payload.User.RoomIdStr,
|
||||
payload.User.RoomData);
|
||||
}
|
||||
catch (DouyinLiveStatusRequestException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (HttpRequestException ex)
|
||||
{
|
||||
throw new DouyinLiveStatusRequestException("抖音直播状态网络请求失败,请稍后重试", innerException: ex);
|
||||
}
|
||||
catch (OperationCanceledException ex) when (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw new DouyinLiveStatusRequestException("抖音直播状态查询超时,请稍后重试", innerException: ex);
|
||||
}
|
||||
}
|
||||
|
||||
private static List<KeyValuePair<string, string>> BuildParameters(string secUid, string cookie)
|
||||
{
|
||||
var cookies = ParseCookie(cookie);
|
||||
return new List<KeyValuePair<string, string>>
|
||||
{
|
||||
new("device_platform", "webapp"),
|
||||
new("aid", "6383"),
|
||||
new("channel", "channel_pc_web"),
|
||||
new("update_version_code", "170400"),
|
||||
new("pc_client_type", "1"),
|
||||
new("pc_libra_divert", "Windows"),
|
||||
new("support_h265", "1"),
|
||||
new("support_dash", "1"),
|
||||
new("version_code", "170400"),
|
||||
new("version_name", "17.4.0"),
|
||||
new("cookie_enabled", "true"),
|
||||
new("screen_width", "1536"),
|
||||
new("screen_height", "864"),
|
||||
new("browser_language", "zh-CN"),
|
||||
new("browser_platform", "Win32"),
|
||||
new("browser_name", "Chrome"),
|
||||
new("browser_version", "119.0.0.0"),
|
||||
new("browser_online", "true"),
|
||||
new("engine_name", "Blink"),
|
||||
new("engine_version", "119.0.0.0"),
|
||||
new("os_name", "Windows"),
|
||||
new("os_version", "10"),
|
||||
new("cpu_core_num", "16"),
|
||||
new("device_memory", "8"),
|
||||
new("platform", "PC"),
|
||||
new("downlink", "10"),
|
||||
new("effective_type", "4g"),
|
||||
new("round_trip_time", "200"),
|
||||
new("uifid", CookieValue(cookies, "UIFID", "UIFID_TEMP")),
|
||||
new("msToken", CookieValue(cookies, "msToken")),
|
||||
new("sec_user_id", secUid),
|
||||
new("publish_video_strategy_type", "2"),
|
||||
new("personal_center_strategy", "1")
|
||||
};
|
||||
}
|
||||
|
||||
private static Dictionary<string, string> ParseCookie(string cookie)
|
||||
{
|
||||
var values = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var part in (cookie ?? string.Empty).Split(';', StringSplitOptions.RemoveEmptyEntries))
|
||||
{
|
||||
var separator = part.IndexOf('=');
|
||||
if (separator <= 0) continue;
|
||||
values[part[..separator].Trim()] = part[(separator + 1)..].Trim();
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
private static string CookieValue(Dictionary<string, string> cookies, params string[] names)
|
||||
{
|
||||
foreach (var name in names)
|
||||
if (cookies.TryGetValue(name, out var value)) return value;
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
private static string BuildQueryString(IEnumerable<KeyValuePair<string, string>> parameters) =>
|
||||
string.Join("&", parameters.Select(x => Encode(x.Key) + "=" + Encode(x.Value ?? string.Empty)));
|
||||
|
||||
private static string Encode(string value) => Uri.EscapeDataString(value ?? string.Empty);
|
||||
|
||||
private sealed class UserProfileResponse
|
||||
{
|
||||
[JsonProperty("status_code")]
|
||||
public int StatusCode { get; set; }
|
||||
|
||||
[JsonProperty("user")]
|
||||
public UserProfile User { get; set; }
|
||||
}
|
||||
|
||||
private sealed class UserProfile
|
||||
{
|
||||
[JsonProperty("live_status")]
|
||||
public int? LiveStatus { get; set; }
|
||||
|
||||
[JsonProperty("room_id")]
|
||||
public string RoomId { get; set; }
|
||||
|
||||
[JsonProperty("room_id_str")]
|
||||
public string RoomIdStr { get; set; }
|
||||
|
||||
[JsonProperty("room_data")]
|
||||
public JToken RoomData { get; set; }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,482 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net;
|
||||
using dy.net.model.dto;
|
||||
using dy.net.model.entity;
|
||||
using dy.net.model.response;
|
||||
using dy.net.utils;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Serilog;
|
||||
using SqlSugar;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
public sealed class DouyinLiveStatusService
|
||||
{
|
||||
private static readonly TimeSpan RefreshInterval = TimeSpan.FromMinutes(5);
|
||||
private static readonly TimeSpan ManualRefreshFloor = TimeSpan.FromSeconds(30);
|
||||
private static readonly TimeSpan StaleAfter = TimeSpan.FromMinutes(10);
|
||||
private static readonly ConcurrentDictionary<string, SemaphoreSlim> NotificationGates = new();
|
||||
private readonly ISqlSugarClient _db;
|
||||
private readonly IDouyinLiveStatusClient _client;
|
||||
private readonly LiveEmailNotificationService _emailNotifications;
|
||||
|
||||
public DouyinLiveStatusService(
|
||||
ISqlSugarClient db,
|
||||
IDouyinLiveStatusClient client,
|
||||
LiveEmailNotificationService emailNotifications)
|
||||
{
|
||||
_db = db;
|
||||
_client = client;
|
||||
_emailNotifications = emailNotifications;
|
||||
}
|
||||
|
||||
public async Task<FollowLiveStatusDto> SetMonitorAsync(
|
||||
FollowLiveMonitorUpdateDto request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (request == null || string.IsNullOrWhiteSpace(request.Id))
|
||||
throw new InvalidOperationException("博主记录不能为空");
|
||||
var follow = await RequireFollowAsync(request.Id);
|
||||
if (request.Enabled && string.IsNullOrWhiteSpace(follow.SecUid))
|
||||
throw new InvalidOperationException("该博主缺少 SecUid,无法开启直播监测");
|
||||
|
||||
follow.LiveMonitorEnabled = request.Enabled;
|
||||
if (!request.Enabled)
|
||||
{
|
||||
follow.LiveStatus = DouyinLiveStatusState.Unknown;
|
||||
follow.LiveRoomId = null;
|
||||
follow.LiveWebRid = null;
|
||||
follow.LiveTitle = null;
|
||||
follow.LiveStartedAt = null;
|
||||
follow.LiveCheckedAt = null;
|
||||
follow.LiveStatusUpdatedAt = null;
|
||||
follow.LiveCheckError = null;
|
||||
follow.LiveEmailNotificationEnabled = false;
|
||||
follow.LastLiveNotificationKey = null;
|
||||
follow.LastLiveNotificationAttemptAt = null;
|
||||
follow.LastLiveNotificationError = null;
|
||||
}
|
||||
else if (!follow.LiveCheckedAt.HasValue)
|
||||
{
|
||||
follow.LiveStatus = DouyinLiveStatusState.Unknown;
|
||||
follow.LiveCheckError = null;
|
||||
}
|
||||
|
||||
await UpdateFollowLiveColumnsAsync(follow);
|
||||
if (!request.Enabled) await UpdateFollowNotificationColumnsAsync(follow);
|
||||
if (request.Enabled)
|
||||
follow = await ProbeFollowAsync(follow, ignoreRecentCheck: true, cancellationToken);
|
||||
return ToDto(follow);
|
||||
}
|
||||
|
||||
public async Task<FollowLiveStatusDto> SetEmailNotificationAsync(FollowLiveEmailUpdateDto request)
|
||||
{
|
||||
if (request == null || string.IsNullOrWhiteSpace(request.Id))
|
||||
throw new InvalidOperationException("博主记录不能为空");
|
||||
var follow = await RequireFollowAsync(request.Id);
|
||||
if (request.Enabled && !follow.LiveMonitorEnabled)
|
||||
throw new InvalidOperationException("请先开启该博主的直播监测");
|
||||
if (request.Enabled)
|
||||
{
|
||||
var readinessError = await _emailNotifications.GetReadinessErrorAsync();
|
||||
if (!string.IsNullOrWhiteSpace(readinessError)) throw new InvalidOperationException(readinessError);
|
||||
}
|
||||
follow.LiveEmailNotificationEnabled = request.Enabled;
|
||||
follow.LastLiveNotificationError = null;
|
||||
if (!request.Enabled)
|
||||
{
|
||||
follow.LastLiveNotificationKey = null;
|
||||
follow.LastLiveNotificationAttemptAt = null;
|
||||
}
|
||||
await UpdateFollowNotificationColumnsAsync(follow);
|
||||
if (request.Enabled && follow.LiveStatus == DouyinLiveStatusState.Live)
|
||||
{
|
||||
await HandleLiveNotificationAsync(follow, DouyinLiveStatusState.Unknown, CancellationToken.None);
|
||||
follow = await RequireFollowAsync(follow.Id);
|
||||
}
|
||||
return ToDto(follow);
|
||||
}
|
||||
|
||||
public async Task<FollowLiveStatusDto> RefreshAsync(
|
||||
string id,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var follow = await RequireFollowAsync(id);
|
||||
if (!follow.LiveMonitorEnabled)
|
||||
throw new InvalidOperationException("请先开启该博主的直播监测");
|
||||
follow = await ProbeFollowAsync(follow, ignoreRecentCheck: false, cancellationToken);
|
||||
return ToDto(follow);
|
||||
}
|
||||
|
||||
public async Task<List<FollowLiveStatusDto>> QueryAsync(IEnumerable<string> ids)
|
||||
{
|
||||
var normalized = (ids ?? Array.Empty<string>())
|
||||
.Where(x => !string.IsNullOrWhiteSpace(x))
|
||||
.Select(x => x.Trim())
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.Take(100)
|
||||
.ToList();
|
||||
if (normalized.Count == 0) return new List<FollowLiveStatusDto>();
|
||||
var follows = await _db.Queryable<DouyinFollowed>()
|
||||
.Where(x => normalized.Contains(x.Id))
|
||||
.ToListAsync();
|
||||
return follows.Select(ToDto).ToList();
|
||||
}
|
||||
|
||||
public async Task RefreshDueAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var cutoff = DateTime.Now.Subtract(RefreshInterval);
|
||||
var follows = await _db.Queryable<DouyinFollowed>()
|
||||
.Where(x => x.LiveMonitorEnabled && (x.LiveCheckedAt == null || x.LiveCheckedAt <= cutoff))
|
||||
.OrderBy(x => x.LiveCheckedAt)
|
||||
.ToListAsync();
|
||||
if (follows.Count == 0) return;
|
||||
|
||||
var myUserIds = follows.Select(x => x.mySelfId).Where(x => !string.IsNullOrWhiteSpace(x)).Distinct().ToList();
|
||||
var cookies = await _db.Queryable<DouyinCookie>()
|
||||
.Where(x => myUserIds.Contains(x.MyUserId))
|
||||
.ToListAsync();
|
||||
|
||||
foreach (var group in follows.GroupBy(x => x.mySelfId, StringComparer.Ordinal))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var cookie = SelectCookie(cookies, group.Key);
|
||||
if (cookie == null || cookie.Status != 1 || cookie.StatusCode != 0 || string.IsNullOrWhiteSpace(cookie.Cookies))
|
||||
{
|
||||
await MarkGroupUnavailableAsync(group, "授权账号无效或未启用,直播状态暂时无法检查");
|
||||
continue;
|
||||
}
|
||||
if (cookie.LiveCheckCooldownUntil.HasValue && cookie.LiveCheckCooldownUntil > DateTime.Now)
|
||||
{
|
||||
await MarkGroupUnavailableAsync(group,
|
||||
$"直播状态检查已冷却至 {cookie.LiveCheckCooldownUntil.Value.ToLocalTime():MM-dd HH:mm}");
|
||||
continue;
|
||||
}
|
||||
|
||||
var items = group.ToList();
|
||||
for (var index = 0; index < items.Count; index++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var result = await ProbeWithCookieAsync(items[index], cookie, cancellationToken);
|
||||
if (result.ShouldStopAccount) break;
|
||||
if (index < items.Count - 1)
|
||||
await Task.Delay(Random.Shared.Next(750, 1501), cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async Task ApplyFollowListStatusesAsync(
|
||||
IEnumerable<FollowingsItem> followings,
|
||||
DouyinCookie cookie)
|
||||
{
|
||||
if (cookie == null || string.IsNullOrWhiteSpace(cookie.MyUserId)) return;
|
||||
var source = (followings ?? Array.Empty<FollowingsItem>())
|
||||
.Where(x => x != null && !string.IsNullOrWhiteSpace(x.SecUid))
|
||||
.GroupBy(x => x.SecUid, StringComparer.Ordinal)
|
||||
.ToDictionary(x => x.Key, x => x.First(), StringComparer.Ordinal);
|
||||
if (source.Count == 0) return;
|
||||
|
||||
var secUids = source.Keys.ToList();
|
||||
var monitored = await _db.Queryable<DouyinFollowed>()
|
||||
.Where(x => x.mySelfId == cookie.MyUserId && x.LiveMonitorEnabled && secUids.Contains(x.SecUid))
|
||||
.ToListAsync();
|
||||
foreach (var follow in monitored)
|
||||
{
|
||||
if (!source.TryGetValue(follow.SecUid, out var item)) continue;
|
||||
var probe = DouyinLiveStatusParser.Parse(
|
||||
item.LiveStatus,
|
||||
item.RoomId,
|
||||
item.RoomIdStr,
|
||||
item.RoomData);
|
||||
if (probe.Status == DouyinLiveStatusState.Unknown) continue;
|
||||
var previousStatus = follow.LiveStatus;
|
||||
ApplySuccess(follow, probe, DateTime.Now);
|
||||
await UpdateFollowLiveColumnsAsync(follow);
|
||||
await HandleLiveNotificationAsync(follow, previousStatus, CancellationToken.None);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<DouyinFollowed> ProbeFollowAsync(
|
||||
DouyinFollowed follow,
|
||||
bool ignoreRecentCheck,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!ignoreRecentCheck && follow.LiveCheckedAt.HasValue &&
|
||||
DateTime.Now - follow.LiveCheckedAt.Value < ManualRefreshFloor)
|
||||
return follow;
|
||||
|
||||
var cookies = await _db.Queryable<DouyinCookie>()
|
||||
.Where(x => x.MyUserId == follow.mySelfId)
|
||||
.ToListAsync();
|
||||
var cookie = SelectCookie(cookies, follow.mySelfId);
|
||||
if (cookie == null || cookie.Status != 1 || cookie.StatusCode != 0 || string.IsNullOrWhiteSpace(cookie.Cookies))
|
||||
return await MarkFailedAsync(follow, "授权账号无效或未启用,直播状态暂时无法检查");
|
||||
if (cookie.LiveCheckCooldownUntil.HasValue && cookie.LiveCheckCooldownUntil > DateTime.Now)
|
||||
return await MarkFailedAsync(follow,
|
||||
$"直播状态检查已冷却至 {cookie.LiveCheckCooldownUntil.Value.ToLocalTime():MM-dd HH:mm}");
|
||||
|
||||
await ProbeWithCookieAsync(follow, cookie, cancellationToken);
|
||||
return await RequireFollowAsync(follow.Id);
|
||||
}
|
||||
|
||||
private async Task<ProbeResult> ProbeWithCookieAsync(
|
||||
DouyinFollowed follow,
|
||||
DouyinCookie cookie,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var probe = await _client.ProbeAsync(follow.SecUid, cookie.Cookies, cancellationToken);
|
||||
if (probe.Status == DouyinLiveStatusState.Unknown)
|
||||
throw new DouyinLiveStatusRequestException("抖音响应中缺少可识别的直播状态");
|
||||
var previousStatus = follow.LiveStatus;
|
||||
ApplySuccess(follow, probe, DateTime.Now);
|
||||
await UpdateFollowLiveColumnsAsync(follow);
|
||||
await HandleLiveNotificationAsync(follow, previousStatus, cancellationToken);
|
||||
await ResetAccountHealthAsync(cookie);
|
||||
return new ProbeResult(false);
|
||||
}
|
||||
catch (DouyinLiveStatusRequestException ex)
|
||||
{
|
||||
var message = Limit(ex.Message, 1000);
|
||||
await MarkFailedAsync(follow, message);
|
||||
if (ex.RequiresAccountCooldown)
|
||||
{
|
||||
await ApplyAccountCooldownAsync(cookie, message);
|
||||
Log.Warning("直播状态检查触发账号冷却:Account={Account}, HTTP={StatusCode}",
|
||||
cookie.UserName, ex.StatusCode.HasValue ? (int)ex.StatusCode.Value : null);
|
||||
}
|
||||
else
|
||||
{
|
||||
Log.Warning("直播状态检查失败:Account={Account}, Blogger={Blogger}, Reason={Reason}",
|
||||
cookie.UserName, follow.UperName, message);
|
||||
}
|
||||
return new ProbeResult(ex.RequiresAccountCooldown);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
var message = "直播状态检查失败,请稍后重试";
|
||||
await MarkFailedAsync(follow, message);
|
||||
Log.Warning(ex, "直播状态检查发生异常:Account={Account}, Blogger={Blogger}",
|
||||
cookie.UserName, follow.UperName);
|
||||
return new ProbeResult(false);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ApplyAccountCooldownAsync(DouyinCookie cookie, string error)
|
||||
{
|
||||
cookie.ConsecutiveLiveCheckFailures = Math.Max(0, cookie.ConsecutiveLiveCheckFailures) + 1;
|
||||
var minutes = cookie.ConsecutiveLiveCheckFailures switch
|
||||
{
|
||||
1 => 15,
|
||||
2 => 30,
|
||||
3 => 60,
|
||||
4 => 120,
|
||||
_ => 360
|
||||
};
|
||||
cookie.LiveCheckCooldownUntil = DateTime.Now.AddMinutes(minutes);
|
||||
cookie.LastLiveCheckError = Limit(error, 1000);
|
||||
cookie.LiveCheckHealthUpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(cookie)
|
||||
.UpdateColumns(x => new
|
||||
{
|
||||
x.ConsecutiveLiveCheckFailures,
|
||||
x.LiveCheckCooldownUntil,
|
||||
x.LastLiveCheckError,
|
||||
x.LiveCheckHealthUpdatedAt
|
||||
})
|
||||
.ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
private async Task ResetAccountHealthAsync(DouyinCookie cookie)
|
||||
{
|
||||
if (cookie.ConsecutiveLiveCheckFailures == 0 && !cookie.LiveCheckCooldownUntil.HasValue &&
|
||||
string.IsNullOrWhiteSpace(cookie.LastLiveCheckError)) return;
|
||||
cookie.ConsecutiveLiveCheckFailures = 0;
|
||||
cookie.LiveCheckCooldownUntil = null;
|
||||
cookie.LastLiveCheckError = null;
|
||||
cookie.LiveCheckHealthUpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(cookie)
|
||||
.UpdateColumns(x => new
|
||||
{
|
||||
x.ConsecutiveLiveCheckFailures,
|
||||
x.LiveCheckCooldownUntil,
|
||||
x.LastLiveCheckError,
|
||||
x.LiveCheckHealthUpdatedAt
|
||||
})
|
||||
.ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
private async Task MarkGroupUnavailableAsync(IEnumerable<DouyinFollowed> follows, string message)
|
||||
{
|
||||
foreach (var follow in follows)
|
||||
await MarkFailedAsync(follow, message, updateCheckedAt: false);
|
||||
}
|
||||
|
||||
private async Task<DouyinFollowed> MarkFailedAsync(
|
||||
DouyinFollowed follow,
|
||||
string message,
|
||||
bool updateCheckedAt = true)
|
||||
{
|
||||
if (updateCheckedAt) follow.LiveCheckedAt = DateTime.Now;
|
||||
follow.LiveCheckError = Limit(message, 1000);
|
||||
await UpdateFollowLiveColumnsAsync(follow);
|
||||
return follow;
|
||||
}
|
||||
|
||||
private static void ApplySuccess(DouyinFollowed follow, DouyinLiveStatusProbe probe, DateTime now)
|
||||
{
|
||||
follow.LiveStatus = probe.Status;
|
||||
follow.LiveCheckedAt = now;
|
||||
follow.LiveStatusUpdatedAt = now;
|
||||
follow.LiveCheckError = null;
|
||||
if (probe.Status == DouyinLiveStatusState.Live)
|
||||
{
|
||||
follow.LiveRoomId = Limit(probe.RoomId, 100);
|
||||
follow.LiveWebRid = Limit(probe.WebRid, 100);
|
||||
follow.LiveTitle = Limit(probe.Title, 500);
|
||||
follow.LiveStartedAt = probe.StartedAt;
|
||||
}
|
||||
else
|
||||
{
|
||||
follow.LiveRoomId = null;
|
||||
follow.LiveWebRid = null;
|
||||
follow.LiveTitle = null;
|
||||
follow.LiveStartedAt = null;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task UpdateFollowLiveColumnsAsync(DouyinFollowed follow)
|
||||
{
|
||||
await _db.Updateable(follow)
|
||||
.UpdateColumns(x => new
|
||||
{
|
||||
x.LiveMonitorEnabled,
|
||||
x.LiveStatus,
|
||||
x.LiveRoomId,
|
||||
x.LiveWebRid,
|
||||
x.LiveTitle,
|
||||
x.LiveCheckedAt,
|
||||
x.LiveStatusUpdatedAt,
|
||||
x.LiveStartedAt,
|
||||
x.LiveCheckError,
|
||||
x.LiveEmailNotificationEnabled
|
||||
})
|
||||
.ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
private async Task HandleLiveNotificationAsync(
|
||||
DouyinFollowed follow,
|
||||
DouyinLiveStatusState previousStatus,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var gate = NotificationGates.GetOrAdd(follow.Id, _ => new SemaphoreSlim(1, 1));
|
||||
await gate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
follow = await RequireFollowAsync(follow.Id);
|
||||
if (follow.LiveStatus != DouyinLiveStatusState.Live)
|
||||
{
|
||||
if (follow.LastLiveNotificationKey != null || follow.LastLiveNotificationError != null)
|
||||
{
|
||||
follow.LastLiveNotificationKey = null;
|
||||
follow.LastLiveNotificationAttemptAt = null;
|
||||
follow.LastLiveNotificationError = null;
|
||||
await UpdateFollowNotificationColumnsAsync(follow);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (!follow.LiveEmailNotificationEnabled) return;
|
||||
|
||||
var notificationKey = !string.IsNullOrWhiteSpace(follow.LiveWebRid)
|
||||
? "web:" + follow.LiveWebRid
|
||||
: !string.IsNullOrWhiteSpace(follow.LiveRoomId)
|
||||
? "room:" + follow.LiveRoomId
|
||||
: "live";
|
||||
if (string.Equals(follow.LastLiveNotificationKey, notificationKey, StringComparison.Ordinal)) return;
|
||||
if (previousStatus == DouyinLiveStatusState.Live &&
|
||||
follow.LastLiveNotificationAttemptAt.HasValue &&
|
||||
DateTime.Now - follow.LastLiveNotificationAttemptAt.Value < TimeSpan.FromMinutes(15)) return;
|
||||
|
||||
follow.LastLiveNotificationAttemptAt = DateTime.Now;
|
||||
var result = await _emailNotifications.SendLiveStartedAsync(follow, cancellationToken);
|
||||
if (result.Sent)
|
||||
{
|
||||
follow.LastLiveNotificationKey = notificationKey;
|
||||
follow.LastLiveNotifiedAt = DateTime.Now;
|
||||
follow.LastLiveNotificationError = null;
|
||||
}
|
||||
else if (!string.IsNullOrWhiteSpace(result.Error))
|
||||
{
|
||||
follow.LastLiveNotificationError = Limit(result.Error, 1000);
|
||||
}
|
||||
await UpdateFollowNotificationColumnsAsync(follow);
|
||||
}
|
||||
finally
|
||||
{
|
||||
gate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task UpdateFollowNotificationColumnsAsync(DouyinFollowed follow)
|
||||
{
|
||||
await _db.Updateable(follow)
|
||||
.UpdateColumns(x => new
|
||||
{
|
||||
x.LiveEmailNotificationEnabled,
|
||||
x.LastLiveNotificationKey,
|
||||
x.LastLiveNotificationAttemptAt,
|
||||
x.LastLiveNotifiedAt,
|
||||
x.LastLiveNotificationError
|
||||
})
|
||||
.ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
private async Task<DouyinFollowed> RequireFollowAsync(string id) =>
|
||||
await _db.Queryable<DouyinFollowed>().InSingleAsync(id)
|
||||
?? throw new KeyNotFoundException("关注博主记录不存在");
|
||||
|
||||
private static DouyinCookie SelectCookie(IEnumerable<DouyinCookie> cookies, string myUserId) =>
|
||||
cookies
|
||||
.Where(x => string.Equals(x.MyUserId, myUserId, StringComparison.Ordinal))
|
||||
.OrderByDescending(x => x.Status == 1 && x.StatusCode == 0 && !string.IsNullOrWhiteSpace(x.Cookies))
|
||||
.FirstOrDefault();
|
||||
|
||||
private static FollowLiveStatusDto ToDto(DouyinFollowed follow)
|
||||
{
|
||||
var stale = follow.LiveMonitorEnabled &&
|
||||
(!follow.LiveStatusUpdatedAt.HasValue ||
|
||||
DateTime.Now - follow.LiveStatusUpdatedAt.Value > StaleAfter ||
|
||||
!string.IsNullOrWhiteSpace(follow.LiveCheckError));
|
||||
var roomUrl = follow.LiveStatus == DouyinLiveStatusState.Live && !string.IsNullOrWhiteSpace(follow.LiveWebRid)
|
||||
? "https://live.douyin.com/" + Uri.EscapeDataString(follow.LiveWebRid)
|
||||
: null;
|
||||
return new FollowLiveStatusDto
|
||||
{
|
||||
Id = follow.Id,
|
||||
LiveMonitorEnabled = follow.LiveMonitorEnabled,
|
||||
LiveStatus = follow.LiveStatus,
|
||||
LiveRoomId = follow.LiveRoomId,
|
||||
LiveWebRid = follow.LiveWebRid,
|
||||
LiveTitle = follow.LiveTitle,
|
||||
LiveRoomUrl = roomUrl,
|
||||
LiveCheckedAt = follow.LiveCheckedAt,
|
||||
LiveStatusUpdatedAt = follow.LiveStatusUpdatedAt,
|
||||
LiveStartedAt = follow.LiveStartedAt,
|
||||
LiveCheckError = follow.LiveCheckError,
|
||||
LiveStatusStale = stale,
|
||||
LiveEmailNotificationEnabled = follow.LiveEmailNotificationEnabled,
|
||||
LastLiveNotifiedAt = follow.LastLiveNotifiedAt,
|
||||
LastLiveNotificationError = follow.LastLiveNotificationError
|
||||
};
|
||||
}
|
||||
|
||||
private static string Limit(string value, int maxLength)
|
||||
{
|
||||
value = value?.Trim();
|
||||
return value != null && value.Length > maxLength ? value[..maxLength] : value;
|
||||
}
|
||||
|
||||
private readonly record struct ProbeResult(bool ShouldStopAccount);
|
||||
}
|
||||
}
|
||||
@@ -102,12 +102,14 @@ namespace dy.net.service
|
||||
/// 视频合成(优化后:解决 FFmpeg 进程冲突,支持容错重试)
|
||||
/// </summary>
|
||||
public async Task<bool> MergeToVideo(string cookie, string rootPath, MediaMergeRequest request,
|
||||
string outputVideoPath, string fileNamefolder, bool mergeImg2Viedo, bool downImage = false, bool downMp3 = false)
|
||||
string outputVideoPath, string fileNamefolder, bool mergeImg2Viedo, bool downImage = false,
|
||||
bool downMp3 = false, Action<string> reportWarning = null, Action<string> reportError = null)
|
||||
{
|
||||
// 输入参数校验(避免无效执行)
|
||||
if (request == null || request.ImageUrls == null || request.ImageUrls.Count == 0)
|
||||
{
|
||||
Log.Error("图片URL列表为空,无法合成视频");
|
||||
reportError?.Invoke("图片 URL 列表为空");
|
||||
return false;
|
||||
}
|
||||
//if (mergeImg2Viedo && (request.AudioUrls == null || request.AudioUrls.Count == 0))
|
||||
@@ -128,7 +130,9 @@ namespace dy.net.service
|
||||
request.ImageUrls, Path.Combine(tempDir, "raw-images"), "image_", "webp", cookie);
|
||||
if (!string.IsNullOrEmpty(imageError) || rawImages == null || rawImages.Count == 0)
|
||||
{
|
||||
Log.Error($"图片下载失败:{imageError ?? "未下载到任何图片"}");
|
||||
var reason = imageError ?? "未下载到任何图片";
|
||||
Log.Error($"图片下载失败:{reason}");
|
||||
reportError?.Invoke($"图片下载失败:{reason}");
|
||||
return false;
|
||||
}
|
||||
// 保存下载的图片(如果需要)
|
||||
@@ -138,19 +142,25 @@ namespace dy.net.service
|
||||
}
|
||||
|
||||
// 2. 下载音频
|
||||
List<DouyinMergeVideoDto> rawAudios =new List<DouyinMergeVideoDto>();
|
||||
if (mergeImg2Viedo && request.AudioUrls != null && request.AudioUrls.Count > 0)
|
||||
List<DouyinMergeVideoDto> rawAudios = new List<DouyinMergeVideoDto>();
|
||||
var audioWarningReported = false;
|
||||
if ((mergeImg2Viedo || downMp3) && request.AudioUrls != null && request.AudioUrls.Count > 0)
|
||||
{
|
||||
var (audios, audioError) = await DownloadMediaAsync(
|
||||
request.AudioUrls.Select(x => new DouyinMergeVideoDto { Path = x })?.ToList(), Path.Combine(tempDir, "raw-audios"), "audio_", "mp3", cookie);
|
||||
if (!string.IsNullOrEmpty(audioError) || audios == null || audios.Count == 0)
|
||||
{
|
||||
Log.Error($"音频下载失败:{audioError ?? "未下载到任何音频"}");
|
||||
return false;
|
||||
var warning = downMp3
|
||||
? "作品音频不可用,已降级处理,未生成单独音频文件"
|
||||
: "作品音频不可用,图文视频已降级为无声视频";
|
||||
Log.Warning("音频下载失败:{Reason};{Fallback}",
|
||||
audioError ?? "未下载到任何音频", warning);
|
||||
reportWarning?.Invoke(warning);
|
||||
audioWarningReported = true;
|
||||
}
|
||||
rawAudios = audios;
|
||||
else rawAudios = audios;
|
||||
// 保存下载的音频(如果需要)
|
||||
if (downMp3)
|
||||
if (downMp3 && rawAudios.Count > 0)
|
||||
{
|
||||
var ext = Path.GetExtension(rawAudios.FirstOrDefault()?.Path);
|
||||
await SaveDownloadedFilesAsync(rawAudios, fileNamefolder, ext, Path.GetFileNameWithoutExtension(outputVideoPath));
|
||||
@@ -161,7 +171,11 @@ namespace dy.net.service
|
||||
if (!mergeImg2Viedo)
|
||||
{
|
||||
Log.Debug($"根据系统配置设置不下载图文视频-[{outputVideoPath}],{(downImage ? "下载图片" : "不下载图片")},{(downMp3 ? "下载音频" : "不下载音频")}");
|
||||
return true;
|
||||
// When only an audio attachment was requested, an unavailable audio source
|
||||
// leaves no persistent artifact and must remain a real item-level failure.
|
||||
var hasArtifact = downImage || !downMp3 || rawAudios.Count > 0;
|
||||
if (!hasArtifact) reportError?.Invoke("音频下载失败,且当前任务未配置保存图片");
|
||||
return hasArtifact;
|
||||
}
|
||||
|
||||
// 3. 调整图片显示时长
|
||||
@@ -194,14 +208,20 @@ namespace dy.net.service
|
||||
|
||||
if (rawAudios.Count == 0)
|
||||
{
|
||||
var musicPath = GetRandomMergeMusic();
|
||||
rawAudios = new List<DouyinMergeVideoDto> { new DouyinMergeVideoDto { Path= musicPath } };
|
||||
Log.Debug("版权原因无法下载音频,使用默认无声音频文件");
|
||||
if (!audioWarningReported)
|
||||
{
|
||||
var warning = downMp3
|
||||
? "作品没有可用音频,已生成无声视频,未生成单独音频文件"
|
||||
: "作品没有可用音频,已生成无声视频";
|
||||
reportWarning?.Invoke(warning);
|
||||
audioWarningReported = true;
|
||||
}
|
||||
Log.Warning("图文作品没有可用音频,将直接生成无声视频");
|
||||
}
|
||||
|
||||
string resultPath = await ffmpegHelper.CreateVideoFromImagesAndAudioAsync(
|
||||
rawImages, // 所有下载的图片(确保无遗漏)
|
||||
rawAudios.FirstOrDefault()?.Path, // 取第一个音频(可根据需求调整)
|
||||
rawAudios.FirstOrDefault()?.Path, // 音频缺失时 FFmpegHelper 会生成无声视频
|
||||
outputVideoPath,
|
||||
ffmpegHelper.VideoWidth,
|
||||
ffmpegHelper.VideoHeight,
|
||||
@@ -218,11 +238,13 @@ namespace dy.net.service
|
||||
}
|
||||
});
|
||||
|
||||
if (!mergeSuccess) reportError?.Invoke("FFmpeg 未生成有效的视频文件");
|
||||
return mergeSuccess;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Error(ex, "视频合成过程中发生未处理异常");
|
||||
reportError?.Invoke($"{ex.GetType().Name}:{ex.GetBaseException().Message}");
|
||||
return false;
|
||||
}
|
||||
finally
|
||||
@@ -330,6 +352,7 @@ namespace dy.net.service
|
||||
List<DouyinMergeVideoDto> urls, string saveDir, string prefix, string ext, string cookie)
|
||||
{
|
||||
var successPaths = new List<DouyinMergeVideoDto>();
|
||||
var failures = new List<string>();
|
||||
for (var i = 0; i < urls.Count; i++)
|
||||
{
|
||||
var url = urls[i];
|
||||
@@ -339,12 +362,12 @@ namespace dy.net.service
|
||||
|
||||
try
|
||||
{
|
||||
var (Success, ActualSavePath) = await douyinHttpClientService.DownloadAsync(url.Path, savePath, cookie);
|
||||
if (Success)
|
||||
var result = await douyinHttpClientService.DownloadAsync(url.Path, savePath, cookie);
|
||||
if (result.Success)
|
||||
{
|
||||
successPaths.Add(new DouyinMergeVideoDto
|
||||
{
|
||||
Path = ActualSavePath,
|
||||
Path = result.ActualSavePath,
|
||||
Height = url.Height,
|
||||
Width =
|
||||
url.Width
|
||||
@@ -352,18 +375,22 @@ namespace dy.net.service
|
||||
}
|
||||
else
|
||||
{
|
||||
Serilog.Log.Error($"下载失败:{url} → {savePath}");
|
||||
failures.Add($"{result.SourceHost ?? "未知来源"}/HTTP {result.HttpStatusCode?.ToString() ?? "-"}/{result.FailureKind}:{result.Message}");
|
||||
Serilog.Log.Warning("合成素材下载失败:Host={Host}, Status={Status}, Kind={Kind}, Target={Target}",
|
||||
result.SourceHost, result.HttpStatusCode, result.FailureKind, savePath);
|
||||
}
|
||||
//Console.WriteLine($"下载成功:{url} → {savePath}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
var error = $"下载失败:{url},错误:{ex.Message}";
|
||||
Serilog.Log.Error(error);
|
||||
var error = $"合成素材下载失败:{ex.GetType().Name}:{ex.GetBaseException().Message}";
|
||||
Serilog.Log.Error("合成素材下载异常:Type={ExceptionType}, Target={Target}", ex.GetType().Name, savePath);
|
||||
return (new List<DouyinMergeVideoDto>(), error);
|
||||
}
|
||||
}
|
||||
return (successPaths, null);
|
||||
return successPaths.Count == 0 && failures.Count > 0
|
||||
? (successPaths, string.Join(";", failures.Take(3)))
|
||||
: (successPaths, null);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
using dy.net.model.dto;
|
||||
using dy.net.model.entity;
|
||||
using dy.net.model.response;
|
||||
using SqlSugar;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
public class DouyinMigrationSourceResolver
|
||||
{
|
||||
private readonly DouyinHttpClientService _http;
|
||||
private readonly ISqlSugarClient _db;
|
||||
|
||||
public DouyinMigrationSourceResolver(DouyinHttpClientService http, ISqlSugarClient db)
|
||||
{
|
||||
_http = http;
|
||||
_db = db;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<string>> ResolveAsync(DouyinVideo video, DouyinCookie cookie, CancellationToken cancellationToken)
|
||||
{
|
||||
if (video == null || cookie == null || string.IsNullOrWhiteSpace(cookie.Cookies)) return Array.Empty<string>();
|
||||
var category = string.IsNullOrWhiteSpace(video.CateId)
|
||||
? null
|
||||
: await _db.Queryable<DouyinCollectCate>().InSingleAsync(video.CateId);
|
||||
var followed = video.ViedoType == VideoTypeEnum.dy_follows
|
||||
? await _db.Queryable<DouyinFollowed>().Where(x => x.UperId == video.AuthorId).FirstAsync()
|
||||
: null;
|
||||
|
||||
var cursor = "0";
|
||||
for (var page = 0; page < 500 && !cancellationToken.IsCancellationRequested; page++)
|
||||
{
|
||||
var response = await FetchAsync(video, cookie, category, followed, cursor);
|
||||
if (response?.StatusCode != 0 || response.AwemeList == null) return Array.Empty<string>();
|
||||
var match = response.AwemeList.FirstOrDefault(x => x.AwemeId == video.AwemeId);
|
||||
if (match != null) return ExtractUrls(match);
|
||||
if (response.HasMore != 1) break;
|
||||
var next = response.Cursor ?? response.MaxCursor;
|
||||
if (string.IsNullOrWhiteSpace(next) || next == cursor) break;
|
||||
cursor = next;
|
||||
}
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
private async Task<DouyinVideoInfoResponse> FetchAsync(
|
||||
DouyinVideo video,
|
||||
DouyinCookie cookie,
|
||||
DouyinCollectCate category,
|
||||
DouyinFollowed followed,
|
||||
string cursor) => video.ViedoType switch
|
||||
{
|
||||
VideoTypeEnum.dy_favorite when !string.IsNullOrWhiteSpace(cookie.SecUserId) =>
|
||||
await _http.SyncFavoriteVideos("18", cursor, cookie.SecUserId, cookie.Cookies),
|
||||
VideoTypeEnum.dy_follows when !string.IsNullOrWhiteSpace(followed?.SecUid) =>
|
||||
await _http.SyncUpderPostVideos("18", cursor, followed.SecUid, cookie.Cookies),
|
||||
VideoTypeEnum.dy_custom_collect when !string.IsNullOrWhiteSpace(category?.XId) =>
|
||||
await _http.SyncCollectVideosByCollectId(cursor, "18", cookie.Cookies, category.XId),
|
||||
VideoTypeEnum.dy_mix when !string.IsNullOrWhiteSpace(category?.XId ?? video.CateXId) =>
|
||||
await _http.SyncMixViedosByMixId(cursor, "18", cookie.Cookies, category?.XId ?? video.CateXId),
|
||||
VideoTypeEnum.dy_series when !string.IsNullOrWhiteSpace(category?.XId ?? video.CateXId) =>
|
||||
await _http.SyncSeriesViedosByMSeriesId(cursor, "18", cookie.Cookies, category?.XId ?? video.CateXId),
|
||||
_ => await _http.SyncCollectVideos(cursor, "18", cookie.Cookies)
|
||||
};
|
||||
|
||||
private static IReadOnlyList<string> ExtractUrls(Aweme aweme)
|
||||
{
|
||||
return (aweme.Video?.BitRate ?? new List<VideoBitRate>())
|
||||
.Where(x => x?.PlayAddr?.UrlList != null)
|
||||
.OrderByDescending(x => x.BitRateValue)
|
||||
.SelectMany(x => x.PlayAddr.UrlList)
|
||||
.Where(x => !string.IsNullOrWhiteSpace(x))
|
||||
.Distinct()
|
||||
.ToArray();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,8 @@ namespace dy.net.service
|
||||
{
|
||||
private readonly ISchedulerFactory _schedulerFactory;
|
||||
private readonly DouyinCookieService douyinCookieService;
|
||||
private readonly DouyinCommonService _commonService;
|
||||
private readonly VideoTaskService _videoTasks;
|
||||
private const string DefaultJobGroup = "dysync.net";
|
||||
private const int DefaultIntervalMinutes = 30;
|
||||
private const int DefaultCronStartDelaySeconds = 30;
|
||||
@@ -85,15 +87,91 @@ namespace dy.net.service
|
||||
"dy.job.key.sync_follow_user_once",
|
||||
"dy.trigger.key.sync_follow_user_once",
|
||||
"抖音关注同步任务(单次执行)")
|
||||
},
|
||||
{
|
||||
VideoTypeEnum.dy_live_monitor,
|
||||
new JobConfig(
|
||||
typeof(DouyinLiveStatusJob),
|
||||
"dy.job.key.live_monitor",
|
||||
"dy.trigger.key.live_monitor",
|
||||
"抖音博主直播状态监测任务")
|
||||
}
|
||||
};
|
||||
|
||||
public DouyinQuartzJobService(ISchedulerFactory schedulerFactory,DouyinCookieService douyinCookieService)
|
||||
public DouyinQuartzJobService(
|
||||
ISchedulerFactory schedulerFactory,
|
||||
DouyinCookieService douyinCookieService,
|
||||
DouyinCommonService commonService,
|
||||
VideoTaskService videoTasks)
|
||||
{
|
||||
_schedulerFactory = schedulerFactory ?? throw new ArgumentNullException(nameof(schedulerFactory));
|
||||
this.douyinCookieService = douyinCookieService;
|
||||
_commonService = commonService;
|
||||
_videoTasks = videoTasks;
|
||||
}
|
||||
|
||||
public async Task<List<VideoDownloadTask>> TriggerVideoJobsNowAsync(VideoTypeEnum? requestedType = null)
|
||||
{
|
||||
var cookies = await douyinCookieService.GetOpendCookiesAsync();
|
||||
if (cookies == null || cookies.Count == 0) throw new InvalidOperationException("没有已启用的抖音授权账号。");
|
||||
var storageType = _commonService.GetConfig()?.StorageType ?? StorageType.Local;
|
||||
var enabled = GetTaskEnableConditions(cookies, storageType);
|
||||
var types = new List<VideoTypeEnum>();
|
||||
if (enabled.IsFavoriteEnabled) types.Add(VideoTypeEnum.dy_favorite);
|
||||
if (enabled.IsCollectEnabled) types.Add(VideoTypeEnum.dy_collects);
|
||||
if (enabled.IsFollowedEnabled) types.Add(VideoTypeEnum.dy_follows);
|
||||
if (enabled.IsCustomCollectEnabled) types.Add(VideoTypeEnum.dy_custom_collect);
|
||||
if (enabled.IsMixEnabled) types.Add(VideoTypeEnum.dy_mix);
|
||||
if (enabled.IsSeriesEnabled) types.Add(VideoTypeEnum.dy_series);
|
||||
if (types.Count == 0) throw new InvalidOperationException("没有已启用且路径完整的视频同步类型。");
|
||||
if (requestedType.HasValue)
|
||||
{
|
||||
if (!IsVideoSyncType(requestedType.Value))
|
||||
throw new InvalidOperationException("指定的任务类型不是可手动执行的视频同步类型。");
|
||||
if (!types.Contains(requestedType.Value))
|
||||
{
|
||||
var message = requestedType.Value == VideoTypeEnum.dy_follows
|
||||
? "关注视频同步未启用:请检查账号的关注下载开关、当前存储的关注路径,以及是否有博主开启同步。"
|
||||
: $"{requestedType.Value.GetDesc()}同步未启用或当前存储路径未配置。";
|
||||
throw new InvalidOperationException(message);
|
||||
}
|
||||
types = new List<VideoTypeEnum> { requestedType.Value };
|
||||
}
|
||||
|
||||
var scheduler = await _schedulerFactory.GetScheduler();
|
||||
var created = new List<VideoDownloadTask>();
|
||||
foreach (var type in types)
|
||||
{
|
||||
var config = JobConfigs[type];
|
||||
var jobKey = new JobKey(config.JobKey, DefaultJobGroup);
|
||||
if (!await scheduler.CheckExists(jobKey))
|
||||
await StartJobAsync(type, (_commonService.GetConfig()?.Cron ?? DefaultIntervalMinutes).ToString());
|
||||
var task = await _videoTasks.CreateTaskAsync(VideoTaskType.Sync, VideoTaskTrigger.Manual,
|
||||
$"{type.GetDesc()}手动同步", type, storageType);
|
||||
try
|
||||
{
|
||||
await scheduler.TriggerJob(jobKey, new JobDataMap
|
||||
{
|
||||
["video-task-id"] = task.Id,
|
||||
["video-task-trigger"] = "manual"
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
var message = $"{type.GetDesc()}手动同步未能进入调度,请稍后重试。";
|
||||
await _videoTasks.CompleteTaskAsync(task.Id, message);
|
||||
Log.Error(ex, "手动同步任务调度失败:{VideoType}, TaskId={TaskId}", type, task.Id);
|
||||
throw new InvalidOperationException(message, ex);
|
||||
}
|
||||
created.Add(task);
|
||||
}
|
||||
return created;
|
||||
}
|
||||
|
||||
internal static bool IsVideoSyncType(VideoTypeEnum type) => type is
|
||||
VideoTypeEnum.dy_favorite or VideoTypeEnum.dy_collects or VideoTypeEnum.dy_follows or
|
||||
VideoTypeEnum.dy_custom_collect or VideoTypeEnum.dy_mix or VideoTypeEnum.dy_series;
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// 初始化或重启所有抖音定时任务
|
||||
@@ -126,7 +204,8 @@ namespace dy.net.service
|
||||
await RemoveAllExistingJobs(scheduler);
|
||||
|
||||
// 4. 检查各类型任务的启用条件
|
||||
var taskEnableConditions = GetTaskEnableConditions(validCookies);
|
||||
var storageType = _commonService.GetConfig()?.StorageType ?? StorageType.Local;
|
||||
var taskEnableConditions = GetTaskEnableConditions(validCookies, storageType);
|
||||
|
||||
// 5. 启动符合条件的定时任务
|
||||
int successfullyStartedJobs = 0;
|
||||
@@ -144,6 +223,14 @@ namespace dy.net.service
|
||||
continue;
|
||||
}
|
||||
|
||||
// 直播监测完全独立于视频同步,固定每5分钟检查已单独开启的博主。
|
||||
if (jobKey == VideoTypeEnum.dy_live_monitor)
|
||||
{
|
||||
bool startSuccess = await StartSingleJobAsync(jobKey, "5");
|
||||
if (startSuccess) successfullyStartedJobs++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// 根据不同任务类型和启用条件启动任务
|
||||
bool isTaskEnabled = jobKey switch
|
||||
{
|
||||
@@ -196,16 +283,40 @@ namespace dy.net.service
|
||||
/// </summary>
|
||||
/// <param name="cookies">有效的抖音Cookie列表</param>
|
||||
/// <returns>任务启用条件集合</returns>
|
||||
private static TaskEnableConditions GetTaskEnableConditions(IEnumerable<DouyinCookie> cookies)
|
||||
private static TaskEnableConditions GetTaskEnableConditions(IEnumerable<DouyinCookie> cookies, StorageType storageType)
|
||||
{
|
||||
bool HasPath(DouyinCookie cookie, VideoTypeEnum type)
|
||||
{
|
||||
if (storageType.IsRemote())
|
||||
{
|
||||
return type switch
|
||||
{
|
||||
VideoTypeEnum.dy_favorite => !string.IsNullOrWhiteSpace(cookie.WebDavFavoritePath),
|
||||
VideoTypeEnum.dy_follows => !string.IsNullOrWhiteSpace(cookie.WebDavFollowPath),
|
||||
VideoTypeEnum.dy_mix => !string.IsNullOrWhiteSpace(cookie.WebDavMixPath),
|
||||
VideoTypeEnum.dy_series => !string.IsNullOrWhiteSpace(cookie.WebDavSeriesPath),
|
||||
_ => !string.IsNullOrWhiteSpace(cookie.WebDavCollectPath)
|
||||
};
|
||||
}
|
||||
|
||||
return type switch
|
||||
{
|
||||
VideoTypeEnum.dy_favorite => !string.IsNullOrWhiteSpace(cookie.FavSavePath),
|
||||
VideoTypeEnum.dy_follows => !string.IsNullOrWhiteSpace(cookie.UpSavePath),
|
||||
VideoTypeEnum.dy_mix => !string.IsNullOrWhiteSpace(cookie.MixPath) || !string.IsNullOrWhiteSpace(cookie.SavePath),
|
||||
VideoTypeEnum.dy_series => !string.IsNullOrWhiteSpace(cookie.SeriesPath) || !string.IsNullOrWhiteSpace(cookie.SavePath),
|
||||
_ => !string.IsNullOrWhiteSpace(cookie.SavePath)
|
||||
};
|
||||
}
|
||||
|
||||
return new TaskEnableConditions
|
||||
{
|
||||
IsCollectEnabled = cookies.Any(x => x.DownCollect && !x.UseCollectFolder && !string.IsNullOrWhiteSpace(x.SavePath)),
|
||||
IsFavoriteEnabled = cookies.Any(x => x.DownFavorite && !string.IsNullOrWhiteSpace(x.FavSavePath)),
|
||||
IsFollowedEnabled = cookies.Any(x => x.DownFollowd && !string.IsNullOrWhiteSpace(x.UpSavePath)),
|
||||
IsMixEnabled = cookies.Any(x => x.DownMix && !string.IsNullOrWhiteSpace(x.MixPath)),
|
||||
IsSeriesEnabled = cookies.Any(x => x.DownSeries && !string.IsNullOrWhiteSpace(x.SeriesPath)),
|
||||
IsCustomCollectEnabled = cookies.Any(x => x.UseCollectFolder && !string.IsNullOrWhiteSpace(x.SavePath))
|
||||
IsCollectEnabled = cookies.Any(x => x.DownCollect && !x.UseCollectFolder && HasPath(x, VideoTypeEnum.dy_collects)),
|
||||
IsFavoriteEnabled = cookies.Any(x => x.DownFavorite && HasPath(x, VideoTypeEnum.dy_favorite)),
|
||||
IsFollowedEnabled = cookies.Any(x => x.DownFollowd && HasPath(x, VideoTypeEnum.dy_follows)),
|
||||
IsMixEnabled = cookies.Any(x => x.DownMix && HasPath(x, VideoTypeEnum.dy_mix)),
|
||||
IsSeriesEnabled = cookies.Any(x => x.DownSeries && HasPath(x, VideoTypeEnum.dy_series)),
|
||||
IsCustomCollectEnabled = cookies.Any(x => x.UseCollectFolder && HasPath(x, VideoTypeEnum.dy_custom_collect))
|
||||
};
|
||||
}
|
||||
|
||||
@@ -423,4 +534,4 @@ namespace dy.net.service
|
||||
public bool IsCustomCollectEnabled { get; set; }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Text.RegularExpressions;
|
||||
using dy.net.model.dto;
|
||||
using dy.net.model.entity;
|
||||
using SqlSugar;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
public sealed class DouyinUserLookupService
|
||||
{
|
||||
private static readonly TimeSpan CacheLifetime = TimeSpan.FromMinutes(5);
|
||||
private static readonly SemaphoreSlim SearchGate = new(1, 1);
|
||||
private static readonly ConcurrentDictionary<string, CacheEntry> Cache = new(StringComparer.Ordinal);
|
||||
private static readonly Regex LabeledDouyinNo = new(
|
||||
@"^(?:抖音号|douyin)\s*[::]\s*([^\s,,;;]+)",
|
||||
RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly Regex LabeledDouyinNoPrefix = new(
|
||||
@"^(?:抖音号|douyin)\s*[::]",
|
||||
RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
|
||||
private readonly ISqlSugarClient _db;
|
||||
private readonly DouyinUserSearchClient _searchClient;
|
||||
|
||||
public DouyinUserLookupService(ISqlSugarClient db, DouyinUserSearchClient searchClient)
|
||||
{
|
||||
_db = db;
|
||||
_searchClient = searchClient;
|
||||
}
|
||||
|
||||
public async Task<DouyinFollowLookupResult> ResolveAsync(
|
||||
DouyinFollowLookupRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (request == null) throw new InvalidOperationException("查询参数不能为空");
|
||||
var query = NormalizeDouyinNo(request.DouyinNo);
|
||||
if (string.IsNullOrWhiteSpace(request.CookieId)) throw new InvalidOperationException("请选择抖音授权账号");
|
||||
|
||||
var cookie = await _db.Queryable<DouyinCookie>().InSingleAsync(request.CookieId);
|
||||
if (cookie == null) throw new InvalidOperationException("选择的抖音授权账号不存在");
|
||||
if (cookie.Status != 1 || cookie.StatusCode != 0 || string.IsNullOrWhiteSpace(cookie.Cookies))
|
||||
throw new InvalidOperationException("当前账号 Cookie 无效或未启用,请先更新抖音授权");
|
||||
if (string.IsNullOrWhiteSpace(cookie.MyUserId))
|
||||
throw new InvalidOperationException("当前账号缺少用户 UID,请先完善抖音授权配置");
|
||||
|
||||
var cacheKey = cookie.Id + ":" + query.ToUpperInvariant();
|
||||
var candidates = TryGetCached(cacheKey);
|
||||
if (candidates == null)
|
||||
{
|
||||
await SearchGate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
candidates = TryGetCached(cacheKey);
|
||||
if (candidates == null)
|
||||
{
|
||||
candidates = await _searchClient.SearchAsync(query, cookie.Cookies, cancellationToken);
|
||||
Cache[cacheKey] = new CacheEntry(DateTimeOffset.UtcNow.Add(CacheLifetime), Clone(candidates));
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
SearchGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
if (candidates.Count == 0) throw new InvalidOperationException("未找到对应博主,请检查抖音号或使用手动添加");
|
||||
var secUids = candidates.Select(x => x.SecUid).Distinct().ToList();
|
||||
var existing = await _db.Queryable<DouyinFollowed>()
|
||||
.Where(x => x.mySelfId == cookie.MyUserId && secUids.Contains(x.SecUid))
|
||||
.Select(x => x.SecUid)
|
||||
.ToListAsync();
|
||||
var existingSet = existing.ToHashSet(StringComparer.Ordinal);
|
||||
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
candidate.ExactMatch = string.Equals(candidate.UniqueId, query, StringComparison.OrdinalIgnoreCase)
|
||||
|| string.Equals(candidate.ShortId, query, StringComparison.OrdinalIgnoreCase);
|
||||
candidate.AlreadyExists = existingSet.Contains(candidate.SecUid);
|
||||
}
|
||||
|
||||
return new DouyinFollowLookupResult
|
||||
{
|
||||
Query = query,
|
||||
Candidates = candidates
|
||||
.OrderByDescending(x => x.ExactMatch)
|
||||
.ThenByDescending(x => x.FollowerCount)
|
||||
.Take(10)
|
||||
.ToList()
|
||||
};
|
||||
}
|
||||
|
||||
public static string NormalizeDouyinNo(string input)
|
||||
{
|
||||
var value = (input ?? string.Empty).Trim();
|
||||
var match = LabeledDouyinNo.Match(value);
|
||||
if (match.Success)
|
||||
value = match.Groups[1].Value;
|
||||
else if (LabeledDouyinNoPrefix.IsMatch(value))
|
||||
value = string.Empty;
|
||||
value = value.Trim().TrimStart('@').Trim();
|
||||
if (string.IsNullOrWhiteSpace(value)) throw new InvalidOperationException("请输入抖音号");
|
||||
if (value.Length > 64) throw new InvalidOperationException("抖音号长度不能超过 64 个字符");
|
||||
if (value.Any(char.IsWhiteSpace) || value.Any(char.IsControl))
|
||||
throw new InvalidOperationException("抖音号格式不正确");
|
||||
return value;
|
||||
}
|
||||
|
||||
private static List<DouyinFollowCandidate> TryGetCached(string key)
|
||||
{
|
||||
if (!Cache.TryGetValue(key, out var entry)) return null;
|
||||
if (entry.ExpiresAt <= DateTimeOffset.UtcNow)
|
||||
{
|
||||
Cache.TryRemove(key, out _);
|
||||
return null;
|
||||
}
|
||||
return Clone(entry.Candidates);
|
||||
}
|
||||
|
||||
private static List<DouyinFollowCandidate> Clone(IEnumerable<DouyinFollowCandidate> source) =>
|
||||
source.Select(x => new DouyinFollowCandidate
|
||||
{
|
||||
SecUid = x.SecUid,
|
||||
UperId = x.UperId,
|
||||
DouyinNo = x.DouyinNo,
|
||||
UniqueId = x.UniqueId,
|
||||
ShortId = x.ShortId,
|
||||
UperName = x.UperName,
|
||||
UperAvatar = x.UperAvatar,
|
||||
Signature = x.Signature,
|
||||
Enterprise = x.Enterprise,
|
||||
FollowerCount = x.FollowerCount,
|
||||
ExactMatch = x.ExactMatch,
|
||||
AlreadyExists = x.AlreadyExists
|
||||
}).ToList();
|
||||
|
||||
private sealed record CacheEntry(DateTimeOffset ExpiresAt, List<DouyinFollowCandidate> Candidates);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
using dy.net.model.dto;
|
||||
using dy.net.utils;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
public sealed class DouyinUserSearchClient
|
||||
{
|
||||
private const string SearchPath = "/aweme/v1/web/discover/search/";
|
||||
private readonly IHttpClientFactory _clientFactory;
|
||||
private readonly DouyinABogusSigner _signer = new();
|
||||
|
||||
public DouyinUserSearchClient(IHttpClientFactory clientFactory) => _clientFactory = clientFactory;
|
||||
|
||||
public async Task<List<DouyinFollowCandidate>> SearchAsync(
|
||||
string douyinNo,
|
||||
string cookie,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var parameters = BuildParameters(douyinNo, cookie);
|
||||
var unsignedQuery = BuildQueryString(parameters);
|
||||
var signature = _signer.Sign(unsignedQuery);
|
||||
var requestUri = SearchPath + "?" + unsignedQuery + "&a_bogus=" + Encode(signature);
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
|
||||
request.Headers.Referrer = new Uri("https://www.douyin.com/root/search/" + Encode(douyinNo) + "?type=user");
|
||||
request.Headers.TryAddWithoutValidation("Accept", "*/*");
|
||||
request.Headers.TryAddWithoutValidation("Cookie", cookie);
|
||||
|
||||
var client = _clientFactory.CreateClient(DouyinRequestParamManager.DY_HTTP_CLIENT);
|
||||
try
|
||||
{
|
||||
using var response = await client.SendAsync(
|
||||
request,
|
||||
HttpCompletionOption.ResponseHeadersRead,
|
||||
cancellationToken);
|
||||
if (!response.IsSuccessStatusCode)
|
||||
throw new InvalidOperationException($"抖音用户搜索请求失败(HTTP {(int)response.StatusCode})");
|
||||
|
||||
var content = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
SearchResponse payload;
|
||||
try
|
||||
{
|
||||
payload = JsonConvert.DeserializeObject<SearchResponse>(content);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
throw new InvalidOperationException("抖音返回了风控验证页面,请稍后重试或使用手动添加");
|
||||
}
|
||||
if (payload == null)
|
||||
throw new InvalidOperationException("抖音用户搜索没有返回有效数据");
|
||||
if (payload.StatusCode != 0)
|
||||
throw new InvalidOperationException("抖音用户搜索被拒绝,请确认 Cookie 有效后重试");
|
||||
|
||||
return (payload.Users ?? new List<SearchUserEntry>())
|
||||
.Select(x => x.User)
|
||||
.Where(x => x != null &&
|
||||
!string.IsNullOrWhiteSpace(x.SecUid) &&
|
||||
!string.IsNullOrWhiteSpace(x.Uid) &&
|
||||
!string.IsNullOrWhiteSpace(x.Nickname))
|
||||
.Select(x => new DouyinFollowCandidate
|
||||
{
|
||||
SecUid = x.SecUid,
|
||||
UperId = x.Uid,
|
||||
UniqueId = x.UniqueId,
|
||||
ShortId = x.ShortId,
|
||||
DouyinNo = !string.IsNullOrWhiteSpace(x.UniqueId) ? x.UniqueId : x.ShortId,
|
||||
UperName = x.Nickname,
|
||||
UperAvatar = x.Avatar?.Urls?.FirstOrDefault() ?? string.Empty,
|
||||
Signature = x.Signature,
|
||||
Enterprise = !string.IsNullOrWhiteSpace(x.Enterprise) ? x.Enterprise : x.CustomVerify,
|
||||
FollowerCount = x.FollowerCount
|
||||
})
|
||||
.GroupBy(x => x.SecUid, StringComparer.Ordinal)
|
||||
.Select(x => x.First())
|
||||
.Take(10)
|
||||
.ToList();
|
||||
}
|
||||
catch (HttpRequestException)
|
||||
{
|
||||
// 不向上抛出包含查询 URI 的原始异常,避免 msToken 出现在全局日志中。
|
||||
throw new InvalidOperationException("抖音用户搜索网络请求失败,请稍后重试");
|
||||
}
|
||||
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw new InvalidOperationException("抖音用户搜索超时,请稍后重试");
|
||||
}
|
||||
}
|
||||
|
||||
private static List<KeyValuePair<string, string>> BuildParameters(string douyinNo, string cookie)
|
||||
{
|
||||
var cookies = ParseCookie(cookie);
|
||||
return new List<KeyValuePair<string, string>>
|
||||
{
|
||||
new("device_platform", "webapp"),
|
||||
new("aid", "6383"),
|
||||
new("channel", "channel_pc_web"),
|
||||
new("update_version_code", "170400"),
|
||||
new("pc_client_type", "1"),
|
||||
new("pc_libra_divert", "Windows"),
|
||||
new("support_h265", "1"),
|
||||
new("support_dash", "1"),
|
||||
new("version_code", "170400"),
|
||||
new("version_name", "17.4.0"),
|
||||
new("cookie_enabled", "true"),
|
||||
new("screen_width", "1536"),
|
||||
new("screen_height", "864"),
|
||||
new("browser_language", "zh-CN"),
|
||||
new("browser_platform", "Win32"),
|
||||
new("browser_name", "Chrome"),
|
||||
new("browser_version", "119.0.0.0"),
|
||||
new("browser_online", "true"),
|
||||
new("engine_name", "Blink"),
|
||||
new("engine_version", "119.0.0.0"),
|
||||
new("os_name", "Windows"),
|
||||
new("os_version", "10"),
|
||||
new("cpu_core_num", "16"),
|
||||
new("device_memory", "8"),
|
||||
new("platform", "PC"),
|
||||
new("downlink", "10"),
|
||||
new("effective_type", "4g"),
|
||||
new("round_trip_time", "200"),
|
||||
new("uifid", CookieValue(cookies, "UIFID", "UIFID_TEMP")),
|
||||
new("msToken", CookieValue(cookies, "msToken")),
|
||||
new("pc_search_top_1_params", "{\"enable_ai_search_top_1\":1}"),
|
||||
new("search_channel", "aweme_user_web"),
|
||||
new("keyword", douyinNo),
|
||||
new("search_source", "switch_tab"),
|
||||
new("query_correct_type", "1"),
|
||||
new("is_filter_search", "0"),
|
||||
new("from_group_id", string.Empty),
|
||||
new("disable_rs", "0"),
|
||||
new("offset", "0"),
|
||||
new("count", "10"),
|
||||
new("need_filter_settings", "0"),
|
||||
new("list_type", "single")
|
||||
};
|
||||
}
|
||||
|
||||
private static Dictionary<string, string> ParseCookie(string cookie)
|
||||
{
|
||||
var values = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var part in (cookie ?? string.Empty).Split(';', StringSplitOptions.RemoveEmptyEntries))
|
||||
{
|
||||
var separator = part.IndexOf('=');
|
||||
if (separator <= 0) continue;
|
||||
values[part[..separator].Trim()] = part[(separator + 1)..].Trim();
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
private static string CookieValue(Dictionary<string, string> cookies, params string[] names)
|
||||
{
|
||||
foreach (var name in names)
|
||||
if (cookies.TryGetValue(name, out var value)) return value;
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
private static string BuildQueryString(IEnumerable<KeyValuePair<string, string>> parameters) =>
|
||||
string.Join("&", parameters.Select(x => Encode(x.Key) + "=" + Encode(x.Value ?? string.Empty)));
|
||||
|
||||
private static string Encode(string value) => Uri.EscapeDataString(value ?? string.Empty);
|
||||
|
||||
private sealed class SearchResponse
|
||||
{
|
||||
[JsonProperty("status_code")]
|
||||
public int StatusCode { get; set; }
|
||||
|
||||
[JsonProperty("user_list")]
|
||||
public List<SearchUserEntry> Users { get; set; }
|
||||
}
|
||||
|
||||
private sealed class SearchUserEntry
|
||||
{
|
||||
[JsonProperty("user_info")]
|
||||
public SearchUser User { get; set; }
|
||||
}
|
||||
|
||||
private sealed class SearchUser
|
||||
{
|
||||
[JsonProperty("sec_uid")]
|
||||
public string SecUid { get; set; }
|
||||
|
||||
[JsonProperty("uid")]
|
||||
public string Uid { get; set; }
|
||||
|
||||
[JsonProperty("unique_id")]
|
||||
public string UniqueId { get; set; }
|
||||
|
||||
[JsonProperty("short_id")]
|
||||
public string ShortId { get; set; }
|
||||
|
||||
[JsonProperty("nickname")]
|
||||
public string Nickname { get; set; }
|
||||
|
||||
[JsonProperty("signature")]
|
||||
public string Signature { get; set; }
|
||||
|
||||
[JsonProperty("enterprise_verify_reason")]
|
||||
public string Enterprise { get; set; }
|
||||
|
||||
[JsonProperty("custom_verify")]
|
||||
public string CustomVerify { get; set; }
|
||||
|
||||
[JsonProperty("follower_count")]
|
||||
public long FollowerCount { get; set; }
|
||||
|
||||
[JsonProperty("avatar_thumb")]
|
||||
public SearchAvatar Avatar { get; set; }
|
||||
}
|
||||
|
||||
private sealed class SearchAvatar
|
||||
{
|
||||
[JsonProperty("url_list")]
|
||||
public List<string> Urls { get; set; }
|
||||
}
|
||||
}
|
||||
}
|
||||
+305
-62
@@ -5,6 +5,9 @@ using dy.net.repository;
|
||||
using dy.net.utils;
|
||||
using Serilog;
|
||||
using SqlSugar;
|
||||
using dy.net.storage;
|
||||
using Newtonsoft.Json;
|
||||
using MediaStorageType = dy.net.model.dto.StorageType;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
@@ -15,12 +18,45 @@ namespace dy.net.service
|
||||
|
||||
private readonly DouyinVideoRepository _dyCollectVideoRepository;
|
||||
private readonly DouyinCookieRepository douyinCookieRepository;
|
||||
private readonly MediaStorageRouter _storageRouter;
|
||||
private readonly VideoTaskService _videoTasks;
|
||||
|
||||
public DouyinVideoService(DouyinVideoRepository dyCollectVideoRepository, DouyinCookieRepository douyinCookieRepository, ISqlSugarClient sqlSugarClient)
|
||||
public DouyinVideoService(DouyinVideoRepository dyCollectVideoRepository, DouyinCookieRepository douyinCookieRepository, ISqlSugarClient sqlSugarClient, MediaStorageRouter storageRouter, VideoTaskService videoTasks)
|
||||
{
|
||||
_dyCollectVideoRepository = dyCollectVideoRepository;
|
||||
this.douyinCookieRepository = douyinCookieRepository;
|
||||
this.sqlSugarClient = sqlSugarClient;
|
||||
_storageRouter = storageRouter;
|
||||
_videoTasks = videoTasks;
|
||||
}
|
||||
|
||||
public async Task<StorageRecordInventory> GetStorageInventoryAsync(MediaStorageType currentStorageType)
|
||||
{
|
||||
var records = await sqlSugarClient.Queryable<DouyinVideo>()
|
||||
.Select(x => new DouyinVideo { StorageType = x.StorageType, FileSize = x.FileSize })
|
||||
.ToListAsync();
|
||||
var local = records.Where(x => x.StorageType == MediaStorageType.Local).ToList();
|
||||
var webDav = records.Where(x => x.StorageType == MediaStorageType.WebDav).ToList();
|
||||
var openList = records.Where(x => x.StorageType == MediaStorageType.OpenList).ToList();
|
||||
var currentCount = currentStorageType switch
|
||||
{
|
||||
MediaStorageType.WebDav => webDav.Count,
|
||||
MediaStorageType.OpenList => openList.Count,
|
||||
_ => local.Count
|
||||
};
|
||||
return new StorageRecordInventory
|
||||
{
|
||||
CurrentStorageType = currentStorageType,
|
||||
TotalRecordCount = records.Count,
|
||||
LocalRecordCount = local.Count,
|
||||
WebDavRecordCount = webDav.Count,
|
||||
OpenListRecordCount = openList.Count,
|
||||
CurrentStorageRecordCount = currentCount,
|
||||
OtherStorageRecordCount = records.Count - currentCount,
|
||||
LocalDeclaredBytes = local.Sum(x => x.FileSize),
|
||||
WebDavDeclaredBytes = webDav.Sum(x => x.FileSize),
|
||||
OpenListDeclaredBytes = openList.Sum(x => x.FileSize)
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -76,9 +112,23 @@ namespace dy.net.service
|
||||
{
|
||||
if (updateMap.TryGetValue(existingVideo.AwemeId, out var updateData))
|
||||
{
|
||||
// 保留原记录主键,并把实际主键回写给任务条目使用。
|
||||
updateData.Id = existingVideo.Id;
|
||||
existingVideo.VideoSavePath = updateData.VideoSavePath;
|
||||
existingVideo.VideoCoverSavePath = updateData.VideoCoverSavePath;
|
||||
existingVideo.ViedoType = updateData.ViedoType;
|
||||
existingVideo.StorageType = updateData.StorageType;
|
||||
existingVideo.FileSize = updateData.FileSize;
|
||||
existingVideo.FileHash = updateData.FileHash;
|
||||
existingVideo.Resolution = updateData.Resolution;
|
||||
existingVideo.DynamicVideos = updateData.DynamicVideos;
|
||||
existingVideo.OnlyImgOrOnlyMp3 = updateData.OnlyImgOrOnlyMp3;
|
||||
existingVideo.IsMergeVideo = updateData.IsMergeVideo;
|
||||
existingVideo.VideoUrl = updateData.VideoUrl;
|
||||
existingVideo.AuthorAvatar = updateData.AuthorAvatar;
|
||||
existingVideo.CateId = updateData.CateId;
|
||||
existingVideo.CateXId = updateData.CateXId;
|
||||
existingVideo.SyncTime = updateData.SyncTime;
|
||||
}
|
||||
}
|
||||
// 批量更新数据库
|
||||
@@ -98,6 +148,167 @@ namespace dy.net.service
|
||||
return transaction;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 普通同步已经把同一作品提交到远端存储后,安全清理不再被本地记录引用的旧文件。
|
||||
/// 数据库或远端主媒体未验证通过时不会删除任何本地文件。
|
||||
/// </summary>
|
||||
public async Task<int> CleanupSupersededLocalArtifactsAsync(DouyinVideo snapshot, string replacementPath)
|
||||
{
|
||||
if (snapshot == null || snapshot.StorageType != MediaStorageType.Local
|
||||
|| string.IsNullOrWhiteSpace(snapshot.AwemeId) || string.IsNullOrWhiteSpace(replacementPath)) return 0;
|
||||
var current = await sqlSugarClient.Queryable<DouyinVideo>()
|
||||
.Where(x => x.AwemeId == snapshot.AwemeId).FirstAsync();
|
||||
if (current == null || !current.StorageType.IsRemote()
|
||||
|| !string.Equals(current.VideoSavePath, replacementPath, StringComparison.Ordinal))
|
||||
throw new InvalidOperationException("新存储记录尚未提交,保留旧本地文件。");
|
||||
var remoteLength = await _storageRouter.Resolve(current.StorageType).GetLengthAsync(replacementPath);
|
||||
if (!remoteLength.HasValue || remoteLength.Value <= 0)
|
||||
throw new InvalidOperationException("新存储主媒体不存在或为空,保留旧本地文件。");
|
||||
|
||||
var cookie = await sqlSugarClient.Queryable<DouyinCookie>().InSingleAsync(snapshot.CookieId);
|
||||
var roots = StorageMigrationPathPolicy.GetLocalRoots(cookie);
|
||||
if (roots.Count == 0) throw new InvalidOperationException("旧存储根目录不可用,保留旧本地文件。");
|
||||
var localVideos = await sqlSugarClient.Queryable<DouyinVideo>()
|
||||
.Where(x => x.StorageType == MediaStorageType.Local).ToListAsync();
|
||||
var deleted = 0;
|
||||
foreach (var candidate in BuildLocalCleanupCandidates(snapshot))
|
||||
{
|
||||
if (!File.Exists(candidate.Path)) continue;
|
||||
if (!SafeLocalMigrationFile.TryResolve(candidate.Path, roots, out var safePath, out var error))
|
||||
throw new InvalidOperationException($"拒绝清理不安全旧路径 {candidate.Path}:{error}");
|
||||
if (IsLocalPathReferenced(candidate.Path, localVideos, snapshot.Id, candidate.Shared)) continue;
|
||||
File.Delete(safePath);
|
||||
deleted++;
|
||||
}
|
||||
DeleteEmptyParents(snapshot.VideoSavePath, roots);
|
||||
return deleted;
|
||||
}
|
||||
|
||||
public async Task<int> RetryTaskItemLocalCleanupAsync(string taskId, string itemId)
|
||||
{
|
||||
var task = await sqlSugarClient.Queryable<VideoDownloadTask>().InSingleAsync(taskId)
|
||||
?? throw new KeyNotFoundException("任务不存在。");
|
||||
if (task.Type != VideoTaskType.Sync)
|
||||
throw new InvalidOperationException("仅普通同步任务支持重试旧本地文件清理。");
|
||||
var item = await sqlSugarClient.Queryable<VideoDownloadTaskItem>().InSingleAsync(itemId)
|
||||
?? throw new KeyNotFoundException("任务条目不存在。");
|
||||
if (item.TaskId != taskId || !VideoTaskService.CanRetryCleanup(item))
|
||||
throw new InvalidOperationException("该条目当前没有可重试的旧本地文件清理。");
|
||||
|
||||
DouyinVideo replacement;
|
||||
try
|
||||
{
|
||||
replacement = JsonConvert.DeserializeObject<DouyinVideo>(item.RetrySnapshotJson);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
throw new InvalidOperationException("旧本地记录快照损坏,无法安全重试清理。", ex);
|
||||
}
|
||||
var snapshot = replacement?.SupersededStorageSnapshot;
|
||||
if (snapshot == null || snapshot.StorageType != MediaStorageType.Local)
|
||||
throw new InvalidOperationException("任务没有可验证的旧本地记录快照,拒绝清理。");
|
||||
var current = !string.IsNullOrWhiteSpace(item.VideoId)
|
||||
? await sqlSugarClient.Queryable<DouyinVideo>().InSingleAsync(item.VideoId)
|
||||
: null;
|
||||
current ??= await sqlSugarClient.Queryable<DouyinVideo>()
|
||||
.Where(x => x.AwemeId == item.AwemeId).FirstAsync();
|
||||
if (current == null)
|
||||
throw new InvalidOperationException("当前视频记录不存在,拒绝清理旧文件。");
|
||||
|
||||
try
|
||||
{
|
||||
var deleted = await CleanupSupersededLocalArtifactsAsync(snapshot, current.VideoSavePath);
|
||||
item.CleanupPending = false;
|
||||
item.CleanupError = null;
|
||||
item.WarningMessage = RemoveCleanupWarning(item.WarningMessage);
|
||||
item.Stage = string.IsNullOrWhiteSpace(item.WarningMessage)
|
||||
? VideoTaskItemStage.Succeeded
|
||||
: VideoTaskItemStage.SucceededWithWarnings;
|
||||
item.UpdatedAt = DateTime.Now;
|
||||
await sqlSugarClient.Updateable(item).ExecuteCommandAsync();
|
||||
await _videoTasks.RefreshCountsAsync(taskId);
|
||||
return deleted;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
var reason = ex.GetBaseException().Message;
|
||||
item.CleanupPending = true;
|
||||
item.CleanupError = reason;
|
||||
item.UpdatedAt = DateTime.Now;
|
||||
await sqlSugarClient.Updateable(item).ExecuteCommandAsync();
|
||||
await _videoTasks.RefreshCountsAsync(taskId);
|
||||
throw new InvalidOperationException($"旧本地文件清理仍失败:{reason}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
internal static string RemoveCleanupWarning(string warning)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(warning)) return null;
|
||||
var failureIndex = warning.IndexOf("旧本地文件清理失败", StringComparison.Ordinal);
|
||||
if (failureIndex < 0) return warning.Trim();
|
||||
var start = warning.LastIndexOf(";已切换到", failureIndex, StringComparison.Ordinal);
|
||||
if (start >= 0) return warning[..start].Trim().TrimEnd(';');
|
||||
start = warning.LastIndexOf("已切换到", failureIndex, StringComparison.Ordinal);
|
||||
if (start == 0) return null;
|
||||
return warning[..failureIndex].Trim().TrimEnd(';', ',', ',');
|
||||
}
|
||||
|
||||
private static List<LocalCleanupCandidate> BuildLocalCleanupCandidates(DouyinVideo snapshot)
|
||||
{
|
||||
var result = new List<LocalCleanupCandidate>
|
||||
{
|
||||
new(snapshot.VideoSavePath, false),
|
||||
new(snapshot.VideoCoverSavePath, snapshot.ViedoType is VideoTypeEnum.dy_mix or VideoTypeEnum.dy_series),
|
||||
new(snapshot.AuthorAvatar, true)
|
||||
};
|
||||
if (!string.IsNullOrWhiteSpace(snapshot.VideoSavePath))
|
||||
{
|
||||
var directory = Path.GetDirectoryName(snapshot.VideoSavePath);
|
||||
var basename = Path.GetFileNameWithoutExtension(snapshot.VideoSavePath);
|
||||
result.Add(new LocalCleanupCandidate(Path.Combine(directory ?? string.Empty, basename + ".nfo"), false));
|
||||
if (snapshot.ViedoType is VideoTypeEnum.dy_mix or VideoTypeEnum.dy_series)
|
||||
result.Add(new LocalCleanupCandidate(Path.Combine(directory ?? string.Empty, "tvshow.nfo"), true));
|
||||
}
|
||||
if (!string.IsNullOrWhiteSpace(snapshot.DynamicVideos))
|
||||
{
|
||||
try
|
||||
{
|
||||
foreach (var attachment in JsonConvert.DeserializeObject<List<DouyinMergeVideoDto>>(snapshot.DynamicVideos) ?? new())
|
||||
result.Add(new LocalCleanupCandidate(attachment.Path, false));
|
||||
}
|
||||
catch (JsonException) { }
|
||||
}
|
||||
return result.Where(x => !string.IsNullOrWhiteSpace(x.Path)).DistinctBy(x => x.Path).ToList();
|
||||
}
|
||||
|
||||
private static bool IsLocalPathReferenced(string path, IEnumerable<DouyinVideo> localVideos, string excludedVideoId, bool shared)
|
||||
{
|
||||
foreach (var video in localVideos.Where(x => x.Id != excludedVideoId))
|
||||
{
|
||||
if (path == video.VideoSavePath || path == video.VideoCoverSavePath || path == video.AuthorAvatar) return true;
|
||||
if (shared && !string.IsNullOrWhiteSpace(video.VideoSavePath)
|
||||
&& string.Equals(Path.GetDirectoryName(video.VideoSavePath), Path.GetDirectoryName(path), StringComparison.Ordinal)) return true;
|
||||
if (!string.IsNullOrWhiteSpace(video.DynamicVideos) && video.DynamicVideos.Contains(path, StringComparison.Ordinal)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static void DeleteEmptyParents(string filePath, IReadOnlyList<string> roots)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(filePath)) return;
|
||||
var directory = Path.GetDirectoryName(Path.GetFullPath(filePath));
|
||||
var root = roots.Where(x => SafeLocalMigrationFile.IsWithin(filePath, x))
|
||||
.OrderByDescending(x => x.Length).FirstOrDefault();
|
||||
while (!string.IsNullOrWhiteSpace(directory) && !string.Equals(directory, root, StringComparison.Ordinal)
|
||||
&& Directory.Exists(directory) && !Directory.EnumerateFileSystemEntries(directory).Any())
|
||||
{
|
||||
Directory.Delete(directory, false);
|
||||
directory = Path.GetDirectoryName(directory);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record LocalCleanupCandidate(string Path, bool Shared);
|
||||
|
||||
public async Task<bool> UpdateOne(DouyinVideo video)
|
||||
{
|
||||
return await _dyCollectVideoRepository.UpdateAsync(video);
|
||||
@@ -246,31 +457,57 @@ namespace dy.net.service
|
||||
return false;
|
||||
}
|
||||
|
||||
// Permanent deletion is an explicit, separate operation. A normal re-download never removes
|
||||
// the database row or old media first; VideoRedownloadWorker replaces the main file atomically.
|
||||
if (forever)
|
||||
{
|
||||
foreach (var video in videos)
|
||||
{
|
||||
try
|
||||
{
|
||||
var storage = _storageRouter.Resolve(video.StorageType);
|
||||
if (video.StorageType.IsRemote())
|
||||
await StorageArtifactCleaner.DeleteWebDavArtifactsAsync(storage, video);
|
||||
else
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(video.VideoSavePath)) await storage.DeleteAsync(video.VideoSavePath);
|
||||
if (!string.IsNullOrWhiteSpace(video.VideoCoverSavePath)) await storage.DeleteAsync(video.VideoCoverSavePath);
|
||||
}
|
||||
if (!await _dyCollectVideoRepository.DeleteByIdAsync(video.Id)) return false;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Serilog.Log.Error(ex, "永久删除视频失败,数据库记录已保留:{VideoId}", video.Id);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// 3. 构建重新下载记录(提前准备数据,避免事务内耗时操作)
|
||||
var reDownList = new List<DouyinReDownload>();
|
||||
var filePathsToDelete = new List<(string path, bool onlyImgOrMp3)>(); // 收集待删除文件路径,统一处理
|
||||
|
||||
foreach (var video in videos)
|
||||
{
|
||||
// 跳过无保存路径的视频(避免无效文件操作)
|
||||
// 跳过无保存路径的视频(避免创建无效任务)
|
||||
if (string.IsNullOrWhiteSpace(video.VideoSavePath))
|
||||
{
|
||||
Serilog.Log.Debug("视频无保存路径,跳过文件删除:VideoId={0}", video.Id);
|
||||
Serilog.Log.Debug("视频无保存路径,跳过重新下载:VideoId={0}", video.Id);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 构建重新下载记录
|
||||
reDownList.Add(new DouyinReDownload
|
||||
{
|
||||
Id = IdGener.GetLong().ToString(),
|
||||
Id = Guid.NewGuid().ToString("N"),
|
||||
CreateTime = DateTime.UtcNow, // 统一使用UTC时间,避免时区问题
|
||||
Status = 0, // 0=待下载(建议用枚举替代魔法值)
|
||||
SavePath = video.VideoSavePath,
|
||||
ViedoId = video.AwemeId,
|
||||
CookieId = video.CookieId
|
||||
CookieId = video.CookieId,
|
||||
VideoRecordId = video.Id,
|
||||
UpdateTime = DateTime.UtcNow
|
||||
});
|
||||
|
||||
filePathsToDelete.Add((video.VideoSavePath, video.OnlyImgOrOnlyMp3));
|
||||
}
|
||||
|
||||
// 无有效重新下载记录时直接返回
|
||||
@@ -282,52 +519,40 @@ namespace dy.net.service
|
||||
|
||||
try
|
||||
{
|
||||
// 4. 数据库操作(事务保证一致性:创建重新下载记录 + 删除原视频记录必须同时成功/失败)
|
||||
var transactionResult = await _dyCollectVideoRepository.UseTranAsync(async () =>
|
||||
var videosById = videos.ToDictionary(x => x.Id);
|
||||
var groups = reDownList.GroupBy(x => videosById[x.VideoRecordId].StorageType).ToList();
|
||||
var transaction = await sqlSugarClient.Ado.UseTranAsync(async () =>
|
||||
{
|
||||
// 4.1 批量插入重新下载记录(SqlSugar批量插入效率更高)
|
||||
_dyCollectVideoRepository.InsertReDowns(reDownList);
|
||||
// 4.2 批量删除原视频记录(使用视频实际存在的ID,避免无效删除)
|
||||
var actualDeleteIds = videos.Select(v => v.Id).ToList();
|
||||
var deleteCount = await _dyCollectVideoRepository.DeleteByIdsAsync(actualDeleteIds); // 建议仓储层提供异步删除方法
|
||||
}, e =>
|
||||
{
|
||||
Serilog.Log.Error(e, "数据库事务执行失败:Ids={0}", string.Join(",", videoIds));
|
||||
foreach (var group in groups)
|
||||
{
|
||||
var jobs = group.ToList();
|
||||
var storageLabel = group.Key switch
|
||||
{
|
||||
MediaStorageType.WebDav => "WebDAV",
|
||||
MediaStorageType.OpenList => "OpenList",
|
||||
_ => "本地"
|
||||
};
|
||||
var title = groups.Count == 1
|
||||
? $"重新下载({jobs.Count} 条)"
|
||||
: $"重新下载 · {storageLabel}({jobs.Count} 条)";
|
||||
var task = await _videoTasks.CreateTaskAsync(VideoTaskType.Redownload,
|
||||
VideoTaskTrigger.UserAction, title, storageType: group.Key);
|
||||
foreach (var job in jobs)
|
||||
{
|
||||
var video = videosById[job.VideoRecordId];
|
||||
var item = await _videoTasks.AddItemAsync(task.Id, video.AwemeId, video.CookieId, null,
|
||||
video.ViedoType, video.VideoTitle, video.Author, video.VideoSavePath,
|
||||
new[] { video.VideoUrl }, video, video.FileSize);
|
||||
job.TaskId = task.Id;
|
||||
job.TaskItemId = item.Id;
|
||||
}
|
||||
if (await sqlSugarClient.Insertable(jobs).ExecuteCommandAsync() != jobs.Count)
|
||||
throw new InvalidOperationException($"{storageLabel}重新下载队列写入不完整。");
|
||||
}
|
||||
});
|
||||
|
||||
// 5. 文件删除(非事务操作,失败不回滚数据库,可根据业务调整)
|
||||
// 采用异步文件操作,避免同步IO阻塞线程(需.NET 5+支持)
|
||||
foreach (var video in filePathsToDelete)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (File.Exists(video.path))
|
||||
{
|
||||
File.Delete(video.path); // 异步删除,提升并发性能
|
||||
Serilog.Log.Debug("视频文件删除成功:Path={0}", video.path);
|
||||
|
||||
if (!video.onlyImgOrMp3)//如果是纯图片或纯音频文件,则不删除所在文件夹
|
||||
{
|
||||
//检查这个路径所在文件夹是否还有其他视频文件,如果没有则删除这个文件夹
|
||||
var dir = Path.GetDirectoryName(video.path);
|
||||
|
||||
bool hasMp4File = Directory.EnumerateFiles(dir, "*.mp4", SearchOption.TopDirectoryOnly).Any(); // 只要存在一个MP4文件就返回true;
|
||||
if (!hasMp4File)
|
||||
{
|
||||
Directory.Delete(dir, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
Serilog.Log.Error("视频文件不存在,跳过删除:Path={0}", video);
|
||||
}
|
||||
}
|
||||
catch (IOException ex)
|
||||
{
|
||||
Serilog.Log.Error(ex, "视频文件删除失败:Path={0}", video);
|
||||
}
|
||||
}
|
||||
if (!transaction.IsSuccess)
|
||||
throw new InvalidOperationException("创建重新下载任务失败:" + transaction.ErrorMessage,
|
||||
transaction.ErrorException);
|
||||
|
||||
//var CookieIds = reDownList.Select(x => x.CookieId).Distinct();
|
||||
//foreach (var ck in CookieIds)
|
||||
@@ -362,8 +587,7 @@ namespace dy.net.service
|
||||
// await douyinCookieRepository.UpdateAsync(cookie);
|
||||
|
||||
//}
|
||||
if (!forever)
|
||||
Serilog.Log.Debug("重新下载视频流程执行完成:成功创建{0}条重新下载记录,删除{1}个文件,等待重新下载...", reDownList.Count, filePathsToDelete.Count);
|
||||
Serilog.Log.Debug("安全重新下载任务已创建:{0} 条;旧记录和旧文件会保留到新主媒体完整写入。", reDownList.Count);
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex)
|
||||
@@ -415,7 +639,8 @@ namespace dy.net.service
|
||||
List<string> douyinVideoIds = new List<string>();
|
||||
foreach (var v in videos)
|
||||
{
|
||||
if (!File.Exists(v.VideoSavePath))
|
||||
if (v.OnlyImgOrOnlyMp3 && string.IsNullOrWhiteSpace(v.VideoSavePath)) continue;
|
||||
if (!await _storageRouter.Resolve(v.StorageType).ExistsAsync(v.VideoSavePath))
|
||||
{
|
||||
douyinVideoIds.Add(v.Id);
|
||||
vList.Add(new DeleteInvalidVideoDto { AwId = v.AwemeId, Title = v.VideoTitle, Path = v.VideoSavePath });
|
||||
@@ -443,16 +668,26 @@ namespace dy.net.service
|
||||
|
||||
internal async Task<int> AddDeleteVideo(List<DouyinVideo> videos)
|
||||
{
|
||||
var deletes = videos.Select(video => new DouyinVideoDelete
|
||||
var awemeIds = videos.Select(x => x.AwemeId).Where(x => !string.IsNullOrWhiteSpace(x)).Distinct().ToList();
|
||||
var existing = await sqlSugarClient.Queryable<DouyinVideoDelete>()
|
||||
.Where(x => awemeIds.Contains(x.ViedoId)).Select(x => x.ViedoId).ToListAsync();
|
||||
var deletes = videos.Where(video => !existing.Contains(video.AwemeId)).Select(video => new DouyinVideoDelete
|
||||
{
|
||||
ViedoId = video.AwemeId,
|
||||
VideoTitle = video.VideoTitle,
|
||||
VideoSavePath = video.VideoSavePath,
|
||||
Id = IdGener.GetLong().ToString(),
|
||||
DeleteTime = DateTime.Now
|
||||
DeleteTime = DateTime.Now,
|
||||
CookieId = video.CookieId,
|
||||
VideoType = video.ViedoType,
|
||||
AuthorId = video.AuthorId,
|
||||
Author = video.Author,
|
||||
VideoUrl = video.VideoUrl,
|
||||
RestoreSnapshotJson = JsonConvert.SerializeObject(video)
|
||||
})?.ToList();
|
||||
|
||||
return await sqlSugarClient.Insertable<DouyinVideoDelete>(deletes).ExecuteCommandAsync();
|
||||
if (deletes.Count > 0) await sqlSugarClient.Insertable<DouyinVideoDelete>(deletes).ExecuteCommandAsync();
|
||||
return existing.Distinct().Count() + deletes.Count;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -470,11 +705,15 @@ namespace dy.net.service
|
||||
{
|
||||
if (videos.Count <= 30)
|
||||
{
|
||||
var deletes = await AddDeleteVideo(videos);
|
||||
if (deletes < videos.Select(x => x.AwemeId).Distinct().Count())
|
||||
{
|
||||
Serilog.Log.Error("写入永久排除记录失败,已停止删除媒体");
|
||||
return false;
|
||||
}
|
||||
var result = await ReDownloadViedoAsync(new ReDownViedoDto { Ids = videos.Select(x => x.Id)?.ToList() }, true);
|
||||
if (result)
|
||||
{
|
||||
//加入删除逻辑
|
||||
var deletes = await AddDeleteVideo(videos);
|
||||
Serilog.Log.Debug($"批量永久删除博主{videos.FirstOrDefault()?.Author},共{deletes}条记录");
|
||||
return true;
|
||||
}
|
||||
@@ -488,11 +727,15 @@ namespace dy.net.service
|
||||
{
|
||||
Task.Run(async () =>
|
||||
{
|
||||
var deletes = await AddDeleteVideo(videos);
|
||||
if (deletes < videos.Select(x => x.AwemeId).Distinct().Count())
|
||||
{
|
||||
Serilog.Log.Error("写入永久排除记录失败,已停止后台删除媒体");
|
||||
return;
|
||||
}
|
||||
var result = await ReDownloadViedoAsync(new ReDownViedoDto { Ids = videos.Select(x => x.Id)?.ToList() }, true);
|
||||
if (result)
|
||||
{
|
||||
//加入删除逻辑
|
||||
var deletes = await AddDeleteVideo(videos);
|
||||
Serilog.Log.Debug($"批量永久删除博主{videos.FirstOrDefault()?.Author},{deletes}条记录");
|
||||
}
|
||||
else
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
using dy.net.model.entity;
|
||||
using MailKit.Net.Smtp;
|
||||
using MimeKit;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
public interface IEmailNotificationSender
|
||||
{
|
||||
Task SendAsync(
|
||||
EmailNotificationSettings settings,
|
||||
string password,
|
||||
string subject,
|
||||
string htmlBody,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
public sealed class EmailNotificationSender : IEmailNotificationSender
|
||||
{
|
||||
public async Task SendAsync(
|
||||
EmailNotificationSettings settings,
|
||||
string password,
|
||||
string subject,
|
||||
string htmlBody,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var message = new MimeMessage();
|
||||
message.From.Add(new MailboxAddress(
|
||||
string.IsNullOrWhiteSpace(settings.FromName) ? "dysync.net" : settings.FromName,
|
||||
settings.FromAddress));
|
||||
message.To.AddRange(EmailNotificationSettingsService.ParseRecipients(settings.Recipients));
|
||||
message.Subject = subject;
|
||||
message.Body = new BodyBuilder { HtmlBody = htmlBody }.ToMessageBody();
|
||||
|
||||
using var client = new SmtpClient();
|
||||
client.Timeout = 30_000;
|
||||
await client.ConnectAsync(
|
||||
settings.Host,
|
||||
settings.Port,
|
||||
EmailNotificationSettingsService.ToSocketOptions(settings.SecurityMode),
|
||||
cancellationToken);
|
||||
if (!string.IsNullOrWhiteSpace(settings.UserName))
|
||||
await client.AuthenticateAsync(settings.UserName, password ?? string.Empty, cancellationToken);
|
||||
await client.SendAsync(message, cancellationToken);
|
||||
await client.DisconnectAsync(true, cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
using dy.net.model.dto;
|
||||
using dy.net.model.entity;
|
||||
using MailKit.Security;
|
||||
using Microsoft.AspNetCore.DataProtection;
|
||||
using MimeKit;
|
||||
using SqlSugar;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
public sealed class EmailNotificationSettingsService
|
||||
{
|
||||
private const string SettingsId = "default";
|
||||
private readonly ISqlSugarClient _db;
|
||||
private readonly IDataProtector _protector;
|
||||
private readonly IEmailNotificationSender _sender;
|
||||
|
||||
public EmailNotificationSettingsService(
|
||||
ISqlSugarClient db,
|
||||
IDataProtectionProvider provider,
|
||||
IEmailNotificationSender sender)
|
||||
{
|
||||
_db = db;
|
||||
_protector = provider.CreateProtector("dysync.email.password.v1");
|
||||
_sender = sender;
|
||||
}
|
||||
|
||||
public async Task<EmailNotificationSettings> GetAsync() =>
|
||||
await _db.Queryable<EmailNotificationSettings>().InSingleAsync(SettingsId)
|
||||
?? new EmailNotificationSettings();
|
||||
|
||||
public async Task<string> GetPasswordAsync(EmailNotificationSettings settings = null)
|
||||
{
|
||||
settings ??= await GetAsync();
|
||||
if (string.IsNullOrWhiteSpace(settings.ProtectedPassword)) return string.Empty;
|
||||
try { return _protector.Unprotect(settings.ProtectedPassword); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
Serilog.Log.Error(ex, "SMTP 密码解密失败,请重新输入密码");
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<EmailNotificationSettings> BuildCandidateAsync(EmailNotificationSettingsDto dto)
|
||||
{
|
||||
dto ??= new EmailNotificationSettingsDto();
|
||||
var existing = await GetAsync();
|
||||
var password = string.IsNullOrWhiteSpace(dto.Password)
|
||||
? await GetPasswordAsync(existing)
|
||||
: dto.Password;
|
||||
var candidate = new EmailNotificationSettings
|
||||
{
|
||||
Id = SettingsId,
|
||||
Enabled = dto.Enabled,
|
||||
Host = dto.Host?.Trim(),
|
||||
Port = dto.Port <= 0 ? 465 : dto.Port,
|
||||
SecurityMode = dto.SecurityMode,
|
||||
UserName = dto.UserName?.Trim(),
|
||||
ProtectedPassword = string.IsNullOrWhiteSpace(password) ? null : _protector.Protect(password),
|
||||
FromAddress = dto.FromAddress?.Trim(),
|
||||
FromName = Limit(dto.FromName, 200),
|
||||
Recipients = NormalizeRecipients(dto.Recipients),
|
||||
LastTestedAt = existing.LastTestedAt,
|
||||
LastTestMessage = existing.LastTestMessage
|
||||
};
|
||||
Validate(candidate, password, requireComplete: candidate.Enabled);
|
||||
return candidate;
|
||||
}
|
||||
|
||||
public async Task<EmailNotificationSettingsDto> SaveAsync(EmailNotificationSettingsDto dto)
|
||||
{
|
||||
var candidate = await BuildCandidateAsync(dto);
|
||||
await _db.Storageable(candidate).ExecuteCommandAsync();
|
||||
return ToDto(candidate);
|
||||
}
|
||||
|
||||
public async Task<string> TestAsync(EmailNotificationSettingsDto dto, CancellationToken cancellationToken)
|
||||
{
|
||||
var candidate = await BuildCandidateAsync(dto);
|
||||
var password = await GetPasswordAsync(candidate);
|
||||
Validate(candidate, password, requireComplete: true);
|
||||
var now = DateTime.Now;
|
||||
await _sender.SendAsync(
|
||||
candidate,
|
||||
password,
|
||||
"dysync.net 邮箱通知测试成功",
|
||||
$"<h2>邮箱通知配置可用</h2><p>测试时间:{now:yyyy-MM-dd HH:mm:ss}</p><p>开播通知将只在博主首次进入直播状态时发送。</p>",
|
||||
cancellationToken);
|
||||
candidate.LastTestedAt = DateTime.Now;
|
||||
candidate.LastTestMessage = "测试邮件发送成功";
|
||||
await _db.Storageable(candidate).ExecuteCommandAsync();
|
||||
return candidate.LastTestMessage;
|
||||
}
|
||||
|
||||
public async Task<string> GetReadinessErrorAsync()
|
||||
{
|
||||
var settings = await GetAsync();
|
||||
if (!settings.Enabled) return "请先在系统设置中启用邮箱通知";
|
||||
var password = await GetPasswordAsync(settings);
|
||||
try
|
||||
{
|
||||
Validate(settings, password, requireComplete: true);
|
||||
return null;
|
||||
}
|
||||
catch (InvalidOperationException ex)
|
||||
{
|
||||
return "邮箱通知配置不完整:" + ex.Message;
|
||||
}
|
||||
}
|
||||
|
||||
public EmailNotificationSettingsDto ToDto(EmailNotificationSettings settings) => new()
|
||||
{
|
||||
Enabled = settings.Enabled,
|
||||
Host = settings.Host,
|
||||
Port = settings.Port,
|
||||
SecurityMode = settings.SecurityMode,
|
||||
UserName = settings.UserName,
|
||||
HasPassword = !string.IsNullOrWhiteSpace(settings.ProtectedPassword),
|
||||
FromAddress = settings.FromAddress,
|
||||
FromName = settings.FromName,
|
||||
Recipients = settings.Recipients,
|
||||
LastTestedAt = settings.LastTestedAt,
|
||||
LastTestMessage = settings.LastTestMessage
|
||||
};
|
||||
|
||||
public static SecureSocketOptions ToSocketOptions(EmailSecurityMode mode) => mode switch
|
||||
{
|
||||
EmailSecurityMode.None => SecureSocketOptions.None,
|
||||
EmailSecurityMode.StartTls => SecureSocketOptions.StartTls,
|
||||
_ => SecureSocketOptions.SslOnConnect
|
||||
};
|
||||
|
||||
private static void Validate(EmailNotificationSettings settings, string password, bool requireComplete)
|
||||
{
|
||||
if (settings.Port is < 1 or > 65535) throw new InvalidOperationException("SMTP 端口必须在 1–65535 之间");
|
||||
if (!requireComplete) return;
|
||||
if (string.IsNullOrWhiteSpace(settings.Host)) throw new InvalidOperationException("请填写 SMTP 服务器");
|
||||
if (string.IsNullOrWhiteSpace(settings.FromAddress) || !MailboxAddress.TryParse(settings.FromAddress, out _))
|
||||
throw new InvalidOperationException("发件邮箱格式不正确");
|
||||
if (ParseRecipients(settings.Recipients).Count == 0) throw new InvalidOperationException("请至少填写一个收件邮箱");
|
||||
if (!string.IsNullOrWhiteSpace(settings.UserName) && string.IsNullOrWhiteSpace(password))
|
||||
throw new InvalidOperationException("请填写 SMTP 密码或授权码");
|
||||
}
|
||||
|
||||
public static List<MailboxAddress> ParseRecipients(string recipients)
|
||||
{
|
||||
var result = new List<MailboxAddress>();
|
||||
foreach (var value in (recipients ?? string.Empty)
|
||||
.Split(new[] { ',', ';', '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)
|
||||
.Select(x => x.Trim()).Distinct(StringComparer.OrdinalIgnoreCase).Take(20))
|
||||
{
|
||||
if (!MailboxAddress.TryParse(value, out var address))
|
||||
throw new InvalidOperationException($"收件邮箱格式不正确:{value}");
|
||||
result.Add(address);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static string NormalizeRecipients(string recipients) =>
|
||||
string.Join(";", ParseRecipients(recipients).Select(x => x.Address));
|
||||
|
||||
private static string Limit(string value, int maxLength)
|
||||
{
|
||||
value = value?.Trim();
|
||||
return value != null && value.Length > maxLength ? value[..maxLength] : value;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
using System.Net;
|
||||
using dy.net.model.entity;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
public sealed class LiveEmailNotificationService
|
||||
{
|
||||
private readonly EmailNotificationSettingsService _settingsService;
|
||||
private readonly IEmailNotificationSender _sender;
|
||||
|
||||
public LiveEmailNotificationService(
|
||||
EmailNotificationSettingsService settingsService,
|
||||
IEmailNotificationSender sender)
|
||||
{
|
||||
_settingsService = settingsService;
|
||||
_sender = sender;
|
||||
}
|
||||
|
||||
public Task<string> GetReadinessErrorAsync() => _settingsService.GetReadinessErrorAsync();
|
||||
|
||||
public async Task<(bool Sent, string Error)> SendLiveStartedAsync(
|
||||
DouyinFollowed follow,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
var settings = await _settingsService.GetAsync();
|
||||
if (!settings.Enabled || !follow.LiveEmailNotificationEnabled) return (false, null);
|
||||
var password = await _settingsService.GetPasswordAsync(settings);
|
||||
var roomUrl = !string.IsNullOrWhiteSpace(follow.LiveWebRid)
|
||||
? "https://live.douyin.com/" + Uri.EscapeDataString(follow.LiveWebRid)
|
||||
: "https://www.douyin.com/user/" + Uri.EscapeDataString(follow.SecUid ?? string.Empty);
|
||||
var blogger = WebUtility.HtmlEncode(follow.UperName ?? "关注博主");
|
||||
var title = WebUtility.HtmlEncode(follow.LiveTitle ?? "正在直播");
|
||||
var safeUrl = WebUtility.HtmlEncode(roomUrl);
|
||||
var subject = $"[dysync.net] {follow.UperName ?? "关注博主"} 开播了";
|
||||
var body = $"<h2>{blogger} 正在直播</h2><p>{title}</p>" +
|
||||
$"<p>检测时间:{DateTime.Now:yyyy-MM-dd HH:mm:ss}</p>" +
|
||||
$"<p><a href=\"{safeUrl}\">进入直播间</a></p>";
|
||||
await _sender.SendAsync(settings, password, subject, body, cancellationToken);
|
||||
return (true, null);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
Serilog.Log.Warning(ex, "发送开播邮件失败:Blogger={Blogger}", follow.UperName);
|
||||
return (false, SafeError(ex));
|
||||
}
|
||||
}
|
||||
|
||||
private static string SafeError(Exception ex)
|
||||
{
|
||||
var message = ex.GetBaseException().Message?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(message)) return "开播邮件发送失败,请检查邮箱设置";
|
||||
return message.Length <= 1000 ? message : message[..1000];
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,351 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using dy.net.model.dto;
|
||||
using dy.net.model.entity;
|
||||
using dy.net.storage;
|
||||
using SqlSugar;
|
||||
using MediaStorageType = dy.net.model.dto.StorageType;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
public sealed class OpenListDirectoryRepairService
|
||||
{
|
||||
private readonly ISqlSugarClient _db;
|
||||
private readonly DouyinCommonService _common;
|
||||
private readonly OpenListSettingsService _settingsService;
|
||||
private readonly OpenListClient _client;
|
||||
|
||||
public OpenListDirectoryRepairService(
|
||||
ISqlSugarClient db,
|
||||
DouyinCommonService common,
|
||||
OpenListSettingsService settingsService,
|
||||
OpenListClient client)
|
||||
{
|
||||
_db = db;
|
||||
_common = common;
|
||||
_settingsService = settingsService;
|
||||
_client = client;
|
||||
}
|
||||
|
||||
public async Task<OpenListDirectoryRepairPreflightResult> PreflightAsync(
|
||||
OpenListDirectoryRepairPreflightRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var plan = await BuildPlanAsync(request?.LogicalPath, cancellationToken);
|
||||
return plan.Result;
|
||||
}
|
||||
|
||||
public async Task<OpenListDirectoryRepairTaskDetail> CreateAsync(
|
||||
CreateOpenListDirectoryRepairRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var plan = await BuildPlanAsync(request?.LogicalPath, cancellationToken);
|
||||
if (!plan.Result.CanStart) throw new InvalidOperationException(string.Join(";", plan.Result.Errors));
|
||||
if (!string.Equals(request?.ConfigurationFingerprint, plan.Result.ConfigurationFingerprint, StringComparison.Ordinal))
|
||||
throw new InvalidOperationException("OpenList 配置或目录内容已变化,请重新预检。");
|
||||
if (await _db.Queryable<OpenListDirectoryRepairTask>().Where(x =>
|
||||
x.Status == OpenListDirectoryRepairStatus.Queued
|
||||
|| x.Status == OpenListDirectoryRepairStatus.Scanning
|
||||
|| x.Status == OpenListDirectoryRepairStatus.AwaitingConfirmation
|
||||
|| x.Status == OpenListDirectoryRepairStatus.Cleaning
|
||||
|| x.Status == OpenListDirectoryRepairStatus.Paused).AnyAsync())
|
||||
throw new InvalidOperationException("已有未结束的 OpenList 目录修复任务。");
|
||||
|
||||
var now = DateTime.Now;
|
||||
var task = new OpenListDirectoryRepairTask
|
||||
{
|
||||
Id = Guid.NewGuid().ToString("N"),
|
||||
Status = OpenListDirectoryRepairStatus.Queued,
|
||||
ConfigurationFingerprint = plan.Result.ConfigurationFingerprint,
|
||||
RequestedPath = plan.Result.RequestedPath,
|
||||
CanonicalPath = plan.Result.CanonicalPath,
|
||||
ActualParentPath = plan.ActualParentPath,
|
||||
RequestedName = plan.RequestedName,
|
||||
CandidatePattern = plan.Result.CandidatePattern,
|
||||
TotalCount = plan.Candidates.Count,
|
||||
PendingCount = plan.Candidates.Count,
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now
|
||||
};
|
||||
var items = plan.Candidates.Select((candidate, index) => new OpenListDirectoryRepairItem
|
||||
{
|
||||
Id = Guid.NewGuid().ToString("N"),
|
||||
TaskId = task.Id,
|
||||
DirectoryName = candidate.Name,
|
||||
ActualPath = candidate.Path,
|
||||
Status = OpenListDirectoryRepairItemStatus.Pending,
|
||||
CreatedAt = now.AddTicks(index),
|
||||
UpdatedAt = now
|
||||
}).ToList();
|
||||
var transaction = await _db.Ado.UseTranAsync(async () =>
|
||||
{
|
||||
await _db.Insertable(task).ExecuteCommandAsync();
|
||||
if (items.Count > 0) await _db.Insertable(items).ExecuteCommandAsync();
|
||||
});
|
||||
if (!transaction.IsSuccess)
|
||||
throw transaction.ErrorException ?? new InvalidOperationException("创建目录修复任务失败。");
|
||||
return await GetAsync(task.Id);
|
||||
}
|
||||
|
||||
public async Task<OpenListDirectoryRepairTaskDetail> GetAsync(string id)
|
||||
{
|
||||
var task = await RequireTaskAsync(id);
|
||||
var token = task.Status == OpenListDirectoryRepairStatus.AwaitingConfirmation
|
||||
? await BuildConfirmationTokenAsync(task) : null;
|
||||
return new OpenListDirectoryRepairTaskDetail
|
||||
{
|
||||
Task = task,
|
||||
StatusText = StatusText(task.Status),
|
||||
ConfirmationToken = token,
|
||||
CanConfirmCleanup = task.Status == OpenListDirectoryRepairStatus.AwaitingConfirmation && task.EmptyCount > 0,
|
||||
CanCancel = task.Status is OpenListDirectoryRepairStatus.Queued or OpenListDirectoryRepairStatus.Scanning
|
||||
or OpenListDirectoryRepairStatus.AwaitingConfirmation or OpenListDirectoryRepairStatus.Cleaning,
|
||||
CanResume = task.Status is OpenListDirectoryRepairStatus.Cancelled or OpenListDirectoryRepairStatus.Paused,
|
||||
CanRetryFailed = task.FailedCount > 0 && task.Status is (OpenListDirectoryRepairStatus.PartiallyFailed
|
||||
or OpenListDirectoryRepairStatus.Cancelled or OpenListDirectoryRepairStatus.Paused)
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<OpenListDirectoryRepairTaskDetail> GetLatestAsync()
|
||||
{
|
||||
var task = await _db.Queryable<OpenListDirectoryRepairTask>()
|
||||
.OrderByDescending(x => x.CreatedAt).FirstAsync();
|
||||
return task == null ? null : await GetAsync(task.Id);
|
||||
}
|
||||
|
||||
public async Task<OpenListDirectoryRepairItemPage> GetItemsAsync(
|
||||
string id,
|
||||
OpenListDirectoryRepairItemPageRequest request)
|
||||
{
|
||||
_ = await RequireTaskAsync(id);
|
||||
request ??= new OpenListDirectoryRepairItemPageRequest();
|
||||
var page = Math.Max(1, request.PageIndex);
|
||||
var size = Math.Clamp(request.PageSize, 1, 100);
|
||||
var query = _db.Queryable<OpenListDirectoryRepairItem>().Where(x => x.TaskId == id)
|
||||
.WhereIF(request.Status.HasValue, x => x.Status == request.Status.Value)
|
||||
.WhereIF(!string.IsNullOrWhiteSpace(request.Keyword), x =>
|
||||
x.DirectoryName.Contains(request.Keyword) || x.ErrorMessage.Contains(request.Keyword));
|
||||
return new OpenListDirectoryRepairItemPage
|
||||
{
|
||||
TotalCount = await query.CountAsync(),
|
||||
Items = await query.OrderBy(x => x.CreatedAt).Skip((page - 1) * size).Take(size).ToListAsync()
|
||||
};
|
||||
}
|
||||
|
||||
public async Task ConfirmCleanupAsync(
|
||||
string id,
|
||||
ConfirmOpenListDirectoryRepairRequest request)
|
||||
{
|
||||
var task = await RequireTaskAsync(id);
|
||||
if (task.Status != OpenListDirectoryRepairStatus.AwaitingConfirmation)
|
||||
throw new InvalidOperationException("只有扫描完成并等待确认的任务可以开始清理。");
|
||||
await EnsureFingerprintAsync(task);
|
||||
var token = await BuildConfirmationTokenAsync(task);
|
||||
if (string.IsNullOrWhiteSpace(request?.ConfirmationToken)
|
||||
|| !CryptographicOperations.FixedTimeEquals(
|
||||
Encoding.UTF8.GetBytes(token), Encoding.UTF8.GetBytes(request.ConfirmationToken)))
|
||||
throw new InvalidOperationException("目录状态已变化,请刷新任务详情后重新确认。");
|
||||
if (task.EmptyCount <= 0) throw new InvalidOperationException("没有已确认为空的目录可以清理。");
|
||||
task.Status = OpenListDirectoryRepairStatus.Cleaning;
|
||||
task.ErrorMessage = null;
|
||||
task.CompletedAt = null;
|
||||
task.UpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(task).ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
public async Task CancelAsync(string id)
|
||||
{
|
||||
var task = await RequireTaskAsync(id);
|
||||
if (task.Status is not (OpenListDirectoryRepairStatus.Queued or OpenListDirectoryRepairStatus.Scanning
|
||||
or OpenListDirectoryRepairStatus.AwaitingConfirmation or OpenListDirectoryRepairStatus.Cleaning))
|
||||
throw new InvalidOperationException("该任务当前不能取消。");
|
||||
task.Status = OpenListDirectoryRepairStatus.Cancelled;
|
||||
task.CurrentDirectory = null;
|
||||
task.CompletedAt = DateTime.Now;
|
||||
task.UpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(task).ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
public async Task ResumeAsync(string id)
|
||||
{
|
||||
var task = await RequireTaskAsync(id);
|
||||
if (task.Status is not (OpenListDirectoryRepairStatus.Cancelled or OpenListDirectoryRepairStatus.Paused))
|
||||
throw new InvalidOperationException("只有已取消或因配置变化暂停的任务可以恢复。");
|
||||
await EnsureFingerprintAsync(task);
|
||||
await _db.Updateable<OpenListDirectoryRepairItem>()
|
||||
.SetColumns(x => new OpenListDirectoryRepairItem
|
||||
{
|
||||
Status = OpenListDirectoryRepairItemStatus.Pending,
|
||||
ErrorMessage = null,
|
||||
CompletedAt = null,
|
||||
UpdatedAt = DateTime.Now
|
||||
})
|
||||
.Where(x => x.TaskId == id && (x.Status == OpenListDirectoryRepairItemStatus.Inspecting
|
||||
|| x.Status == OpenListDirectoryRepairItemStatus.Deleting)).ExecuteCommandAsync();
|
||||
await RefreshCountsAsync(id);
|
||||
task = await RequireTaskAsync(id);
|
||||
task.Status = task.EmptyCount > 0 && task.PendingCount == 0
|
||||
? OpenListDirectoryRepairStatus.AwaitingConfirmation
|
||||
: OpenListDirectoryRepairStatus.Queued;
|
||||
task.CurrentDirectory = null;
|
||||
task.CompletedAt = null;
|
||||
task.ErrorMessage = null;
|
||||
task.UpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(task).ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
public async Task RetryFailedAsync(string id)
|
||||
{
|
||||
var task = await RequireTaskAsync(id);
|
||||
if (task.Status is not (OpenListDirectoryRepairStatus.PartiallyFailed
|
||||
or OpenListDirectoryRepairStatus.Cancelled or OpenListDirectoryRepairStatus.Paused))
|
||||
throw new InvalidOperationException("该任务当前不能重试失败项。");
|
||||
await EnsureFingerprintAsync(task);
|
||||
var changed = await _db.Updateable<OpenListDirectoryRepairItem>()
|
||||
.SetColumns(x => new OpenListDirectoryRepairItem
|
||||
{
|
||||
Status = OpenListDirectoryRepairItemStatus.Pending,
|
||||
ErrorMessage = null,
|
||||
CompletedAt = null,
|
||||
UpdatedAt = DateTime.Now
|
||||
})
|
||||
.Where(x => x.TaskId == id && x.Status == OpenListDirectoryRepairItemStatus.Failed)
|
||||
.ExecuteCommandAsync();
|
||||
if (changed == 0) throw new InvalidOperationException("没有可重试的失败目录。");
|
||||
task.Status = OpenListDirectoryRepairStatus.Queued;
|
||||
task.CompletedAt = null;
|
||||
task.ErrorMessage = null;
|
||||
task.UpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(task).ExecuteCommandAsync();
|
||||
await RefreshCountsAsync(id);
|
||||
}
|
||||
|
||||
public async Task RefreshCountsAsync(string id)
|
||||
{
|
||||
var items = await _db.Queryable<OpenListDirectoryRepairItem>().Where(x => x.TaskId == id).ToListAsync();
|
||||
var pending = items.Count(x => x.Status is OpenListDirectoryRepairItemStatus.Pending
|
||||
or OpenListDirectoryRepairItemStatus.Inspecting or OpenListDirectoryRepairItemStatus.Deleting);
|
||||
await _db.Updateable<OpenListDirectoryRepairTask>()
|
||||
.SetColumns(x => new OpenListDirectoryRepairTask
|
||||
{
|
||||
TotalCount = items.Count,
|
||||
PendingCount = pending,
|
||||
EmptyCount = items.Count(x => x.Status == OpenListDirectoryRepairItemStatus.EmptyConfirmed),
|
||||
DeletedCount = items.Count(x => x.Status == OpenListDirectoryRepairItemStatus.Deleted),
|
||||
SkippedCount = items.Count(x => x.Status == OpenListDirectoryRepairItemStatus.SkippedNonEmpty),
|
||||
FailedCount = items.Count(x => x.Status == OpenListDirectoryRepairItemStatus.Failed),
|
||||
MissingCount = items.Count(x => x.Status == OpenListDirectoryRepairItemStatus.Missing),
|
||||
UpdatedAt = DateTime.Now
|
||||
}).Where(x => x.Id == id).ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
public async Task EnsureFingerprintAsync(OpenListDirectoryRepairTask task)
|
||||
{
|
||||
var settings = await _settingsService.GetAsync();
|
||||
var current = StorageConfigurationFingerprint.Create(settings);
|
||||
if (!string.Equals(current, task.ConfigurationFingerprint, StringComparison.Ordinal))
|
||||
throw new InvalidOperationException("OpenList 配置已变化,请恢复原配置或重新创建目录修复任务。");
|
||||
}
|
||||
|
||||
public async Task<(OpenListSettings Settings, string Password)> GetConnectionAsync()
|
||||
{
|
||||
var settings = await _settingsService.GetAsync();
|
||||
var password = await _settingsService.GetPasswordAsync(settings);
|
||||
if (string.IsNullOrWhiteSpace(password)) throw new InvalidOperationException("OpenList 密码无效,请重新保存配置。");
|
||||
return (settings, password);
|
||||
}
|
||||
|
||||
private async Task<RepairPlan> BuildPlanAsync(string logicalPath, CancellationToken cancellationToken)
|
||||
{
|
||||
var result = new OpenListDirectoryRepairPreflightResult();
|
||||
if ((_common.GetConfig()?.StorageType ?? MediaStorageType.Local) != MediaStorageType.OpenList)
|
||||
{
|
||||
result.Errors.Add("当前存储不是 OpenList,不能执行目录修复。");
|
||||
return new RepairPlan(result);
|
||||
}
|
||||
logicalPath = StoragePath.NormalizeRemote(string.IsNullOrWhiteSpace(logicalPath) ? "/collect/Kk" : logicalPath);
|
||||
if (logicalPath == "/" || logicalPath.Count(x => x == '/') < 2)
|
||||
{
|
||||
result.Errors.Add("目录修复路径必须位于 OpenList 基础目录的子目录中。");
|
||||
return new RepairPlan(result);
|
||||
}
|
||||
|
||||
var (settings, password) = await GetConnectionAsync();
|
||||
result.ConfigurationFingerprint = StorageConfigurationFingerprint.Create(settings);
|
||||
result.RequestedPath = logicalPath;
|
||||
var requestedActual = OpenListTransferService.ToActualPath(settings, logicalPath);
|
||||
var canonicalActual = await _client.ResolveCanonicalObjectPathAsync(settings, password,
|
||||
requestedActual, false, cancellationToken);
|
||||
var info = await _client.TryGetObjectAsync(settings, password, canonicalActual, cancellationToken);
|
||||
if (info is not { IsDirectory: true })
|
||||
{
|
||||
result.Errors.Add($"没有找到可复用的规范目录:{logicalPath}。");
|
||||
return new RepairPlan(result);
|
||||
}
|
||||
var requestedName = Path.GetFileName(logicalPath);
|
||||
var canonicalName = info.Name;
|
||||
if (string.Equals(requestedName, canonicalName, StringComparison.Ordinal)
|
||||
|| !string.Equals(requestedName, canonicalName, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
result.Errors.Add($"路径不存在纯大小写冲突:请求 {requestedName},实际 {canonicalName}。");
|
||||
return new RepairPlan(result);
|
||||
}
|
||||
var actualParent = StoragePath.DirectoryName(canonicalActual);
|
||||
var escaped = Regex.Escape(requestedName);
|
||||
var pattern = $"^{escaped}_\\d{{8}}_\\d{{6}}$";
|
||||
var regex = new Regex(pattern, RegexOptions.CultureInvariant | RegexOptions.Compiled);
|
||||
var listing = await _client.ListDirectoriesAsync(settings, password, actualParent, true, cancellationToken);
|
||||
var candidates = listing.Directories.Where(x => regex.IsMatch(x.Name))
|
||||
.OrderBy(x => x.Name, StringComparer.Ordinal).ToList();
|
||||
result.CanonicalPath = OpenListTransferService.ToLogicalPath(settings, canonicalActual);
|
||||
result.CandidatePattern = pattern;
|
||||
result.CandidateCount = candidates.Count;
|
||||
result.CanStart = candidates.Count > 0;
|
||||
if (candidates.Count == 0) result.Errors.Add("没有找到匹配的时间后缀异常目录。");
|
||||
else result.Warnings.Add($"找到 {candidates.Count} 个候选目录;后台任务会逐个确认为空,非空目录绝不会删除。");
|
||||
return new RepairPlan(result)
|
||||
{
|
||||
ActualParentPath = actualParent,
|
||||
RequestedName = requestedName,
|
||||
Candidates = candidates
|
||||
};
|
||||
}
|
||||
|
||||
private async Task<string> BuildConfirmationTokenAsync(OpenListDirectoryRepairTask task)
|
||||
{
|
||||
var items = await _db.Queryable<OpenListDirectoryRepairItem>().Where(x => x.TaskId == task.Id
|
||||
&& x.Status == OpenListDirectoryRepairItemStatus.EmptyConfirmed)
|
||||
.OrderBy(x => x.Id).ToListAsync();
|
||||
var source = task.Id + "\n" + task.ConfigurationFingerprint + "\n"
|
||||
+ string.Join("\n", items.Select(x => $"{x.Id}|{x.ActualPath}|{x.UpdatedAt.Ticks}|{x.EntryCount}"));
|
||||
return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(source))).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private async Task<OpenListDirectoryRepairTask> RequireTaskAsync(string id) =>
|
||||
await _db.Queryable<OpenListDirectoryRepairTask>().InSingleAsync(id)
|
||||
?? throw new KeyNotFoundException("目录修复任务不存在。");
|
||||
|
||||
private static string StatusText(OpenListDirectoryRepairStatus status) => status switch
|
||||
{
|
||||
OpenListDirectoryRepairStatus.Queued => "等待扫描",
|
||||
OpenListDirectoryRepairStatus.Scanning => "扫描中",
|
||||
OpenListDirectoryRepairStatus.AwaitingConfirmation => "等待确认清理",
|
||||
OpenListDirectoryRepairStatus.Cleaning => "清理中",
|
||||
OpenListDirectoryRepairStatus.Completed => "已完成",
|
||||
OpenListDirectoryRepairStatus.PartiallyFailed => "部分失败",
|
||||
OpenListDirectoryRepairStatus.Cancelled => "已取消",
|
||||
OpenListDirectoryRepairStatus.Paused => "配置变化,已暂停",
|
||||
_ => status.ToString()
|
||||
};
|
||||
|
||||
private sealed class RepairPlan
|
||||
{
|
||||
public RepairPlan(OpenListDirectoryRepairPreflightResult result) => Result = result;
|
||||
public OpenListDirectoryRepairPreflightResult Result { get; }
|
||||
public string ActualParentPath { get; set; }
|
||||
public string RequestedName { get; set; }
|
||||
public List<OpenListDirectoryItemDto> Candidates { get; set; } = new();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,369 @@
|
||||
using dy.net.model.dto;
|
||||
using dy.net.model.entity;
|
||||
using dy.net.storage;
|
||||
using SqlSugar;
|
||||
using MediaStorageType = dy.net.model.dto.StorageType;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
public sealed class OpenListDirectoryRepairWorker : BackgroundService
|
||||
{
|
||||
private const int InspectionBatchSize = 4;
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
private readonly ILogger<OpenListDirectoryRepairWorker> _logger;
|
||||
|
||||
public OpenListDirectoryRepairWorker(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
ILogger<OpenListDirectoryRepairWorker> logger)
|
||||
{
|
||||
_scopeFactory = scopeFactory;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
await RecoverInterruptedAsync();
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!await RunOneAsync(stoppingToken))
|
||||
await Task.Delay(TimeSpan.FromSeconds(2), stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { }
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "OpenList 目录修复后台服务异常");
|
||||
await Task.Delay(TimeSpan.FromSeconds(3), stoppingToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RecoverInterruptedAsync()
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<ISqlSugarClient>();
|
||||
var now = DateTime.Now;
|
||||
await db.Updateable<OpenListDirectoryRepairItem>()
|
||||
.SetColumns(x => new OpenListDirectoryRepairItem
|
||||
{
|
||||
Status = OpenListDirectoryRepairItemStatus.Pending,
|
||||
ErrorMessage = "应用重启,目录将在恢复后重新检查。",
|
||||
UpdatedAt = now
|
||||
})
|
||||
.Where(x => x.Status == OpenListDirectoryRepairItemStatus.Inspecting).ExecuteCommandAsync();
|
||||
await db.Updateable<OpenListDirectoryRepairItem>()
|
||||
.SetColumns(x => new OpenListDirectoryRepairItem
|
||||
{
|
||||
Status = OpenListDirectoryRepairItemStatus.EmptyConfirmed,
|
||||
ErrorMessage = "应用在删除确认后重启,需再次确认清理。",
|
||||
UpdatedAt = now
|
||||
})
|
||||
.Where(x => x.Status == OpenListDirectoryRepairItemStatus.Deleting).ExecuteCommandAsync();
|
||||
var interrupted = await db.Queryable<OpenListDirectoryRepairTask>().Where(x =>
|
||||
x.Status == OpenListDirectoryRepairStatus.Scanning
|
||||
|| x.Status == OpenListDirectoryRepairStatus.Cleaning).ToListAsync();
|
||||
foreach (var task in interrupted)
|
||||
{
|
||||
task.Status = task.Status == OpenListDirectoryRepairStatus.Cleaning
|
||||
? OpenListDirectoryRepairStatus.AwaitingConfirmation
|
||||
: OpenListDirectoryRepairStatus.Queued;
|
||||
task.CurrentDirectory = null;
|
||||
task.ErrorMessage = task.Status == OpenListDirectoryRepairStatus.AwaitingConfirmation
|
||||
? "应用重启后已停止删除,请重新确认。"
|
||||
: "应用重启后已恢复扫描。";
|
||||
task.UpdatedAt = now;
|
||||
await db.Updateable(task).ExecuteCommandAsync();
|
||||
await scope.ServiceProvider.GetRequiredService<OpenListDirectoryRepairService>()
|
||||
.RefreshCountsAsync(task.Id);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<bool> RunOneAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<ISqlSugarClient>();
|
||||
var service = scope.ServiceProvider.GetRequiredService<OpenListDirectoryRepairService>();
|
||||
var task = await db.Queryable<OpenListDirectoryRepairTask>().Where(x =>
|
||||
x.Status == OpenListDirectoryRepairStatus.Queued
|
||||
|| x.Status == OpenListDirectoryRepairStatus.Scanning
|
||||
|| x.Status == OpenListDirectoryRepairStatus.Cleaning)
|
||||
.OrderBy(x => x.CreatedAt).FirstAsync();
|
||||
if (task == null) return false;
|
||||
try
|
||||
{
|
||||
await service.EnsureFingerprintAsync(task);
|
||||
}
|
||||
catch (InvalidOperationException ex)
|
||||
{
|
||||
task.Status = OpenListDirectoryRepairStatus.Paused;
|
||||
task.ErrorMessage = ex.Message;
|
||||
task.CurrentDirectory = null;
|
||||
task.UpdatedAt = DateTime.Now;
|
||||
await db.Updateable(task).ExecuteCommandAsync();
|
||||
return true;
|
||||
}
|
||||
|
||||
if (task.Status == OpenListDirectoryRepairStatus.Queued)
|
||||
{
|
||||
task.Status = OpenListDirectoryRepairStatus.Scanning;
|
||||
task.StartedAt ??= DateTime.Now;
|
||||
task.CompletedAt = null;
|
||||
task.UpdatedAt = DateTime.Now;
|
||||
await db.Updateable(task).ExecuteCommandAsync();
|
||||
}
|
||||
if (task.Status == OpenListDirectoryRepairStatus.Scanning)
|
||||
return await InspectOneAsync(scope.ServiceProvider, task, cancellationToken);
|
||||
return await DeleteOneAsync(scope.ServiceProvider, task, cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task<bool> InspectOneAsync(
|
||||
IServiceProvider services,
|
||||
OpenListDirectoryRepairTask task,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var db = services.GetRequiredService<ISqlSugarClient>();
|
||||
var service = services.GetRequiredService<OpenListDirectoryRepairService>();
|
||||
var client = services.GetRequiredService<OpenListClient>();
|
||||
var items = await db.Queryable<OpenListDirectoryRepairItem>().Where(x =>
|
||||
x.TaskId == task.Id && x.Status == OpenListDirectoryRepairItemStatus.Pending)
|
||||
.OrderBy(x => x.CreatedAt).Take(InspectionBatchSize).ToListAsync();
|
||||
if (items.Count == 0)
|
||||
{
|
||||
await CompleteScanAsync(services, task.Id);
|
||||
return true;
|
||||
}
|
||||
|
||||
var startedAt = DateTime.Now;
|
||||
foreach (var item in items)
|
||||
{
|
||||
item.Status = OpenListDirectoryRepairItemStatus.Inspecting;
|
||||
item.Attempts++;
|
||||
item.ErrorMessage = null;
|
||||
item.UpdatedAt = startedAt;
|
||||
}
|
||||
task.CurrentDirectory = items[0].ActualPath;
|
||||
task.UpdatedAt = startedAt;
|
||||
await db.Updateable(items).ExecuteCommandAsync();
|
||||
await db.Updateable(task).ExecuteCommandAsync();
|
||||
var (settings, password) = await service.GetConnectionAsync();
|
||||
await Task.WhenAll(items.Select(async item =>
|
||||
{
|
||||
try
|
||||
{
|
||||
var inspection = await client.InspectKnownDirectoryAsync(
|
||||
settings, password, item.ActualPath, true, cancellationToken);
|
||||
item.EntryCount = inspection.EntryCount;
|
||||
item.Status = !inspection.Exists
|
||||
? OpenListDirectoryRepairItemStatus.Missing
|
||||
: inspection.EntryCount == 0
|
||||
? OpenListDirectoryRepairItemStatus.EmptyConfirmed
|
||||
: OpenListDirectoryRepairItemStatus.SkippedNonEmpty;
|
||||
item.ErrorMessage = inspection.Exists && inspection.EntryCount > 0
|
||||
? $"目录包含 {inspection.EntryCount} 个对象,已安全跳过。" : null;
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
|
||||
catch (Exception ex)
|
||||
{
|
||||
item.Status = OpenListDirectoryRepairItemStatus.Failed;
|
||||
item.ErrorMessage = ex.GetBaseException().Message;
|
||||
}
|
||||
item.CompletedAt = DateTime.Now;
|
||||
item.UpdatedAt = item.CompletedAt.Value;
|
||||
}));
|
||||
await db.Updateable(items).ExecuteCommandAsync();
|
||||
await service.RefreshCountsAsync(task.Id);
|
||||
return true;
|
||||
}
|
||||
|
||||
private static async Task CompleteScanAsync(IServiceProvider services, string taskId)
|
||||
{
|
||||
var db = services.GetRequiredService<ISqlSugarClient>();
|
||||
var service = services.GetRequiredService<OpenListDirectoryRepairService>();
|
||||
await service.RefreshCountsAsync(taskId);
|
||||
var task = await db.Queryable<OpenListDirectoryRepairTask>().InSingleAsync(taskId);
|
||||
if (task?.Status != OpenListDirectoryRepairStatus.Scanning) return;
|
||||
task.Status = task.EmptyCount > 0
|
||||
? OpenListDirectoryRepairStatus.AwaitingConfirmation
|
||||
: task.FailedCount > 0
|
||||
? OpenListDirectoryRepairStatus.PartiallyFailed
|
||||
: OpenListDirectoryRepairStatus.Completed;
|
||||
task.CurrentDirectory = null;
|
||||
task.ScanCompletedAt = DateTime.Now;
|
||||
task.CompletedAt = task.Status is OpenListDirectoryRepairStatus.Completed
|
||||
or OpenListDirectoryRepairStatus.PartiallyFailed ? DateTime.Now : null;
|
||||
task.ErrorMessage = task.FailedCount > 0 ? $"{task.FailedCount} 个目录检查失败,未执行删除。" : null;
|
||||
task.UpdatedAt = DateTime.Now;
|
||||
await db.Updateable(task).ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
private static async Task<bool> DeleteOneAsync(
|
||||
IServiceProvider services,
|
||||
OpenListDirectoryRepairTask task,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var db = services.GetRequiredService<ISqlSugarClient>();
|
||||
var service = services.GetRequiredService<OpenListDirectoryRepairService>();
|
||||
var client = services.GetRequiredService<OpenListClient>();
|
||||
var item = await db.Queryable<OpenListDirectoryRepairItem>().Where(x =>
|
||||
x.TaskId == task.Id && x.Status == OpenListDirectoryRepairItemStatus.EmptyConfirmed)
|
||||
.OrderBy(x => x.CreatedAt).FirstAsync();
|
||||
if (item == null)
|
||||
{
|
||||
await CompleteCleanupAsync(services, task.Id, cancellationToken);
|
||||
return true;
|
||||
}
|
||||
|
||||
item.Status = OpenListDirectoryRepairItemStatus.Deleting;
|
||||
item.Attempts++;
|
||||
item.ErrorMessage = null;
|
||||
item.UpdatedAt = DateTime.Now;
|
||||
task.CurrentDirectory = item.ActualPath;
|
||||
task.UpdatedAt = item.UpdatedAt;
|
||||
await db.Updateable(item).ExecuteCommandAsync();
|
||||
await db.Updateable(task).ExecuteCommandAsync();
|
||||
try
|
||||
{
|
||||
var (settings, password) = await service.GetConnectionAsync();
|
||||
var before = await client.InspectKnownDirectoryAsync(settings, password, item.ActualPath, true, cancellationToken);
|
||||
if (!before.Exists)
|
||||
{
|
||||
item.Status = OpenListDirectoryRepairItemStatus.Missing;
|
||||
}
|
||||
else if (before.EntryCount > 0)
|
||||
{
|
||||
item.EntryCount = before.EntryCount;
|
||||
item.Status = OpenListDirectoryRepairItemStatus.SkippedNonEmpty;
|
||||
item.ErrorMessage = $"删除前发现目录已有 {before.EntryCount} 个对象,已安全跳过。";
|
||||
}
|
||||
else
|
||||
{
|
||||
await client.DeleteKnownObjectAsync(settings, password, item.ActualPath, cancellationToken);
|
||||
var after = await client.InspectKnownDirectoryAsync(settings, password, item.ActualPath, true, cancellationToken);
|
||||
if (after.Exists) throw new IOException("OpenList 返回删除成功,但目录仍然存在。");
|
||||
item.Status = OpenListDirectoryRepairItemStatus.Deleted;
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
|
||||
catch (Exception ex)
|
||||
{
|
||||
item.Status = OpenListDirectoryRepairItemStatus.Failed;
|
||||
item.ErrorMessage = ex.GetBaseException().Message;
|
||||
}
|
||||
item.CompletedAt = DateTime.Now;
|
||||
item.UpdatedAt = item.CompletedAt.Value;
|
||||
await db.Updateable(item).ExecuteCommandAsync();
|
||||
await service.RefreshCountsAsync(task.Id);
|
||||
return true;
|
||||
}
|
||||
|
||||
private static async Task CompleteCleanupAsync(
|
||||
IServiceProvider services,
|
||||
string taskId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var db = services.GetRequiredService<ISqlSugarClient>();
|
||||
var service = services.GetRequiredService<OpenListDirectoryRepairService>();
|
||||
await service.RefreshCountsAsync(taskId);
|
||||
var task = await db.Queryable<OpenListDirectoryRepairTask>().InSingleAsync(taskId);
|
||||
if (task?.Status != OpenListDirectoryRepairStatus.Cleaning) return;
|
||||
task.Status = task.FailedCount > 0
|
||||
? OpenListDirectoryRepairStatus.PartiallyFailed
|
||||
: OpenListDirectoryRepairStatus.Completed;
|
||||
task.CurrentDirectory = null;
|
||||
task.CompletedAt = DateTime.Now;
|
||||
task.ErrorMessage = task.FailedCount > 0 ? $"{task.FailedCount} 个目录清理失败,可重试。" : null;
|
||||
task.UpdatedAt = DateTime.Now;
|
||||
await db.Updateable(task).ExecuteCommandAsync();
|
||||
await RecoverAffectedTasksAsync(services, task, cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task RecoverAffectedTasksAsync(
|
||||
IServiceProvider services,
|
||||
OpenListDirectoryRepairTask repair,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var db = services.GetRequiredService<ISqlSugarClient>();
|
||||
var videos = await db.Queryable<DouyinVideo>().Where(x => x.StorageType == MediaStorageType.OpenList
|
||||
&& x.VideoSavePath.StartsWith(repair.RequestedPath + "/")).ToListAsync();
|
||||
var storage = services.GetRequiredService<OpenListMediaStorage>();
|
||||
foreach (var video in videos)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var canonicalMain = ReplacePrefix(video.VideoSavePath, repair.RequestedPath, repair.CanonicalPath);
|
||||
var length = await storage.GetLengthAsync(canonicalMain, cancellationToken);
|
||||
if (!length.HasValue || length <= 0 || video.FileSize > 0 && length.Value != video.FileSize) continue;
|
||||
video.VideoSavePath = canonicalMain;
|
||||
video.VideoCoverSavePath = ReplacePrefix(video.VideoCoverSavePath, repair.RequestedPath, repair.CanonicalPath);
|
||||
video.AuthorAvatar = ReplacePrefix(video.AuthorAvatar, repair.RequestedPath, repair.CanonicalPath);
|
||||
if (!string.IsNullOrWhiteSpace(video.DynamicVideos))
|
||||
video.DynamicVideos = video.DynamicVideos.Replace(repair.RequestedPath + "/",
|
||||
repair.CanonicalPath + "/", StringComparison.Ordinal);
|
||||
await db.Updateable(video).ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
var failedItems = await db.Queryable<VideoDownloadTaskItem>().Where(x =>
|
||||
(x.Stage == VideoTaskItemStage.Failed || x.Stage == VideoTaskItemStage.WaitingForStorage)
|
||||
&& x.RetrySnapshotJson != null
|
||||
&& (x.ErrorType == VideoTaskErrorType.StorageUnavailable
|
||||
|| x.ErrorType == VideoTaskErrorType.IntegrityCheckFailed)).ToListAsync();
|
||||
var taskIds = new HashSet<string>(StringComparer.Ordinal);
|
||||
foreach (var item in failedItems)
|
||||
{
|
||||
if (!(item.TargetPath?.StartsWith(repair.RequestedPath + "/", StringComparison.Ordinal) == true
|
||||
|| item.ErrorMessage?.Contains("Operation canceled", StringComparison.OrdinalIgnoreCase) == true
|
||||
|| item.ErrorMessage?.Contains("超时", StringComparison.Ordinal) == true
|
||||
|| item.ErrorType == VideoTaskErrorType.IntegrityCheckFailed)) continue;
|
||||
item.TargetPath = ReplacePrefix(item.TargetPath, repair.RequestedPath, repair.CanonicalPath);
|
||||
item.RetrySnapshotJson = item.RetrySnapshotJson.Replace(repair.RequestedPath + "/",
|
||||
repair.CanonicalPath + "/", StringComparison.Ordinal);
|
||||
item.Stage = VideoTaskItemStage.Pending;
|
||||
item.ErrorType = VideoTaskErrorType.None;
|
||||
item.ErrorMessage = null;
|
||||
item.CompletedAt = null;
|
||||
item.WorkerManaged = true;
|
||||
item.UpdatedAt = DateTime.Now;
|
||||
taskIds.Add(item.TaskId);
|
||||
}
|
||||
if (failedItems.Count > 0) await db.Updateable(failedItems).ExecuteCommandAsync();
|
||||
foreach (var taskId in taskIds)
|
||||
{
|
||||
await db.Updateable<VideoDownloadTask>()
|
||||
.SetColumns(x => new VideoDownloadTask
|
||||
{
|
||||
Status = VideoTaskStatus.Queued,
|
||||
CompletedAt = null,
|
||||
ErrorMessage = null,
|
||||
UpdatedAt = DateTime.Now
|
||||
}).Where(x => x.Id == taskId).ExecuteCommandAsync();
|
||||
await services.GetRequiredService<VideoTaskService>().RefreshCountsAsync(taskId);
|
||||
}
|
||||
|
||||
try { _ = await services.GetRequiredService<VideoTaskService>().ProbeStorageAsync(cancellationToken); }
|
||||
catch (Exception ex) { Serilog.Log.Warning(ex, "目录修复后 OpenList 健康检测失败"); }
|
||||
try
|
||||
{
|
||||
var migrations = await db.Queryable<StorageMigrationTask>().Where(x =>
|
||||
x.TargetStorageType == MediaStorageType.OpenList
|
||||
&& x.Status == StorageMigrationTaskStatus.PartiallyFailed).ToListAsync();
|
||||
foreach (var migration in migrations)
|
||||
await services.GetRequiredService<StorageMigrationService>().RetryFailedAsync(migration.Id);
|
||||
}
|
||||
catch (Exception ex) { Serilog.Log.Warning(ex, "目录修复后迁移失败项自动重试未能提交"); }
|
||||
try
|
||||
{
|
||||
_ = await services.GetRequiredService<DouyinQuartzJobService>()
|
||||
.TriggerVideoJobsNowAsync(VideoTypeEnum.dy_follows);
|
||||
}
|
||||
catch (Exception ex) { Serilog.Log.Warning(ex, "目录修复后关注同步未能自动提交"); }
|
||||
}
|
||||
|
||||
private static string ReplacePrefix(string value, string oldPrefix, string newPrefix)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value)) return value;
|
||||
return value.StartsWith(oldPrefix + "/", StringComparison.Ordinal)
|
||||
? newPrefix + value[oldPrefix.Length..]
|
||||
: value;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
using dy.net.model.dto;
|
||||
using dy.net.model.entity;
|
||||
using dy.net.extension;
|
||||
using Microsoft.AspNetCore.DataProtection;
|
||||
using SqlSugar;
|
||||
using dy.net.utils;
|
||||
using MediaStorageType = dy.net.model.dto.StorageType;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
public class OpenListSettingsService
|
||||
{
|
||||
private const string SettingsId = "default";
|
||||
private readonly ISqlSugarClient _db;
|
||||
private readonly IDataProtector _protector;
|
||||
private readonly WebDavSettingsService _legacySettings;
|
||||
|
||||
public OpenListSettingsService(
|
||||
ISqlSugarClient db,
|
||||
IDataProtectionProvider provider,
|
||||
WebDavSettingsService legacySettings)
|
||||
{
|
||||
_db = db;
|
||||
_protector = provider.CreateProtector("dysync.openlist.password.v1");
|
||||
_legacySettings = legacySettings;
|
||||
}
|
||||
|
||||
public async Task<OpenListSettings> GetAsync()
|
||||
{
|
||||
var existing = await _db.Queryable<OpenListSettings>().InSingleAsync(SettingsId);
|
||||
if (existing != null)
|
||||
{
|
||||
Normalize(existing);
|
||||
return existing;
|
||||
}
|
||||
|
||||
var legacy = await _legacySettings.GetAsync();
|
||||
var password = await _legacySettings.GetPasswordAsync(legacy);
|
||||
var suggested = new OpenListSettings
|
||||
{
|
||||
Id = SettingsId,
|
||||
Endpoint = NormalizeLegacyEndpoint(legacy.Endpoint),
|
||||
BasePath = string.IsNullOrWhiteSpace(legacy.BasePath) ? "/dysync" : legacy.BasePath,
|
||||
LocalStagingPath = GetDefaultLocalStagingPath(),
|
||||
SourcePath = "/dysync-staging",
|
||||
UserName = legacy.UserName,
|
||||
ProtectedPassword = string.IsNullOrWhiteSpace(password) ? null : _protector.Protect(password)
|
||||
};
|
||||
Normalize(suggested);
|
||||
return suggested;
|
||||
}
|
||||
|
||||
public async Task<string> GetPasswordAsync(OpenListSettings settings = null)
|
||||
{
|
||||
settings ??= await GetAsync();
|
||||
if (string.IsNullOrWhiteSpace(settings.ProtectedPassword)) return string.Empty;
|
||||
try
|
||||
{
|
||||
return _protector.Unprotect(settings.ProtectedPassword);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Serilog.Log.Error(ex, "OpenList 密码解密失败,请重新输入密码");
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<OpenListSettings> BuildCandidateAsync(OpenListTestRequest request)
|
||||
{
|
||||
request ??= new OpenListTestRequest();
|
||||
var existing = await GetAsync();
|
||||
var password = string.IsNullOrWhiteSpace(request.Password)
|
||||
? await GetPasswordAsync(existing)
|
||||
: request.Password;
|
||||
|
||||
var candidate = new OpenListSettings
|
||||
{
|
||||
Id = SettingsId,
|
||||
Endpoint = string.IsNullOrWhiteSpace(request.Endpoint) ? existing.Endpoint : request.Endpoint.Trim(),
|
||||
BasePath = string.IsNullOrWhiteSpace(request.BasePath) ? existing.BasePath : request.BasePath.Trim(),
|
||||
LocalStagingPath = string.IsNullOrWhiteSpace(request.LocalStagingPath)
|
||||
? existing.LocalStagingPath : request.LocalStagingPath.Trim(),
|
||||
SourcePath = string.IsNullOrWhiteSpace(request.SourcePath) ? existing.SourcePath : request.SourcePath.Trim(),
|
||||
UserName = string.IsNullOrWhiteSpace(request.UserName) ? existing.UserName : request.UserName.Trim(),
|
||||
ProtectedPassword = string.IsNullOrWhiteSpace(password) ? null : _protector.Protect(password)
|
||||
};
|
||||
Normalize(candidate);
|
||||
return candidate;
|
||||
}
|
||||
|
||||
public async Task SaveAsync(OpenListSettings settings, bool tested, string testMessage)
|
||||
{
|
||||
settings.Id = SettingsId;
|
||||
Normalize(settings);
|
||||
settings.LastTestedAt = tested ? DateTime.Now : null;
|
||||
settings.LastTestMessage = testMessage;
|
||||
await _db.Storageable(settings).ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
public async Task<OpenListSettingsDto> ToDtoAsync(MediaStorageType storageType)
|
||||
{
|
||||
var settings = await GetAsync();
|
||||
return new OpenListSettingsDto
|
||||
{
|
||||
StorageType = storageType,
|
||||
Endpoint = settings.Endpoint,
|
||||
BasePath = settings.BasePath,
|
||||
LocalStagingPath = settings.LocalStagingPath,
|
||||
SourcePath = settings.SourcePath,
|
||||
UserName = settings.UserName,
|
||||
HasPassword = !string.IsNullOrWhiteSpace(settings.ProtectedPassword),
|
||||
LastTestedAt = settings.LastTestedAt,
|
||||
LastTestMessage = settings.LastTestMessage
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<bool> HasSavedSettingsAsync() =>
|
||||
await _db.Queryable<OpenListSettings>().Where(x => x.Id == SettingsId).AnyAsync();
|
||||
|
||||
public static string GetDefaultLocalStagingPath()
|
||||
{
|
||||
var dataRoot = !string.IsNullOrWhiteSpace(ServiceExtension.FnDataFolder)
|
||||
? Path.GetDirectoryName(Path.GetFullPath(ServiceExtension.FnDataFolder))
|
||||
: null;
|
||||
return Path.Combine(dataRoot ?? Environment.CurrentDirectory, "openlist-staging");
|
||||
}
|
||||
|
||||
private static string NormalizeLegacyEndpoint(string endpoint)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(endpoint)) return endpoint;
|
||||
var value = endpoint.Trim().TrimEnd('/');
|
||||
var dav = value.LastIndexOf("/dav", StringComparison.OrdinalIgnoreCase);
|
||||
return dav >= 0 && dav + 4 == value.Length ? value[..dav] : value;
|
||||
}
|
||||
|
||||
private static void Normalize(OpenListSettings settings)
|
||||
{
|
||||
settings.Endpoint = NormalizeLegacyEndpoint(settings.Endpoint);
|
||||
if (string.IsNullOrWhiteSpace(settings.BasePath))
|
||||
settings.BasePath = string.IsNullOrWhiteSpace(settings.DestinationPath) ? "/dysync" : settings.DestinationPath;
|
||||
if (string.IsNullOrWhiteSpace(settings.LocalStagingPath))
|
||||
settings.LocalStagingPath = GetDefaultLocalStagingPath();
|
||||
if (string.IsNullOrWhiteSpace(settings.SourcePath)) settings.SourcePath = "/dysync-staging";
|
||||
settings.DestinationPath = settings.BasePath;
|
||||
}
|
||||
|
||||
public async Task InitializeCookiePathsAsync()
|
||||
{
|
||||
var cookies = await _db.Queryable<DouyinCookie>().ToListAsync();
|
||||
foreach (var cookie in cookies)
|
||||
{
|
||||
ApplyDefaultCookiePaths(cookie);
|
||||
}
|
||||
if (cookies.Count > 0) await _db.Updateable(cookies).ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
public void ApplyDefaultCookiePaths(DouyinCookie cookie)
|
||||
{
|
||||
if (cookie == null) return;
|
||||
var root = DouyinFileNameHelper.SanitizeLinuxFileName(cookie.UserName, cookie.Id, true);
|
||||
if (string.IsNullOrWhiteSpace(cookie.WebDavCollectPath)) cookie.WebDavCollectPath = $"/{root}/collect";
|
||||
if (string.IsNullOrWhiteSpace(cookie.WebDavFavoritePath)) cookie.WebDavFavoritePath = $"/{root}/favorite";
|
||||
if (string.IsNullOrWhiteSpace(cookie.WebDavFollowPath)) cookie.WebDavFollowPath = $"/{root}/follow";
|
||||
if (string.IsNullOrWhiteSpace(cookie.WebDavMixPath)) cookie.WebDavMixPath = $"/{root}/mix";
|
||||
if (string.IsNullOrWhiteSpace(cookie.WebDavSeriesPath)) cookie.WebDavSeriesPath = $"/{root}/series";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,547 @@
|
||||
using dy.net.model.dto;
|
||||
using dy.net.model.entity;
|
||||
using dy.net.storage;
|
||||
using SqlSugar;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
/// <summary>
|
||||
/// 将本地中转文件交给 OpenList 服务端复制。状态与外部任务 ID 持久化,应用重启后可继续。
|
||||
/// </summary>
|
||||
public sealed class OpenListTransferService
|
||||
{
|
||||
private const int MaxAttempts = 6;
|
||||
private static readonly TimeSpan LeaseDuration = TimeSpan.FromSeconds(30);
|
||||
private static readonly TimeSpan StallTimeout = TimeSpan.FromMinutes(60);
|
||||
private static readonly TimeSpan ExternalTimeout = TimeSpan.FromHours(24);
|
||||
private static readonly TimeSpan[] RetryDelays =
|
||||
{
|
||||
TimeSpan.FromMinutes(1),
|
||||
TimeSpan.FromMinutes(5),
|
||||
TimeSpan.FromMinutes(15),
|
||||
TimeSpan.FromHours(1),
|
||||
TimeSpan.FromHours(6)
|
||||
};
|
||||
|
||||
private readonly ISqlSugarClient _db;
|
||||
private readonly OpenListSettingsService _settingsService;
|
||||
private readonly OpenListClient _client;
|
||||
|
||||
public OpenListTransferService(
|
||||
ISqlSugarClient db,
|
||||
OpenListSettingsService settingsService,
|
||||
OpenListClient client)
|
||||
{
|
||||
_db = db;
|
||||
_settingsService = settingsService;
|
||||
_client = client;
|
||||
}
|
||||
|
||||
public async Task<long> TransferAsync(
|
||||
string logicalTargetPath,
|
||||
Stream source,
|
||||
long? declaredLength,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var settings = await _settingsService.GetAsync();
|
||||
ValidateSettings(settings);
|
||||
logicalTargetPath = StoragePath.NormalizeRemote(logicalTargetPath);
|
||||
if (logicalTargetPath == "/") throw new ArgumentException("OpenList 文件路径不能为空。", nameof(logicalTargetPath));
|
||||
var password = await _settingsService.GetPasswordAsync(settings);
|
||||
if (string.IsNullOrWhiteSpace(password)) throw new InvalidOperationException("OpenList 密码无效,请重新保存配置。");
|
||||
var fingerprint = StorageConfigurationFingerprint.Create(settings);
|
||||
var requestedLogicalTarget = logicalTargetPath;
|
||||
var actualTarget = await _client.ResolveCanonicalObjectPathAsync(settings, password,
|
||||
ToActualPath(settings, logicalTargetPath), false, cancellationToken);
|
||||
logicalTargetPath = ToLogicalPath(settings, actualTarget);
|
||||
|
||||
if (declaredLength is > 0)
|
||||
{
|
||||
var target = await _client.TryGetObjectAsync(settings, password, actualTarget, cancellationToken);
|
||||
if (target is { IsDirectory: false } && target.Size == declaredLength.Value) return target.Size;
|
||||
}
|
||||
|
||||
var reusable = await _db.Queryable<OpenListTransferJob>().Where(x =>
|
||||
x.ConfigurationFingerprint == fingerprint
|
||||
&& (x.LogicalTargetPath == logicalTargetPath || x.LogicalTargetPath == requestedLogicalTarget)
|
||||
&& x.Status != OpenListTransferStatus.Succeeded
|
||||
&& x.Status != OpenListTransferStatus.Failed
|
||||
&& x.Status != OpenListTransferStatus.Cancelled)
|
||||
.OrderByDescending(x => x.CreatedAt).FirstAsync();
|
||||
var job = reusable ?? CreateJob(settings, fingerprint, logicalTargetPath, actualTarget);
|
||||
if (reusable != null && (job.LogicalTargetPath != logicalTargetPath || job.ActualTargetPath != actualTarget))
|
||||
{
|
||||
job.LogicalTargetPath = logicalTargetPath;
|
||||
job.ActualTargetPath = actualTarget;
|
||||
job.UpdatedAt = DateTime.Now;
|
||||
}
|
||||
|
||||
if (!File.Exists(job.LocalSourcePath)
|
||||
|| new FileInfo(job.LocalSourcePath).Length <= 0
|
||||
|| declaredLength is > 0 && new FileInfo(job.LocalSourcePath).Length != declaredLength.Value)
|
||||
{
|
||||
await StageSourceAsync(job, source, declaredLength, cancellationToken);
|
||||
}
|
||||
else if (job.ExpectedLength <= 0)
|
||||
{
|
||||
job.ExpectedLength = new FileInfo(job.LocalSourcePath).Length;
|
||||
}
|
||||
|
||||
if (reusable == null) await _db.Insertable(job).ExecuteCommandAsync();
|
||||
else
|
||||
{
|
||||
job.Status = OpenListTransferStatus.Queued;
|
||||
job.NextAttemptAt = null;
|
||||
job.ErrorMessage = null;
|
||||
job.UpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(job).ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
await RunUntilSettledAsync(job.Id, cancellationToken);
|
||||
var completed = await _db.Queryable<OpenListTransferJob>().InSingleAsync(job.Id);
|
||||
if (completed?.Status != OpenListTransferStatus.Succeeded)
|
||||
throw new IOException(completed?.ErrorMessage ?? "OpenList 服务端复制未完成。");
|
||||
return completed.ExpectedLength;
|
||||
}
|
||||
|
||||
public async Task<bool> RecoverOneAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var now = DateTime.Now;
|
||||
var job = await _db.Queryable<OpenListTransferJob>().Where(x =>
|
||||
(x.Status == OpenListTransferStatus.Queued
|
||||
|| x.Status == OpenListTransferStatus.WaitingForSource
|
||||
|| x.Status == OpenListTransferStatus.Copying
|
||||
|| x.Status == OpenListTransferStatus.Verifying
|
||||
|| x.Status == OpenListTransferStatus.Promoting
|
||||
|| x.Status == OpenListTransferStatus.WaitingRetry && x.NextAttemptAt <= now)
|
||||
&& (x.LeaseUntil == null || x.LeaseUntil < now))
|
||||
.OrderBy(x => x.CreatedAt).FirstAsync();
|
||||
if (job == null) return false;
|
||||
await RunUntilSettledAsync(job.Id, cancellationToken, throwOnScheduledRetry: false);
|
||||
return true;
|
||||
}
|
||||
|
||||
public static string ToActualPath(OpenListSettings settings, string logicalPath) =>
|
||||
StoragePath.CombineRemote(settings.BasePath, StoragePath.NormalizeRemote(logicalPath));
|
||||
|
||||
public static string ToLogicalPath(OpenListSettings settings, string actualPath)
|
||||
{
|
||||
var actualSegments = StoragePath.NormalizeRemote(actualPath)
|
||||
.Split('/', StringSplitOptions.RemoveEmptyEntries);
|
||||
var baseSegments = StoragePath.NormalizeRemote(settings.BasePath)
|
||||
.Split('/', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (actualSegments.Length < baseSegments.Length)
|
||||
throw new InvalidOperationException($"OpenList 实际路径 '{actualPath}' 不在基础目录 '{settings.BasePath}' 下。");
|
||||
for (var i = 0; i < baseSegments.Length; i++)
|
||||
if (!string.Equals(actualSegments[i], baseSegments[i], StringComparison.OrdinalIgnoreCase))
|
||||
throw new InvalidOperationException($"OpenList 实际路径 '{actualPath}' 不在基础目录 '{settings.BasePath}' 下。");
|
||||
return StoragePath.NormalizeRemote(string.Join('/', actualSegments.Skip(baseSegments.Length)));
|
||||
}
|
||||
|
||||
public static void ValidateSettings(OpenListSettings settings)
|
||||
{
|
||||
if (settings == null) throw new InvalidOperationException("尚未配置 OpenList。");
|
||||
_ = OpenListClient.NormalizeBaseUrl(settings.Endpoint);
|
||||
if (string.IsNullOrWhiteSpace(settings.UserName)) throw new InvalidOperationException("OpenList 用户名不能为空。");
|
||||
if (string.IsNullOrWhiteSpace(settings.LocalStagingPath) || !Path.IsPathRooted(settings.LocalStagingPath))
|
||||
throw new InvalidOperationException("本地中转目录必须是绝对路径。");
|
||||
if (string.IsNullOrWhiteSpace(settings.SourcePath)) throw new InvalidOperationException("OpenList 源挂载目录不能为空。");
|
||||
_ = StoragePath.NormalizeRemote(settings.SourcePath);
|
||||
_ = StoragePath.NormalizeRemote(settings.BasePath);
|
||||
}
|
||||
|
||||
private OpenListTransferJob CreateJob(
|
||||
OpenListSettings settings,
|
||||
string fingerprint,
|
||||
string logicalTarget,
|
||||
string actualTarget)
|
||||
{
|
||||
var id = Guid.NewGuid().ToString("N");
|
||||
var fileName = GetFileName(logicalTarget);
|
||||
var localDirectory = Path.Combine(Path.GetFullPath(settings.LocalStagingPath), id);
|
||||
var now = DateTime.Now;
|
||||
return new OpenListTransferJob
|
||||
{
|
||||
Id = id,
|
||||
Status = OpenListTransferStatus.Queued,
|
||||
ConfigurationFingerprint = fingerprint,
|
||||
LogicalTargetPath = logicalTarget,
|
||||
ActualTargetPath = actualTarget,
|
||||
LocalSourcePath = Path.Combine(localDirectory, fileName),
|
||||
OpenListSourcePath = StoragePath.CombineRemote(settings.SourcePath, id, fileName),
|
||||
StagedTargetPath = StoragePath.CombineRemote(settings.BasePath, $".dysync-staging-{id}", fileName),
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now
|
||||
};
|
||||
}
|
||||
|
||||
private async Task StageSourceAsync(
|
||||
OpenListTransferJob job,
|
||||
Stream source,
|
||||
long? declaredLength,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var directory = Path.GetDirectoryName(job.LocalSourcePath)
|
||||
?? throw new InvalidOperationException("OpenList 本地中转目录无效。");
|
||||
Directory.CreateDirectory(directory);
|
||||
var temporary = job.LocalSourcePath + ".writing-" + Guid.NewGuid().ToString("N");
|
||||
try
|
||||
{
|
||||
await using (var output = new FileStream(temporary, FileMode.CreateNew, FileAccess.Write,
|
||||
FileShare.None, 81920, FileOptions.Asynchronous | FileOptions.SequentialScan))
|
||||
await source.CopyToAsync(output, 81920, cancellationToken);
|
||||
var length = new FileInfo(temporary).Length;
|
||||
if (length <= 0) throw new MediaIntegrityException("媒体来源写入 OpenList 中转文件后为空。");
|
||||
if (declaredLength is > 0 && length != declaredLength.Value)
|
||||
throw new MediaIntegrityException($"下载长度不一致:期望 {declaredLength},实际 {length}。");
|
||||
File.Move(temporary, job.LocalSourcePath, true);
|
||||
job.ExpectedLength = length;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(temporary)) File.Delete(temporary);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RunUntilSettledAsync(
|
||||
string jobId,
|
||||
CancellationToken cancellationToken,
|
||||
bool throwOnScheduledRetry = true)
|
||||
{
|
||||
var leaseOwner = Guid.NewGuid().ToString("N");
|
||||
if (!await TryAcquireLeaseAsync(jobId, leaseOwner))
|
||||
{
|
||||
await WaitForOtherProcessorAsync(jobId, cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
var job = await _db.Queryable<OpenListTransferJob>().InSingleAsync(jobId)
|
||||
?? throw new InvalidOperationException("OpenList 传输任务不存在。");
|
||||
if (job.Status == OpenListTransferStatus.Succeeded) return;
|
||||
if (job.Status is OpenListTransferStatus.Failed or OpenListTransferStatus.Cancelled)
|
||||
throw new IOException(job.ErrorMessage ?? "OpenList 传输任务失败。");
|
||||
if (job.Status == OpenListTransferStatus.WaitingRetry && job.NextAttemptAt > DateTime.Now)
|
||||
{
|
||||
if (throwOnScheduledRetry) throw new IOException(job.ErrorMessage ?? "OpenList 传输等待重试。");
|
||||
return;
|
||||
}
|
||||
|
||||
await RenewLeaseAsync(job, leaseOwner);
|
||||
try
|
||||
{
|
||||
var settled = await ProcessStepAsync(job, cancellationToken);
|
||||
if (settled) return;
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
|
||||
catch (Exception ex)
|
||||
{
|
||||
await ScheduleRetryOrFailAsync(job, ex.GetBaseException().Message);
|
||||
if (throwOnScheduledRetry) throw;
|
||||
return;
|
||||
}
|
||||
await Task.Delay(TimeSpan.FromSeconds(2), cancellationToken);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
await ReleaseLeaseAsync(jobId, leaseOwner);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<bool> ProcessStepAsync(OpenListTransferJob job, CancellationToken cancellationToken)
|
||||
{
|
||||
var settings = await _settingsService.GetAsync();
|
||||
if (!string.Equals(StorageConfigurationFingerprint.Create(settings), job.ConfigurationFingerprint, StringComparison.Ordinal))
|
||||
throw new InvalidOperationException("OpenList 配置已变化,原传输任务不会写入新目标。");
|
||||
var password = await _settingsService.GetPasswordAsync(settings);
|
||||
var canonicalActual = await _client.ResolveCanonicalObjectPathAsync(settings, password,
|
||||
job.ActualTargetPath, false, cancellationToken);
|
||||
var canonicalLogical = ToLogicalPath(settings, canonicalActual);
|
||||
if (!string.Equals(job.ActualTargetPath, canonicalActual, StringComparison.Ordinal)
|
||||
|| !string.Equals(job.LogicalTargetPath, canonicalLogical, StringComparison.Ordinal))
|
||||
{
|
||||
job.ActualTargetPath = canonicalActual;
|
||||
job.LogicalTargetPath = canonicalLogical;
|
||||
job.UpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(job).ExecuteCommandAsync();
|
||||
}
|
||||
var target = await _client.TryGetObjectAsync(settings, password, job.ActualTargetPath, cancellationToken);
|
||||
if (target is { IsDirectory: false } && target.Size == job.ExpectedLength)
|
||||
{
|
||||
await MarkSucceededAsync(job, settings, password, cancellationToken);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (job.Status is OpenListTransferStatus.Queued or OpenListTransferStatus.WaitingForSource
|
||||
or OpenListTransferStatus.WaitingRetry)
|
||||
{
|
||||
if (!File.Exists(job.LocalSourcePath) || new FileInfo(job.LocalSourcePath).Length != job.ExpectedLength)
|
||||
throw new FileNotFoundException("OpenList 本地中转文件不存在或长度已变化。", job.LocalSourcePath);
|
||||
var source = await _client.TryGetObjectAsync(settings, password, job.OpenListSourcePath, cancellationToken);
|
||||
if (source is not { IsDirectory: false } || source.Size != job.ExpectedLength)
|
||||
{
|
||||
job.Status = OpenListTransferStatus.WaitingForSource;
|
||||
job.ErrorMessage = "OpenList 尚未看到本地中转文件,请检查源挂载目录映射。";
|
||||
job.UpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(job).ExecuteCommandAsync();
|
||||
throw new IOException(job.ErrorMessage);
|
||||
}
|
||||
|
||||
await _client.EnsureDirectoryAsync(settings, password,
|
||||
StoragePath.DirectoryName(job.ActualTargetPath), cancellationToken);
|
||||
await _client.EnsureDirectoryAsync(settings, password,
|
||||
StoragePath.DirectoryName(job.StagedTargetPath), cancellationToken);
|
||||
var staged = await _client.TryGetObjectAsync(settings, password, job.StagedTargetPath, cancellationToken);
|
||||
if (staged is { IsDirectory: false } && staged.Size == job.ExpectedLength)
|
||||
{
|
||||
job.Status = OpenListTransferStatus.Verifying;
|
||||
job.ErrorMessage = null;
|
||||
job.UpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(job).ExecuteCommandAsync();
|
||||
return false;
|
||||
}
|
||||
if (staged != null) await _client.DeleteObjectAsync(settings, password, job.StagedTargetPath, cancellationToken);
|
||||
var copy = await _client.CopyFileAsync(settings, password,
|
||||
job.OpenListSourcePath, job.StagedTargetPath, cancellationToken);
|
||||
job.Attempts++;
|
||||
job.ExternalTaskId = copy.TaskIds.FirstOrDefault();
|
||||
job.ExternalTaskStartedAt = DateTime.Now;
|
||||
job.LastProgressAt = DateTime.Now;
|
||||
job.ExternalProgress = 0;
|
||||
job.Status = string.IsNullOrWhiteSpace(job.ExternalTaskId)
|
||||
? OpenListTransferStatus.Verifying
|
||||
: OpenListTransferStatus.Copying;
|
||||
job.ErrorMessage = null;
|
||||
job.NextAttemptAt = null;
|
||||
job.UpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(job).ExecuteCommandAsync();
|
||||
return false;
|
||||
}
|
||||
|
||||
if (job.Status == OpenListTransferStatus.Copying)
|
||||
{
|
||||
if (job.ExternalTaskStartedAt.HasValue && DateTime.Now - job.ExternalTaskStartedAt.Value > ExternalTimeout)
|
||||
{
|
||||
_ = await _client.TryCancelCopyTaskAsync(settings, password, job.ExternalTaskId, cancellationToken);
|
||||
throw new TimeoutException("OpenList 复制任务运行超过 24 小时。");
|
||||
}
|
||||
if (job.LastProgressAt.HasValue && DateTime.Now - job.LastProgressAt.Value > StallTimeout)
|
||||
{
|
||||
var cancelled = await _client.TryCancelCopyTaskAsync(settings, password, job.ExternalTaskId, cancellationToken);
|
||||
throw new TimeoutException(cancelled
|
||||
? "OpenList 复制任务连续 60 分钟没有进度,已取消。"
|
||||
: "OpenList 复制任务连续 60 分钟没有进度且无法取消。");
|
||||
}
|
||||
var task = await _client.TryGetCopyTaskAsync(settings, password, job.ExternalTaskId, cancellationToken);
|
||||
if (task == null)
|
||||
{
|
||||
var staged = await _client.TryGetObjectAsync(settings, password, job.StagedTargetPath, cancellationToken);
|
||||
if (staged is not { IsDirectory: false } || staged.Size != job.ExpectedLength)
|
||||
throw new IOException($"OpenList 复制任务不存在:{job.ExternalTaskId}");
|
||||
job.Status = OpenListTransferStatus.Verifying;
|
||||
}
|
||||
else if (task.State == 2) job.Status = OpenListTransferStatus.Verifying;
|
||||
else if (task.State is 4 or 7)
|
||||
throw new IOException(string.IsNullOrWhiteSpace(task.Error)
|
||||
? $"OpenList 复制任务以状态 {task.State} 结束。" : task.Error);
|
||||
else if (Math.Abs(task.Progress - job.ExternalProgress) > 0.001)
|
||||
{
|
||||
job.ExternalProgress = task.Progress;
|
||||
job.LastProgressAt = DateTime.Now;
|
||||
}
|
||||
job.UpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(job).ExecuteCommandAsync();
|
||||
return false;
|
||||
}
|
||||
|
||||
if (job.Status == OpenListTransferStatus.Verifying)
|
||||
{
|
||||
var staged = await _client.TryGetObjectAsync(settings, password, job.StagedTargetPath, cancellationToken);
|
||||
if (staged is not { IsDirectory: false } || staged.Size != job.ExpectedLength)
|
||||
throw new IOException($"OpenList 暂存文件长度校验失败:期望 {job.ExpectedLength},实际 {staged?.Size.ToString() ?? "不存在"}。");
|
||||
job.Status = OpenListTransferStatus.Promoting;
|
||||
job.UpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(job).ExecuteCommandAsync();
|
||||
return false;
|
||||
}
|
||||
|
||||
if (job.Status == OpenListTransferStatus.Promoting)
|
||||
{
|
||||
await PromoteAsync(job, settings, password, cancellationToken);
|
||||
await MarkSucceededAsync(job, settings, password, cancellationToken);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private async Task PromoteAsync(
|
||||
OpenListTransferJob job,
|
||||
OpenListSettings settings,
|
||||
string password,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var target = await _client.TryGetObjectAsync(settings, password, job.ActualTargetPath, cancellationToken);
|
||||
if (target is { IsDirectory: false } && target.Size == job.ExpectedLength) return;
|
||||
|
||||
if (target != null && string.IsNullOrWhiteSpace(job.BackupTargetPath))
|
||||
{
|
||||
var backupName = $".dysync-old-{job.Id[..8]}-{GetFileName(job.ActualTargetPath)}";
|
||||
await _client.RenameAsync(settings, password, job.ActualTargetPath, backupName, cancellationToken);
|
||||
job.BackupTargetPath = StoragePath.CombineRemote(
|
||||
StoragePath.DirectoryName(job.ActualTargetPath), backupName);
|
||||
job.UpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(job).ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var final = await _client.TryGetObjectAsync(settings, password, job.ActualTargetPath, cancellationToken);
|
||||
if (final == null)
|
||||
await _client.MoveAsync(settings, password, job.StagedTargetPath,
|
||||
StoragePath.DirectoryName(job.ActualTargetPath), false, cancellationToken);
|
||||
final = await _client.TryGetObjectAsync(settings, password, job.ActualTargetPath, cancellationToken);
|
||||
if (final is not { IsDirectory: false } || final.Size != job.ExpectedLength)
|
||||
throw new IOException("OpenList 暂存文件提升后最终目标校验失败。");
|
||||
if (!string.IsNullOrWhiteSpace(job.BackupTargetPath))
|
||||
{
|
||||
await _client.DeleteObjectAsync(settings, password, job.BackupTargetPath, cancellationToken);
|
||||
job.BackupTargetPath = null;
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
var final = await _client.TryGetObjectAsync(settings, password, job.ActualTargetPath, cancellationToken);
|
||||
var backup = string.IsNullOrWhiteSpace(job.BackupTargetPath) ? null
|
||||
: await _client.TryGetObjectAsync(settings, password, job.BackupTargetPath, cancellationToken);
|
||||
if (final == null && backup != null)
|
||||
{
|
||||
await _client.RenameAsync(settings, password, job.BackupTargetPath,
|
||||
GetFileName(job.ActualTargetPath), cancellationToken);
|
||||
job.BackupTargetPath = null;
|
||||
job.UpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(job).ExecuteCommandAsync();
|
||||
}
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task MarkSucceededAsync(
|
||||
OpenListTransferJob job,
|
||||
OpenListSettings settings,
|
||||
string password,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var final = await _client.TryGetObjectAsync(settings, password, job.ActualTargetPath, cancellationToken);
|
||||
if (final is not { IsDirectory: false } || final.Size != job.ExpectedLength)
|
||||
throw new IOException("OpenList 最终目标不存在或长度不一致。");
|
||||
job.Status = OpenListTransferStatus.Succeeded;
|
||||
job.ErrorMessage = null;
|
||||
job.NextAttemptAt = null;
|
||||
job.ExternalTaskId = null;
|
||||
job.CompletedAt = DateTime.Now;
|
||||
job.UpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(job).ExecuteCommandAsync();
|
||||
CleanupLocal(job.LocalSourcePath);
|
||||
try
|
||||
{
|
||||
var stagedDirectory = StoragePath.DirectoryName(job.StagedTargetPath);
|
||||
await _client.DeleteObjectAsync(settings, password, stagedDirectory, cancellationToken);
|
||||
}
|
||||
catch (Exception ex) { Serilog.Log.Debug(ex, "清理 OpenList 暂存目录失败:{Path}", job.StagedTargetPath); }
|
||||
}
|
||||
|
||||
private async Task ScheduleRetryOrFailAsync(OpenListTransferJob job, string error)
|
||||
{
|
||||
job.ExternalTaskId = null;
|
||||
job.ExternalProgress = 0;
|
||||
job.ExternalTaskStartedAt = null;
|
||||
job.LastProgressAt = null;
|
||||
job.ErrorMessage = error;
|
||||
job.UpdatedAt = DateTime.Now;
|
||||
if (job.Attempts >= MaxAttempts)
|
||||
{
|
||||
job.Status = OpenListTransferStatus.Failed;
|
||||
job.CompletedAt = DateTime.Now;
|
||||
job.NextAttemptAt = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
var index = Math.Clamp(Math.Max(1, job.Attempts) - 1, 0, RetryDelays.Length - 1);
|
||||
job.Status = OpenListTransferStatus.WaitingRetry;
|
||||
job.NextAttemptAt = DateTime.Now.Add(RetryDelays[index]);
|
||||
}
|
||||
await _db.Updateable(job).ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
private async Task<bool> TryAcquireLeaseAsync(string jobId, string owner)
|
||||
{
|
||||
var now = DateTime.Now;
|
||||
var changed = await _db.Updateable<OpenListTransferJob>()
|
||||
.SetColumns(x => new OpenListTransferJob
|
||||
{
|
||||
LeaseOwner = owner,
|
||||
LeaseUntil = now.Add(LeaseDuration),
|
||||
UpdatedAt = now
|
||||
})
|
||||
.Where(x => x.Id == jobId && (x.LeaseUntil == null || x.LeaseUntil < now))
|
||||
.ExecuteCommandAsync();
|
||||
return changed == 1;
|
||||
}
|
||||
|
||||
private async Task RenewLeaseAsync(OpenListTransferJob job, string owner)
|
||||
{
|
||||
job.LeaseOwner = owner;
|
||||
job.LeaseUntil = DateTime.Now.Add(LeaseDuration);
|
||||
job.UpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(job).ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
private async Task ReleaseLeaseAsync(string jobId, string owner)
|
||||
{
|
||||
await _db.Updateable<OpenListTransferJob>()
|
||||
.SetColumns(x => new OpenListTransferJob { LeaseOwner = null, LeaseUntil = null, UpdatedAt = DateTime.Now })
|
||||
.Where(x => x.Id == jobId && x.LeaseOwner == owner).ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
private async Task WaitForOtherProcessorAsync(string jobId, CancellationToken cancellationToken)
|
||||
{
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
var job = await _db.Queryable<OpenListTransferJob>().InSingleAsync(jobId)
|
||||
?? throw new InvalidOperationException("OpenList 传输任务不存在。");
|
||||
if (job.Status == OpenListTransferStatus.Succeeded) return;
|
||||
if (job.Status is OpenListTransferStatus.Failed or OpenListTransferStatus.Cancelled
|
||||
or OpenListTransferStatus.WaitingRetry)
|
||||
throw new IOException(job.ErrorMessage ?? "OpenList 传输未完成。");
|
||||
if (job.LeaseUntil == null || job.LeaseUntil < DateTime.Now)
|
||||
{
|
||||
await RunUntilSettledAsync(jobId, cancellationToken);
|
||||
return;
|
||||
}
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
private static void CleanupLocal(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (File.Exists(path)) File.Delete(path);
|
||||
var directory = Path.GetDirectoryName(path);
|
||||
if (!string.IsNullOrWhiteSpace(directory) && Directory.Exists(directory)
|
||||
&& !Directory.EnumerateFileSystemEntries(directory).Any()) Directory.Delete(directory);
|
||||
}
|
||||
catch (Exception ex) { Serilog.Log.Warning(ex, "清理 OpenList 本地中转文件失败:{Path}", path); }
|
||||
}
|
||||
|
||||
private static string GetFileName(string path)
|
||||
{
|
||||
var normalized = StoragePath.NormalizeRemote(path);
|
||||
return normalized[(normalized.LastIndexOf('/') + 1)..];
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
namespace dy.net.service
|
||||
{
|
||||
public sealed class OpenListTransferWorker : BackgroundService
|
||||
{
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
|
||||
public OpenListTransferWorker(IServiceScopeFactory scopeFactory) => _scopeFactory = scopeFactory;
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var worked = await scope.ServiceProvider.GetRequiredService<OpenListTransferService>()
|
||||
.RecoverOneAsync(stoppingToken);
|
||||
if (!worked) await Task.Delay(TimeSpan.FromSeconds(3), stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { }
|
||||
catch (Exception ex)
|
||||
{
|
||||
Serilog.Log.Error(ex, "OpenList 传输恢复服务异常");
|
||||
await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,438 @@
|
||||
using System.Text;
|
||||
using dy.net.model.dto;
|
||||
using dy.net.model.entity;
|
||||
using dy.net.storage;
|
||||
using dy.net.utils;
|
||||
using Newtonsoft.Json;
|
||||
using SqlSugar;
|
||||
using MediaStorageType = dy.net.model.dto.StorageType;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
public class StorageMigrationItemProcessor
|
||||
{
|
||||
private readonly ISqlSugarClient _db;
|
||||
private readonly OpenListSettingsService _settings;
|
||||
private readonly OpenListMediaStorage _openList;
|
||||
private readonly DouyinHttpClientService _http;
|
||||
private readonly DouyinMigrationSourceResolver _sourceResolver;
|
||||
|
||||
public StorageMigrationItemProcessor(
|
||||
ISqlSugarClient db,
|
||||
OpenListSettingsService settings,
|
||||
OpenListMediaStorage openList,
|
||||
DouyinHttpClientService http,
|
||||
DouyinMigrationSourceResolver sourceResolver)
|
||||
{
|
||||
_db = db;
|
||||
_settings = settings;
|
||||
_openList = openList;
|
||||
_http = http;
|
||||
_sourceResolver = sourceResolver;
|
||||
}
|
||||
|
||||
public async Task ProcessAsync(string itemId, CancellationToken cancellationToken)
|
||||
{
|
||||
var item = await _db.Queryable<StorageMigrationItem>().InSingleAsync(itemId);
|
||||
if (item == null || item.Stage != StorageMigrationItemStage.Pending) return;
|
||||
var task = await _db.Queryable<StorageMigrationTask>().InSingleAsync(item.TaskId);
|
||||
if (task?.Status != StorageMigrationTaskStatus.Running) return;
|
||||
if (task.TargetStorageType != MediaStorageType.OpenList)
|
||||
{
|
||||
await _db.Updateable<StorageMigrationTask>()
|
||||
.SetColumns(x => new StorageMigrationTask
|
||||
{
|
||||
Status = StorageMigrationTaskStatus.Cancelled,
|
||||
ErrorMessage = "旧 WebDAV 迁移任务仅保留审计信息,不能继续写入。",
|
||||
CompletedAt = DateTime.Now,
|
||||
UpdatedAt = DateTime.Now
|
||||
}).Where(x => x.Id == task.Id).ExecuteCommandAsync();
|
||||
return;
|
||||
}
|
||||
|
||||
var targetSettings = await _settings.GetAsync();
|
||||
var fingerprint = StorageConfigurationFingerprint.Create(targetSettings);
|
||||
if (!string.Equals(fingerprint, task.ConfigurationFingerprint, StringComparison.Ordinal))
|
||||
{
|
||||
await _db.Updateable<StorageMigrationTask>()
|
||||
.SetColumns(x => new StorageMigrationTask
|
||||
{
|
||||
Status = StorageMigrationTaskStatus.Paused,
|
||||
ErrorMessage = "OpenList 配置已变化,请恢复原配置并重新预检。",
|
||||
UpdatedAt = DateTime.Now
|
||||
}).Where(x => x.Id == task.Id).ExecuteCommandAsync();
|
||||
return;
|
||||
}
|
||||
var targetStorage = _openList.Bind(targetSettings);
|
||||
|
||||
var snapshot = JsonConvert.DeserializeObject<DouyinVideo>(item.OldSnapshotJson)
|
||||
?? throw new InvalidOperationException("迁移条目缺少旧视频快照");
|
||||
var cookie = await _db.Queryable<DouyinCookie>().InSingleAsync(snapshot.CookieId);
|
||||
var roots = StorageMigrationPathPolicy.GetLocalRoots(cookie);
|
||||
var attachments = JsonConvert.DeserializeObject<List<StorageMigrationService.MigrationAttachmentPlan>>(item.AttachmentPlanJson) ?? new();
|
||||
var uploaded = DeserializeSet(item.UploadedPathsJson);
|
||||
var owned = DeserializeSet(item.OwnedRemotePathsJson);
|
||||
var warnings = new List<string>();
|
||||
|
||||
try
|
||||
{
|
||||
if (targetStorage is ICanonicalMediaStorage canonicalStorage)
|
||||
{
|
||||
item.TargetVideoPath = await canonicalStorage.CanonicalizePathAsync(
|
||||
item.TargetVideoPath, true, cancellationToken);
|
||||
foreach (var attachment in attachments.Where(x => !string.IsNullOrWhiteSpace(x.TargetPath)))
|
||||
attachment.TargetPath = await canonicalStorage.CanonicalizePathAsync(
|
||||
attachment.TargetPath, true, cancellationToken);
|
||||
item.AttachmentPlanJson = JsonConvert.SerializeObject(attachments);
|
||||
item.UpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(item).ExecuteCommandAsync();
|
||||
}
|
||||
if (item.AdoptExistingTarget && snapshot.StorageType == MediaStorageType.WebDav)
|
||||
{
|
||||
await AdoptExistingAsync(item, snapshot, targetStorage, cancellationToken);
|
||||
return;
|
||||
}
|
||||
item.Stage = StorageMigrationItemStage.Uploading;
|
||||
item.Attempts++;
|
||||
item.ErrorMessage = null;
|
||||
item.UpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(item).ExecuteCommandAsync();
|
||||
|
||||
var mainLength = await UploadMainAsync(snapshot, cookie, roots, item, targetStorage, uploaded, owned, cancellationToken);
|
||||
item.ExpectedLength = mainLength;
|
||||
|
||||
var newDynamic = new List<DouyinMergeVideoDto>();
|
||||
foreach (var attachment in attachments)
|
||||
{
|
||||
if (attachment.TargetPath == item.TargetVideoPath) continue;
|
||||
try
|
||||
{
|
||||
var length = await UploadAttachmentAsync(attachment, cookie, roots, targetStorage, uploaded, owned, cancellationToken);
|
||||
if (length.HasValue && attachment.Kind == "dynamic")
|
||||
newDynamic.Add(new DouyinMergeVideoDto { Path = attachment.TargetPath, Height = attachment.Height, Width = attachment.Width });
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
warnings.Add($"{AttachmentName(attachment.Kind)}:{ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
item.Stage = StorageMigrationItemStage.Verifying;
|
||||
item.UploadedPathsJson = JsonConvert.SerializeObject(uploaded);
|
||||
item.OwnedRemotePathsJson = JsonConvert.SerializeObject(owned);
|
||||
item.WarningMessage = warnings.Count == 0 ? null : string.Join(";", warnings);
|
||||
item.UpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(item).ExecuteCommandAsync();
|
||||
|
||||
var remoteLength = await targetStorage.GetLengthAsync(item.TargetVideoPath, cancellationToken);
|
||||
if (!remoteLength.HasValue || remoteLength.Value != mainLength)
|
||||
throw new IOException($"远端主媒体长度校验失败,期望 {mainLength},实际 {remoteLength?.ToString() ?? "不存在"}");
|
||||
|
||||
var migrated = JsonConvert.DeserializeObject<DouyinVideo>(item.OldSnapshotJson);
|
||||
migrated.StorageType = MediaStorageType.OpenList;
|
||||
migrated.VideoSavePath = item.TargetVideoPath;
|
||||
migrated.VideoCoverSavePath = attachments.FirstOrDefault(x => x.Kind == "cover" && uploaded.Contains(x.TargetPath))?.TargetPath;
|
||||
migrated.AuthorAvatar = attachments.FirstOrDefault(x => x.Kind == "avatar" && uploaded.Contains(x.TargetPath))?.TargetPath;
|
||||
migrated.DynamicVideos = newDynamic.Count == 0 ? null : JsonConvert.SerializeObject(newDynamic);
|
||||
migrated.FileSize = mainLength;
|
||||
var category = string.IsNullOrWhiteSpace(snapshot.CateId) ? null : await _db.Queryable<DouyinCollectCate>().InSingleAsync(snapshot.CateId);
|
||||
foreach (var nfo in NfoContentBuilder.Build(migrated, category?.Name))
|
||||
{
|
||||
var bytes = Encoding.UTF8.GetBytes(nfo.Value);
|
||||
var existed = await targetStorage.ExistsAsync(nfo.Key, cancellationToken);
|
||||
await using var content = new MemoryStream(bytes, false);
|
||||
await targetStorage.WriteAsync(nfo.Key, content, bytes.Length, "application/xml", cancellationToken);
|
||||
uploaded.Add(nfo.Key);
|
||||
if (!existed) owned.Add(nfo.Key);
|
||||
}
|
||||
|
||||
item.Stage = StorageMigrationItemStage.Committing;
|
||||
item.UploadedPathsJson = JsonConvert.SerializeObject(uploaded);
|
||||
item.OwnedRemotePathsJson = JsonConvert.SerializeObject(owned);
|
||||
item.UpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(item).ExecuteCommandAsync();
|
||||
|
||||
var transaction = await _db.Ado.UseTranAsync(async () =>
|
||||
{
|
||||
var current = await _db.Queryable<DouyinVideo>().InSingleAsync(item.VideoId)
|
||||
?? throw new InvalidOperationException("原视频记录已不存在");
|
||||
if (current.StorageType == MediaStorageType.OpenList && current.VideoSavePath == item.TargetVideoPath)
|
||||
{
|
||||
// A previous process committed the video and died before updating the item.
|
||||
}
|
||||
else if (await TryReconcileCurrentOpenListAsync(
|
||||
current, item, snapshot, targetStorage, cancellationToken))
|
||||
{
|
||||
// A normal sync already converged this record while migration was running.
|
||||
// The verified OpenList record is authoritative and must not be overwritten.
|
||||
}
|
||||
else
|
||||
{
|
||||
if (current.StorageType != snapshot.StorageType || current.VideoSavePath != snapshot.VideoSavePath)
|
||||
throw new InvalidOperationException("原视频记录在迁移期间已变化,拒绝覆盖");
|
||||
current.StorageType = MediaStorageType.OpenList;
|
||||
current.VideoSavePath = migrated.VideoSavePath;
|
||||
current.VideoCoverSavePath = migrated.VideoCoverSavePath;
|
||||
current.AuthorAvatar = migrated.AuthorAvatar;
|
||||
current.DynamicVideos = migrated.DynamicVideos;
|
||||
current.FileSize = migrated.FileSize;
|
||||
var changed = await _db.Updateable(current).ExecuteCommandAsync();
|
||||
if (changed != 1) throw new InvalidOperationException("提交原视频记录失败");
|
||||
}
|
||||
|
||||
item.Stage = warnings.Count == 0 ? StorageMigrationItemStage.Succeeded : StorageMigrationItemStage.SucceededWithWarnings;
|
||||
item.CompletedAt = DateTime.Now;
|
||||
item.UpdatedAt = DateTime.Now;
|
||||
item.WarningMessage = warnings.Count == 0 ? null : string.Join(";", warnings);
|
||||
var itemChanged = await _db.Updateable(item).ExecuteCommandAsync();
|
||||
if (itemChanged != 1) throw new InvalidOperationException("提交迁移条目失败");
|
||||
});
|
||||
if (!transaction.IsSuccess) throw transaction.ErrorException ?? new InvalidOperationException("数据库事务提交失败");
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
await FailAsync(item, "应用正在停止,条目将在下次启动后重试。", resetToPending: true);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Serilog.Log.Warning(ex, "存储迁移条目失败:{ItemId}/{VideoId}", item.Id, item.VideoId);
|
||||
item.UploadedPathsJson = JsonConvert.SerializeObject(uploaded);
|
||||
item.OwnedRemotePathsJson = JsonConvert.SerializeObject(owned);
|
||||
await FailAsync(item, ex.Message, resetToPending: false);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task AdoptExistingAsync(
|
||||
StorageMigrationItem item,
|
||||
DouyinVideo snapshot,
|
||||
IMediaStorage targetStorage,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
item.Stage = StorageMigrationItemStage.Verifying;
|
||||
item.Attempts++;
|
||||
item.ErrorMessage = null;
|
||||
item.UpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(item).ExecuteCommandAsync();
|
||||
|
||||
var length = await targetStorage.GetLengthAsync(item.TargetVideoPath, cancellationToken);
|
||||
if (!length.HasValue || length <= 0
|
||||
|| snapshot.FileSize > 0 && length.Value != snapshot.FileSize)
|
||||
throw new IOException($"OpenList 接管校验失败:期望 {snapshot.FileSize},实际 {length?.ToString() ?? "不存在"}");
|
||||
|
||||
var warnings = new List<string>();
|
||||
foreach (var attachment in new[]
|
||||
{
|
||||
(Name: "封面", Path: snapshot.VideoCoverSavePath),
|
||||
(Name: "头像", Path: snapshot.AuthorAvatar)
|
||||
})
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(attachment.Path)) continue;
|
||||
try
|
||||
{
|
||||
if (!await targetStorage.ExistsAsync(attachment.Path, cancellationToken))
|
||||
warnings.Add($"{attachment.Name}在 OpenList 中不存在");
|
||||
}
|
||||
catch (Exception ex) { warnings.Add($"{attachment.Name}检查失败:{ex.GetBaseException().Message}"); }
|
||||
}
|
||||
|
||||
item.Stage = StorageMigrationItemStage.Committing;
|
||||
item.ExpectedLength = length.Value;
|
||||
item.WarningMessage = warnings.Count == 0 ? null : string.Join(";", warnings);
|
||||
item.UpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(item).ExecuteCommandAsync();
|
||||
|
||||
var transaction = await _db.Ado.UseTranAsync(async () =>
|
||||
{
|
||||
var current = await _db.Queryable<DouyinVideo>().InSingleAsync(item.VideoId)
|
||||
?? throw new InvalidOperationException("原视频记录已不存在");
|
||||
if (current.StorageType == MediaStorageType.OpenList
|
||||
&& current.VideoSavePath == item.TargetVideoPath)
|
||||
{
|
||||
// 上次进程可能已提交视频,但尚未更新迁移条目。
|
||||
}
|
||||
else if (await TryReconcileCurrentOpenListAsync(
|
||||
current, item, snapshot, targetStorage, cancellationToken))
|
||||
{
|
||||
// 普通同步已先完成接管;验证当前远端文件后直接收敛迁移条目。
|
||||
}
|
||||
else
|
||||
{
|
||||
if (current.StorageType != snapshot.StorageType
|
||||
|| current.VideoSavePath != snapshot.VideoSavePath)
|
||||
throw new InvalidOperationException("原视频记录在接管期间已变化,拒绝覆盖");
|
||||
current.StorageType = MediaStorageType.OpenList;
|
||||
current.VideoSavePath = item.TargetVideoPath;
|
||||
current.FileSize = length.Value;
|
||||
if (await _db.Updateable(current).ExecuteCommandAsync() != 1)
|
||||
throw new InvalidOperationException("提交 OpenList 接管记录失败");
|
||||
}
|
||||
item.Stage = warnings.Count == 0
|
||||
? StorageMigrationItemStage.Succeeded
|
||||
: StorageMigrationItemStage.SucceededWithWarnings;
|
||||
item.CompletedAt = DateTime.Now;
|
||||
item.UpdatedAt = DateTime.Now;
|
||||
if (await _db.Updateable(item).ExecuteCommandAsync() != 1)
|
||||
throw new InvalidOperationException("提交 OpenList 接管条目失败");
|
||||
});
|
||||
if (!transaction.IsSuccess)
|
||||
throw transaction.ErrorException ?? new InvalidOperationException("OpenList 接管事务提交失败");
|
||||
}
|
||||
|
||||
private static async Task<bool> TryReconcileCurrentOpenListAsync(
|
||||
DouyinVideo current,
|
||||
StorageMigrationItem item,
|
||||
DouyinVideo snapshot,
|
||||
IMediaStorage targetStorage,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (current?.StorageType != MediaStorageType.OpenList
|
||||
|| string.IsNullOrWhiteSpace(current.VideoSavePath)) return false;
|
||||
|
||||
var canonicalPath = targetStorage is ICanonicalMediaStorage canonicalStorage
|
||||
? await canonicalStorage.CanonicalizePathAsync(current.VideoSavePath, false, cancellationToken)
|
||||
: current.VideoSavePath;
|
||||
var remoteLength = await targetStorage.GetLengthAsync(canonicalPath, cancellationToken);
|
||||
if (!remoteLength.HasValue || remoteLength.Value <= 0)
|
||||
throw new IOException($"并发接管记录已指向 OpenList,但远端主媒体不存在:{canonicalPath}");
|
||||
|
||||
var expectedLength = current.FileSize > 0
|
||||
? current.FileSize
|
||||
: item.ExpectedLength > 0 ? item.ExpectedLength : snapshot?.FileSize ?? 0;
|
||||
if (expectedLength > 0 && remoteLength.Value != expectedLength)
|
||||
throw new MediaIntegrityException(
|
||||
$"并发接管后的远端主媒体长度不一致,期望 {expectedLength},实际 {remoteLength.Value}");
|
||||
|
||||
item.TargetVideoPath = canonicalPath;
|
||||
item.ExpectedLength = remoteLength.Value;
|
||||
return true;
|
||||
}
|
||||
|
||||
private async Task<long> UploadMainAsync(
|
||||
DouyinVideo snapshot,
|
||||
DouyinCookie cookie,
|
||||
IReadOnlyList<string> roots,
|
||||
StorageMigrationItem item,
|
||||
IMediaStorage targetStorage,
|
||||
HashSet<string> uploaded,
|
||||
HashSet<string> owned,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (SafeLocalMigrationFile.TryResolve(snapshot.VideoSavePath, roots, out var local, out _)
|
||||
&& new FileInfo(local).Length > 0)
|
||||
{
|
||||
var expected = new FileInfo(local).Length;
|
||||
await UploadLocalAsync(local, item.TargetVideoPath, expected, targetStorage, uploaded, owned, cancellationToken);
|
||||
return expected;
|
||||
}
|
||||
|
||||
var knownLength = item.ExpectedLength > 0 ? item.ExpectedLength : snapshot.FileSize;
|
||||
var existingLength = await targetStorage.GetLengthAsync(item.TargetVideoPath, cancellationToken);
|
||||
if (knownLength > 0 && existingLength == knownLength)
|
||||
{
|
||||
uploaded.Add(item.TargetVideoPath);
|
||||
return knownLength;
|
||||
}
|
||||
|
||||
var existed = existingLength.HasValue;
|
||||
if (!string.IsNullOrWhiteSpace(snapshot.VideoUrl))
|
||||
{
|
||||
var direct = await _http.DownloadToStorageAsync(MediaStorageType.OpenList, snapshot.VideoUrl, item.TargetVideoPath,
|
||||
cookie?.Cookies, cancellationToken: cancellationToken, maxRetryCount: 1, storageOverride: targetStorage);
|
||||
if (direct.Success)
|
||||
{
|
||||
var directLength = await targetStorage.GetLengthAsync(item.TargetVideoPath, cancellationToken);
|
||||
if (directLength.HasValue && directLength.Value > 0)
|
||||
{
|
||||
uploaded.Add(item.TargetVideoPath);
|
||||
if (!existed) owned.Add(item.TargetVideoPath);
|
||||
return directLength.Value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var urls = (await _sourceResolver.ResolveAsync(snapshot, cookie, cancellationToken)).ToList();
|
||||
if (urls.Count == 0) throw new FileNotFoundException("本地主媒体缺失,记录 URL 和抖音全量列表均未找到可用回源地址");
|
||||
var result = await _http.DownloadToStorageAsync(MediaStorageType.OpenList, urls[0], item.TargetVideoPath,
|
||||
cookie?.Cookies, urls.Skip(1).ToList(), cancellationToken, Math.Min(6, urls.Count), targetStorage);
|
||||
if (!result.Success) throw result.ToException();
|
||||
var length = await targetStorage.GetLengthAsync(item.TargetVideoPath, cancellationToken);
|
||||
if (!length.HasValue || length.Value <= 0) throw new IOException("回源上传后远端主媒体为空");
|
||||
uploaded.Add(item.TargetVideoPath);
|
||||
if (!existed) owned.Add(item.TargetVideoPath);
|
||||
return length.Value;
|
||||
}
|
||||
|
||||
private async Task<long?> UploadAttachmentAsync(
|
||||
StorageMigrationService.MigrationAttachmentPlan attachment,
|
||||
DouyinCookie cookie,
|
||||
IReadOnlyList<string> roots,
|
||||
IMediaStorage targetStorage,
|
||||
HashSet<string> uploaded,
|
||||
HashSet<string> owned,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (SafeLocalMigrationFile.TryResolve(attachment.SourcePath, roots, out var local, out _))
|
||||
{
|
||||
var length = new FileInfo(local).Length;
|
||||
if (length <= 0) throw new IOException("本地附件为空");
|
||||
await UploadLocalAsync(local, attachment.TargetPath, length, targetStorage, uploaded, owned, cancellationToken);
|
||||
return length;
|
||||
}
|
||||
if (string.IsNullOrWhiteSpace(attachment.SourceUrl)) throw new FileNotFoundException("本地附件缺失且无回源地址");
|
||||
var before = await targetStorage.GetLengthAsync(attachment.TargetPath, cancellationToken);
|
||||
var result = await _http.DownloadToStorageAsync(MediaStorageType.OpenList, attachment.SourceUrl, attachment.TargetPath,
|
||||
cookie?.Cookies, cancellationToken: cancellationToken, storageOverride: targetStorage);
|
||||
if (!result.Success) throw result.ToException();
|
||||
var after = await targetStorage.GetLengthAsync(attachment.TargetPath, cancellationToken);
|
||||
if (!after.HasValue || after.Value <= 0) throw new IOException("附件上传后为空");
|
||||
uploaded.Add(attachment.TargetPath);
|
||||
if (!before.HasValue) owned.Add(attachment.TargetPath);
|
||||
return after;
|
||||
}
|
||||
|
||||
private async Task UploadLocalAsync(
|
||||
string local,
|
||||
string target,
|
||||
long expected,
|
||||
IMediaStorage targetStorage,
|
||||
HashSet<string> uploaded,
|
||||
HashSet<string> owned,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var before = await targetStorage.GetLengthAsync(target, cancellationToken);
|
||||
if (before == expected)
|
||||
{
|
||||
uploaded.Add(target);
|
||||
return;
|
||||
}
|
||||
await using var source = new FileStream(local, FileMode.Open, FileAccess.Read, FileShare.Read, 81920, true);
|
||||
await targetStorage.WriteAsync(target, source, expected, null, cancellationToken);
|
||||
var after = await targetStorage.GetLengthAsync(target, cancellationToken);
|
||||
if (after != expected) throw new IOException($"附件长度校验失败,期望 {expected},实际 {after?.ToString() ?? "不存在"}");
|
||||
uploaded.Add(target);
|
||||
if (!before.HasValue) owned.Add(target);
|
||||
}
|
||||
|
||||
private async Task FailAsync(StorageMigrationItem item, string error, bool resetToPending)
|
||||
{
|
||||
item.Stage = resetToPending ? StorageMigrationItemStage.Pending : StorageMigrationItemStage.Failed;
|
||||
item.ErrorMessage = error;
|
||||
item.UpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(item).ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
private static HashSet<string> DeserializeSet(string json)
|
||||
{
|
||||
try { return new HashSet<string>(JsonConvert.DeserializeObject<List<string>>(json) ?? new(), StringComparer.Ordinal); }
|
||||
catch { return new HashSet<string>(StringComparer.Ordinal); }
|
||||
}
|
||||
|
||||
private static string AttachmentName(string kind) => kind switch
|
||||
{
|
||||
"cover" => "封面",
|
||||
"avatar" => "头像",
|
||||
"dynamic" => "图文/动态附件",
|
||||
_ => "附件"
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,958 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using dy.net.model.dto;
|
||||
using dy.net.model.entity;
|
||||
using dy.net.storage;
|
||||
using dy.net.utils;
|
||||
using Newtonsoft.Json;
|
||||
using SqlSugar;
|
||||
using MediaStorageType = dy.net.model.dto.StorageType;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
public class StorageMigrationService
|
||||
{
|
||||
private readonly ISqlSugarClient _db;
|
||||
private readonly DouyinCommonService _commonService;
|
||||
private readonly OpenListSettingsService _settingsService;
|
||||
private readonly OpenListMediaStorage _openList;
|
||||
|
||||
public StorageMigrationService(
|
||||
ISqlSugarClient db,
|
||||
DouyinCommonService commonService,
|
||||
OpenListSettingsService settingsService,
|
||||
OpenListMediaStorage openList)
|
||||
{
|
||||
_db = db;
|
||||
_commonService = commonService;
|
||||
_settingsService = settingsService;
|
||||
_openList = openList;
|
||||
}
|
||||
|
||||
public async Task<StorageMigrationPreflightResult> PreflightAsync(bool verifyCapabilities, CancellationToken cancellationToken)
|
||||
{
|
||||
var result = new StorageMigrationPreflightResult();
|
||||
if (_commonService.GetConfig()?.StorageType != MediaStorageType.OpenList)
|
||||
{
|
||||
result.Errors.Add("请先完成测试并启用 OpenList 原生存储;迁移不会自动切换存储模式。");
|
||||
return result;
|
||||
}
|
||||
|
||||
var settings = await _settingsService.GetAsync();
|
||||
try
|
||||
{
|
||||
OpenListTransferService.ValidateSettings(settings);
|
||||
if (string.IsNullOrWhiteSpace(await _settingsService.GetPasswordAsync(settings)))
|
||||
result.Errors.Add("OpenList 密码无效,请重新输入并保存配置。");
|
||||
else
|
||||
result.ConfigurationFingerprint = StorageConfigurationFingerprint.Create(settings);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
result.Errors.Add(ex.GetBaseException().Message);
|
||||
}
|
||||
if (!settings.LastTestedAt.HasValue)
|
||||
result.Errors.Add("当前 OpenList 配置尚未通过服务端复制能力测试。");
|
||||
if (verifyCapabilities)
|
||||
{
|
||||
var probe = await _openList.ProbeAsync(settings, cancellationToken);
|
||||
if (!probe.Success) result.Errors.Add("OpenList 能力测试失败:" + probe.Message);
|
||||
}
|
||||
|
||||
if (result.Errors.Count > 0) return result;
|
||||
|
||||
List<PlanningEntry> planning;
|
||||
try { planning = await BuildPlanningAsync(); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
result.Errors.Add("读取 OpenList 目标失败:" + ex.GetBaseException().Message);
|
||||
return result;
|
||||
}
|
||||
result.RecordCount = planning.Count;
|
||||
result.LocalRecordCount = planning.Count(x => x.Video.StorageType == MediaStorageType.Local);
|
||||
result.LegacyWebDavRecordCount = planning.Count(x => x.Video.StorageType == MediaStorageType.WebDav);
|
||||
result.AdoptableCount = planning.Count(x => x.AdoptExistingTarget);
|
||||
result.TransferRequiredCount = planning.Count(x => !x.AdoptExistingTarget);
|
||||
result.ReadableFileCount = planning.Count(x => x.LocalReadable);
|
||||
result.MissingFileCount = planning.Count(x => !x.AdoptExistingTarget && !x.LocalReadable);
|
||||
result.TotalBytes = planning.Where(x => x.LocalReadable).Sum(x => x.LocalLength);
|
||||
result.InvalidCookieCount = planning.Where(x => x.Cookie == null || x.Cookie.StatusCode != 0 || string.IsNullOrWhiteSpace(x.Cookie.Cookies))
|
||||
.Select(x => x.Video.CookieId).Distinct().Count();
|
||||
result.MissingTargetPathCount = planning.Count(x => x.Plan == null);
|
||||
result.ConflictCount = planning.Where(x => x.Plan != null)
|
||||
.GroupBy(x => x.Plan.VideoPath, StringComparer.Ordinal)
|
||||
.Where(x => x.Count() > 1).Sum(x => x.Count());
|
||||
|
||||
foreach (var error in planning.Select(x => x.PlanError).Where(x => !string.IsNullOrWhiteSpace(x)).Distinct().Take(20))
|
||||
result.Errors.Add(error);
|
||||
if (result.ConflictCount > 0)
|
||||
result.Errors.Add($"发现 {result.ConflictCount} 条记录映射到重复目标路径,必须先调整目录或命名配置。");
|
||||
if (result.MissingFileCount > 0)
|
||||
result.Warnings.Add($"{result.MissingFileCount} 条记录的本地主媒体缺失或不可安全读取,执行时会先尝试记录 URL,再扫描对应抖音全量列表回源。");
|
||||
if (result.InvalidCookieCount > 0)
|
||||
result.Warnings.Add($"{result.InvalidCookieCount} 个账号 Cookie 无效;这些账号下缺失本地文件的作品可能无法回源。");
|
||||
result.Warnings.Add($"OpenList 无法统一报告所有后端剩余容量;容量未知,预计需读取本地数据 {FormatBytes(result.TotalBytes)}。");
|
||||
result.CanStart = result.RecordCount > 0 && result.Errors.Count == 0;
|
||||
if (result.RecordCount == 0) result.Errors.Add("没有需要接管到 OpenList 的旧存储视频记录。");
|
||||
return result;
|
||||
}
|
||||
|
||||
public async Task<StorageMigrationTask> CreateAsync(CreateStorageMigrationRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
request ??= new CreateStorageMigrationRequest();
|
||||
if (request.Concurrency is < 1 or > 3) throw new InvalidOperationException("迁移并发只能选择 1–3。");
|
||||
var preflight = await PreflightAsync(false, cancellationToken);
|
||||
if (!preflight.CanStart) throw new InvalidOperationException(string.Join(";", preflight.Errors));
|
||||
if (string.IsNullOrWhiteSpace(request.ConfigurationFingerprint)
|
||||
|| !string.Equals(request.ConfigurationFingerprint, preflight.ConfigurationFingerprint, StringComparison.Ordinal))
|
||||
throw new InvalidOperationException("OpenList 配置已变化,请重新执行迁移预检。");
|
||||
|
||||
var active = await _db.Queryable<StorageMigrationTask>().Where(x =>
|
||||
x.Status == StorageMigrationTaskStatus.Queued || x.Status == StorageMigrationTaskStatus.Running
|
||||
|| x.Status == StorageMigrationTaskStatus.Paused || x.Status == StorageMigrationTaskStatus.Cleaning).AnyAsync();
|
||||
if (active) throw new InvalidOperationException("已有未结束的存储迁移任务,请先处理当前任务。");
|
||||
|
||||
var planning = await BuildPlanningAsync();
|
||||
if (planning.Any(x => x.Plan == null || !string.IsNullOrWhiteSpace(x.PlanError)))
|
||||
throw new InvalidOperationException("迁移计划在创建前发生变化,请重新执行预检。");
|
||||
var now = DateTime.Now;
|
||||
var task = new StorageMigrationTask
|
||||
{
|
||||
Id = Guid.NewGuid().ToString("N"),
|
||||
Status = StorageMigrationTaskStatus.Queued,
|
||||
TargetStorageType = MediaStorageType.OpenList,
|
||||
Concurrency = request.Concurrency,
|
||||
TotalCount = planning.Count,
|
||||
PendingCount = planning.Count,
|
||||
TotalBytes = planning.Where(x => x.LocalReadable).Sum(x => x.LocalLength),
|
||||
ConfigurationFingerprint = preflight.ConfigurationFingerprint,
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now
|
||||
};
|
||||
var items = planning.Select(entry => new StorageMigrationItem
|
||||
{
|
||||
Id = Guid.NewGuid().ToString("N"),
|
||||
TaskId = task.Id,
|
||||
VideoId = entry.Video.Id,
|
||||
SourceStorageType = entry.Video.StorageType,
|
||||
AdoptExistingTarget = entry.AdoptExistingTarget,
|
||||
Stage = StorageMigrationItemStage.Pending,
|
||||
ExpectedLength = entry.LocalLength > 0 ? entry.LocalLength : entry.Video.FileSize,
|
||||
OldVideoPath = entry.Video.VideoSavePath,
|
||||
TargetVideoPath = entry.Plan.VideoPath,
|
||||
TargetCoverPath = entry.Plan.CoverPath,
|
||||
OldSnapshotJson = JsonConvert.SerializeObject(entry.Video),
|
||||
AttachmentPlanJson = JsonConvert.SerializeObject(BuildAttachments(entry)),
|
||||
UploadedPathsJson = "[]",
|
||||
OwnedRemotePathsJson = "[]",
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now
|
||||
}).ToList();
|
||||
|
||||
var transaction = await _db.Ado.UseTranAsync(async () =>
|
||||
{
|
||||
await _db.Insertable(task).ExecuteCommandAsync();
|
||||
await _db.Insertable(items).ExecuteCommandAsync();
|
||||
});
|
||||
if (!transaction.IsSuccess) throw new InvalidOperationException("创建迁移任务失败:" + transaction.ErrorMessage);
|
||||
return task;
|
||||
}
|
||||
|
||||
public async Task<StorageMigrationTaskDetail> GetAsync(string id)
|
||||
{
|
||||
var task = await _db.Queryable<StorageMigrationTask>().InSingleAsync(id)
|
||||
?? throw new KeyNotFoundException("迁移任务不存在");
|
||||
return ToDetail(task);
|
||||
}
|
||||
|
||||
public async Task<StorageMigrationTaskDetail> GetLatestAsync()
|
||||
{
|
||||
var task = await _db.Queryable<StorageMigrationTask>().Where(x => !x.IsArchived)
|
||||
.OrderByDescending(x => x.CreatedAt).FirstAsync();
|
||||
return task == null ? null : ToDetail(task);
|
||||
}
|
||||
|
||||
public async Task ArchiveAsync(string id)
|
||||
{
|
||||
var task = await RequireTaskAsync(id);
|
||||
if (!CanArchive(task))
|
||||
throw new InvalidOperationException("该迁移仍有成功项等待清理或回滚,请先处理旧文件后再归档。");
|
||||
task.IsArchived = true;
|
||||
task.UpdatedAt = DateTime.Now;
|
||||
if (await _db.Updateable(task).ExecuteCommandAsync() != 1)
|
||||
throw new InvalidOperationException("归档迁移历史失败。");
|
||||
}
|
||||
|
||||
public async Task<StorageMigrationArchiveResult> ArchiveAllSettledAsync()
|
||||
{
|
||||
var tasks = await _db.Queryable<StorageMigrationTask>().Where(x => !x.IsArchived).ToListAsync();
|
||||
var archivable = tasks.Where(CanArchive).ToList();
|
||||
var requiresCleanup = tasks.Count(x => IsTerminal(x.Status) && !CanArchive(x));
|
||||
if (archivable.Count > 0)
|
||||
{
|
||||
var now = DateTime.Now;
|
||||
foreach (var task in archivable)
|
||||
{
|
||||
task.IsArchived = true;
|
||||
task.UpdatedAt = now;
|
||||
}
|
||||
var changed = await _db.Updateable(archivable).ExecuteCommandAsync();
|
||||
if (changed != archivable.Count) throw new InvalidOperationException("部分迁移历史归档失败。");
|
||||
}
|
||||
return new StorageMigrationArchiveResult
|
||||
{
|
||||
ArchivedCount = archivable.Count,
|
||||
RequiresCleanupCount = requiresCleanup,
|
||||
Message = requiresCleanup > 0
|
||||
? $"已隐藏 {archivable.Count} 条迁移历史;另有 {requiresCleanup} 条包含已迁移视频,请先清理旧文件或回滚。"
|
||||
: $"已隐藏 {archivable.Count} 条已结束的迁移历史。"
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<StorageMigrationItemPage> GetItemsAsync(string taskId, StorageMigrationItemPageRequest request)
|
||||
{
|
||||
request ??= new StorageMigrationItemPageRequest();
|
||||
request.PageIndex = Math.Max(1, request.PageIndex);
|
||||
request.PageSize = Math.Clamp(request.PageSize, 1, 100);
|
||||
var query = _db.Queryable<StorageMigrationItem>().Where(x => x.TaskId == taskId)
|
||||
.WhereIF(request.Stage.HasValue, x => x.Stage == request.Stage.Value);
|
||||
return new StorageMigrationItemPage
|
||||
{
|
||||
TotalCount = await query.CountAsync(),
|
||||
Items = await query.OrderBy(x => x.CreatedAt).Skip((request.PageIndex - 1) * request.PageSize).Take(request.PageSize).ToListAsync()
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<FailedMigrationRecordPreview> PreviewFailedRecordRemovalAsync()
|
||||
{
|
||||
var plan = await BuildFailedRecordRemovalPlanAsync();
|
||||
var preview = new FailedMigrationRecordPreview
|
||||
{
|
||||
ActiveMigrationCount = plan.ActiveMigrationCount,
|
||||
FailedItemCount = plan.Groups.Sum(x => x.Items.Count),
|
||||
DistinctVideoCount = plan.Groups.Count,
|
||||
EligibleRecordCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.Eligible),
|
||||
AlreadyMissingCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.AlreadyMissing),
|
||||
ChangedRecordCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.Changed),
|
||||
PermanentlyExcludedCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.PermanentlyExcluded),
|
||||
InvalidSnapshotCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.InvalidSnapshot)
|
||||
};
|
||||
preview.CanExecute = preview.ActiveMigrationCount == 0
|
||||
&& preview.EligibleRecordCount + preview.AlreadyMissingCount > 0;
|
||||
preview.ConfirmationToken = preview.CanExecute ? plan.ConfirmationToken : null;
|
||||
|
||||
if (preview.ActiveMigrationCount > 0)
|
||||
preview.Errors.Add("存在排队、运行、暂停或清理中的存储迁移任务,请先结束当前迁移再处理历史失败记录。");
|
||||
else if (preview.FailedItemCount == 0)
|
||||
preview.Errors.Add("当前没有可处理的历史迁移失败项。");
|
||||
else if (!preview.CanExecute)
|
||||
preview.Errors.Add("没有可安全删除的视频记录;已变化、已迁移、永久排除或快照损坏的记录不会被修改。");
|
||||
|
||||
preview.Warnings.Add("此操作只删除数据库视频记录,不删除旧本地文件或 OpenList 文件,也不会自动启动同步。");
|
||||
if (preview.ChangedRecordCount > 0)
|
||||
preview.Warnings.Add($"{preview.ChangedRecordCount} 条视频记录已变化、已迁移或已重新创建,将安全跳过。");
|
||||
if (preview.PermanentlyExcludedCount > 0)
|
||||
preview.Warnings.Add($"{preview.PermanentlyExcludedCount} 条视频已被永久排除,请先在任务中心取消排除后再处理。");
|
||||
if (preview.InvalidSnapshotCount > 0)
|
||||
preview.Warnings.Add($"{preview.InvalidSnapshotCount} 条迁移快照无效,将保留原记录和失败状态。");
|
||||
return preview;
|
||||
}
|
||||
|
||||
public async Task<RemoveFailedMigrationRecordsResult> RemoveFailedRecordsAsync(RemoveFailedMigrationRecordsRequest request)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(request?.ConfirmationToken))
|
||||
throw new InvalidOperationException("缺少删除确认信息,请重新打开预览后确认。");
|
||||
|
||||
RemoveFailedMigrationRecordsResult result = null;
|
||||
var transaction = await _db.Ado.UseTranAsync(async () =>
|
||||
{
|
||||
var plan = await BuildFailedRecordRemovalPlanAsync();
|
||||
if (plan.ActiveMigrationCount > 0)
|
||||
throw new InvalidOperationException("存在未结束的存储迁移任务,请先结束当前迁移。");
|
||||
if (!string.Equals(plan.ConfirmationToken, request.ConfirmationToken, StringComparison.Ordinal))
|
||||
throw new InvalidOperationException("迁移失败记录已发生变化,请重新预览并确认。");
|
||||
|
||||
var actionable = plan.Groups.Where(x => x.Disposition is FailedRecordDisposition.Eligible
|
||||
or FailedRecordDisposition.AlreadyMissing).ToList();
|
||||
if (actionable.Count == 0)
|
||||
throw new InvalidOperationException("没有可安全删除的视频记录。");
|
||||
|
||||
var now = DateTime.Now;
|
||||
var changedItems = new List<StorageMigrationItem>();
|
||||
var deletedRecords = 0;
|
||||
foreach (var group in actionable)
|
||||
{
|
||||
if (group.Disposition == FailedRecordDisposition.Eligible)
|
||||
{
|
||||
var current = group.Current;
|
||||
var expectedStorageType = current.StorageType;
|
||||
var deleted = await _db.Deleteable<DouyinVideo>().Where(x => x.Id == current.Id
|
||||
&& x.AwemeId == current.AwemeId && x.StorageType == expectedStorageType
|
||||
&& x.VideoSavePath == current.VideoSavePath).ExecuteCommandAsync();
|
||||
if (deleted != 1)
|
||||
throw new InvalidOperationException($"视频 {current.AwemeId ?? current.Id} 已发生变化,请重新预览。");
|
||||
deletedRecords++;
|
||||
}
|
||||
|
||||
foreach (var item in group.Items)
|
||||
{
|
||||
item.Stage = StorageMigrationItemStage.RecordRemoved;
|
||||
item.ErrorMessage = null;
|
||||
item.WarningMessage = group.Disposition == FailedRecordDisposition.Eligible
|
||||
? "视频记录已删除;旧文件和 OpenList 文件均已保留,等待后续正常同步重新发现。"
|
||||
: "原视频记录已不存在;迁移失败项已归档,等待后续正常同步重新发现。";
|
||||
item.UpdatedAt = now;
|
||||
item.CompletedAt = now;
|
||||
changedItems.Add(item);
|
||||
}
|
||||
}
|
||||
|
||||
if (changedItems.Count > 0)
|
||||
{
|
||||
var changed = await _db.Updateable(changedItems).ExecuteCommandAsync();
|
||||
if (changed != changedItems.Count)
|
||||
throw new InvalidOperationException("更新迁移失败项状态不完整,操作已回滚。");
|
||||
}
|
||||
|
||||
var affectedTaskIds = changedItems.Select(x => x.TaskId).Distinct().ToList();
|
||||
foreach (var taskId in affectedTaskIds) await RefreshCountsAsync(taskId);
|
||||
|
||||
result = new RemoveFailedMigrationRecordsResult
|
||||
{
|
||||
DeletedRecordCount = deletedRecords,
|
||||
AlreadyMissingCount = actionable.Count(x => x.Disposition == FailedRecordDisposition.AlreadyMissing),
|
||||
RemovedItemCount = changedItems.Count,
|
||||
AffectedTaskCount = affectedTaskIds.Count,
|
||||
SkippedChangedCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.Changed),
|
||||
SkippedExcludedCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.PermanentlyExcluded),
|
||||
InvalidSnapshotCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.InvalidSnapshot),
|
||||
Message = $"已删除 {deletedRecords} 条视频记录,更新 {changedItems.Count} 条迁移失败项;旧文件已保留,将随系统后续正常同步重新发现。"
|
||||
};
|
||||
});
|
||||
if (!transaction.IsSuccess)
|
||||
throw transaction.ErrorException ?? new InvalidOperationException("删除迁移失败记录失败,数据库操作已回滚。");
|
||||
return result;
|
||||
}
|
||||
|
||||
public Task PauseAsync(string id) => ChangeStatusAsync(id,
|
||||
new[] { StorageMigrationTaskStatus.Queued, StorageMigrationTaskStatus.Running }, StorageMigrationTaskStatus.Paused, null);
|
||||
|
||||
public async Task ResumeAsync(string id)
|
||||
{
|
||||
var task = await RequireTaskAsync(id);
|
||||
if (task.Status != StorageMigrationTaskStatus.Paused) throw new InvalidOperationException("只有暂停中的任务可以恢复。");
|
||||
var fingerprint = StorageConfigurationFingerprint.Create(await _settingsService.GetAsync());
|
||||
if (!string.Equals(fingerprint, task.ConfigurationFingerprint, StringComparison.Ordinal))
|
||||
throw new InvalidOperationException("OpenList 地址、源挂载、基础目录或账号已变化,请重新预检并创建新任务。");
|
||||
await ChangeStatusAsync(id, new[] { StorageMigrationTaskStatus.Paused }, StorageMigrationTaskStatus.Queued, null);
|
||||
}
|
||||
|
||||
public Task CancelAsync(string id) => ChangeStatusAsync(id,
|
||||
new[] { StorageMigrationTaskStatus.Queued, StorageMigrationTaskStatus.Running, StorageMigrationTaskStatus.Paused },
|
||||
StorageMigrationTaskStatus.Cancelled, null);
|
||||
|
||||
public async Task RetryFailedAsync(string id)
|
||||
{
|
||||
var task = await RequireTaskAsync(id);
|
||||
if (task.Status is not (StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled or StorageMigrationTaskStatus.Paused))
|
||||
throw new InvalidOperationException("当前任务没有可重试的失败项。");
|
||||
var fingerprint = StorageConfigurationFingerprint.Create(await _settingsService.GetAsync());
|
||||
if (fingerprint != task.ConfigurationFingerprint) throw new InvalidOperationException("OpenList 配置已变化,请重新预检。");
|
||||
await _db.Updateable<StorageMigrationItem>()
|
||||
.SetColumns(x => new StorageMigrationItem { Stage = StorageMigrationItemStage.Pending, ErrorMessage = null, UpdatedAt = DateTime.Now })
|
||||
.Where(x => x.TaskId == id && x.Stage == StorageMigrationItemStage.Failed).ExecuteCommandAsync();
|
||||
await _db.Updateable<StorageMigrationTask>()
|
||||
.SetColumns(x => new StorageMigrationTask { Status = StorageMigrationTaskStatus.Queued, CompletedAt = null, ErrorMessage = null, UpdatedAt = DateTime.Now })
|
||||
.Where(x => x.Id == id).ExecuteCommandAsync();
|
||||
await RefreshCountsAsync(id);
|
||||
}
|
||||
|
||||
public async Task CleanupAsync(string id, CancellationToken cancellationToken)
|
||||
{
|
||||
var task = await RequireTaskAsync(id);
|
||||
if (task.Status is not (StorageMigrationTaskStatus.Completed or StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled))
|
||||
throw new InvalidOperationException("只有已结束的任务可以清理旧文件。");
|
||||
var targetStorage = await BindTaskTargetAsync(task);
|
||||
await _db.Updateable<StorageMigrationTask>()
|
||||
.SetColumns(x => new StorageMigrationTask { Status = StorageMigrationTaskStatus.Cleaning, ErrorMessage = null, UpdatedAt = DateTime.Now })
|
||||
.Where(x => x.Id == id).ExecuteCommandAsync();
|
||||
|
||||
var items = await _db.Queryable<StorageMigrationItem>().Where(x => x.TaskId == id
|
||||
&& (x.Stage == StorageMigrationItemStage.Succeeded || x.Stage == StorageMigrationItemStage.SucceededWithWarnings)).ToListAsync();
|
||||
var errors = new List<string>();
|
||||
foreach (var item in items)
|
||||
{
|
||||
try { await CleanupItemAsync(item, targetStorage, cancellationToken); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
errors.Add($"{item.VideoId}: {ex.Message}");
|
||||
item.ErrorMessage = "清理失败:" + ex.Message;
|
||||
item.UpdatedAt = DateTime.Now;
|
||||
await _db.Updateable(item).ExecuteCommandAsync();
|
||||
}
|
||||
}
|
||||
|
||||
var remaining = await _db.Queryable<StorageMigrationItem>().Where(x => x.TaskId == id
|
||||
&& (x.Stage == StorageMigrationItemStage.Succeeded || x.Stage == StorageMigrationItemStage.SucceededWithWarnings)).CountAsync();
|
||||
await _db.Updateable<StorageMigrationTask>()
|
||||
.SetColumns(x => new StorageMigrationTask
|
||||
{
|
||||
Status = remaining == 0 ? StorageMigrationTaskStatus.Cleaned : StorageMigrationTaskStatus.PartiallyFailed,
|
||||
CleanedAt = remaining == 0 ? DateTime.Now : null,
|
||||
ErrorMessage = errors.Count == 0 ? null : string.Join(";", errors.Take(20)),
|
||||
UpdatedAt = DateTime.Now
|
||||
}).Where(x => x.Id == id).ExecuteCommandAsync();
|
||||
await RefreshCountsAsync(id);
|
||||
}
|
||||
|
||||
public async Task RollbackAsync(string id, CancellationToken cancellationToken)
|
||||
{
|
||||
var task = await RequireTaskAsync(id);
|
||||
if (task.Status is not (StorageMigrationTaskStatus.Completed or StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled))
|
||||
throw new InvalidOperationException("当前任务状态不允许回滚。");
|
||||
if (task.CleanedAt.HasValue || await _db.Queryable<StorageMigrationItem>().Where(x => x.TaskId == id && x.Stage == StorageMigrationItemStage.Cleaned).AnyAsync())
|
||||
throw new InvalidOperationException("旧文件清理已开始,不能再执行回滚。");
|
||||
var targetStorage = await BindTaskTargetAsync(task);
|
||||
|
||||
var items = await _db.Queryable<StorageMigrationItem>().Where(x => x.TaskId == id
|
||||
&& (x.Stage == StorageMigrationItemStage.Succeeded || x.Stage == StorageMigrationItemStage.SucceededWithWarnings)).ToListAsync();
|
||||
var snapshots = items.ToDictionary(x => x.Id, x => JsonConvert.DeserializeObject<DouyinVideo>(x.OldSnapshotJson));
|
||||
foreach (var pair in snapshots)
|
||||
{
|
||||
var snapshot = pair.Value ?? throw new InvalidOperationException("旧视频快照损坏,无法回滚。");
|
||||
var item = items.First(x => x.Id == pair.Key);
|
||||
var sourceType = item.SourceStorageType ?? snapshot.StorageType;
|
||||
if (sourceType == MediaStorageType.Local)
|
||||
{
|
||||
var cookie = await _db.Queryable<DouyinCookie>().InSingleAsync(snapshot.CookieId);
|
||||
if (!SafeLocalMigrationFile.TryResolve(snapshot.VideoSavePath,
|
||||
StorageMigrationPathPolicy.GetLocalRoots(cookie), out var local, out var error)
|
||||
|| !File.Exists(local) || new FileInfo(local).Length <= 0)
|
||||
throw new InvalidOperationException($"视频 {snapshot.AwemeId} 的旧主媒体不可用:{error ?? "文件不存在或为空"}");
|
||||
}
|
||||
else if (sourceType == MediaStorageType.WebDav)
|
||||
{
|
||||
var length = await targetStorage.GetLengthAsync(snapshot.VideoSavePath, cancellationToken);
|
||||
if (!length.HasValue || length <= 0
|
||||
|| snapshot.FileSize > 0 && length.Value != snapshot.FileSize)
|
||||
throw new InvalidOperationException($"视频 {snapshot.AwemeId} 的原 WebDAV 文件已不可用,无法恢复旧记录。");
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new InvalidOperationException($"视频 {snapshot.AwemeId} 的旧存储类型不支持回滚。");
|
||||
}
|
||||
}
|
||||
|
||||
var transaction = await _db.Ado.UseTranAsync(async () =>
|
||||
{
|
||||
foreach (var item in items)
|
||||
{
|
||||
var snapshot = snapshots[item.Id];
|
||||
var current = await _db.Queryable<DouyinVideo>().InSingleAsync(item.VideoId)
|
||||
?? throw new InvalidOperationException($"视频记录 {item.VideoId} 已不存在");
|
||||
if (current.StorageType != MediaStorageType.OpenList || current.VideoSavePath != item.TargetVideoPath)
|
||||
throw new InvalidOperationException($"视频 {snapshot.AwemeId} 已被其他操作修改,无法回滚");
|
||||
if (await _db.Updateable(snapshot).ExecuteCommandAsync() != 1)
|
||||
throw new InvalidOperationException($"恢复视频 {snapshot.AwemeId} 失败");
|
||||
item.Stage = StorageMigrationItemStage.RolledBack;
|
||||
item.UpdatedAt = DateTime.Now;
|
||||
item.CompletedAt = DateTime.Now;
|
||||
await _db.Updateable(item).ExecuteCommandAsync();
|
||||
}
|
||||
});
|
||||
if (!transaction.IsSuccess) throw transaction.ErrorException ?? new InvalidOperationException("回滚数据库事务失败");
|
||||
|
||||
var warnings = new List<string>();
|
||||
foreach (var item in items)
|
||||
{
|
||||
var snapshot = snapshots[item.Id];
|
||||
if ((item.SourceStorageType ?? snapshot.StorageType) == MediaStorageType.WebDav)
|
||||
continue;
|
||||
foreach (var remotePath in DeserializePaths(item.OwnedRemotePathsJson))
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!await IsRemotePathReferencedAsync(remotePath, cancellationToken)) await targetStorage.DeleteAsync(remotePath, cancellationToken);
|
||||
}
|
||||
catch (Exception ex) { warnings.Add($"{remotePath}: {ex.Message}"); }
|
||||
}
|
||||
}
|
||||
await _db.Updateable<StorageMigrationTask>()
|
||||
.SetColumns(x => new StorageMigrationTask
|
||||
{
|
||||
Status = StorageMigrationTaskStatus.RolledBack,
|
||||
RolledBackAt = DateTime.Now,
|
||||
ErrorMessage = warnings.Count == 0 ? null : "远端独占文件清理警告:" + string.Join(";", warnings.Take(20)),
|
||||
UpdatedAt = DateTime.Now
|
||||
}).Where(x => x.Id == id).ExecuteCommandAsync();
|
||||
await RefreshCountsAsync(id);
|
||||
}
|
||||
|
||||
private async Task CleanupItemAsync(StorageMigrationItem item, IMediaStorage targetStorage, CancellationToken cancellationToken)
|
||||
{
|
||||
var current = await _db.Queryable<DouyinVideo>().InSingleAsync(item.VideoId)
|
||||
?? throw new InvalidOperationException("视频记录不存在");
|
||||
if (current.StorageType != MediaStorageType.OpenList || current.VideoSavePath != item.TargetVideoPath)
|
||||
throw new InvalidOperationException("数据库已不再指向本任务的 OpenList 目标");
|
||||
var remoteLength = await targetStorage.GetLengthAsync(current.VideoSavePath, cancellationToken);
|
||||
if (!remoteLength.HasValue || remoteLength.Value != item.ExpectedLength)
|
||||
throw new InvalidOperationException("远端主媒体不存在或长度已变化");
|
||||
|
||||
var snapshot = JsonConvert.DeserializeObject<DouyinVideo>(item.OldSnapshotJson)
|
||||
?? throw new InvalidOperationException("旧视频快照损坏");
|
||||
if ((item.SourceStorageType ?? snapshot.StorageType) == MediaStorageType.WebDav)
|
||||
{
|
||||
item.Stage = StorageMigrationItemStage.Cleaned;
|
||||
item.ErrorMessage = null;
|
||||
item.WarningMessage = string.IsNullOrWhiteSpace(item.WarningMessage)
|
||||
? "原 WebDAV 记录与 OpenList 指向同一文件,无需删除媒体。"
|
||||
: item.WarningMessage + ";原 WebDAV 记录与 OpenList 指向同一文件,无需删除媒体。";
|
||||
item.UpdatedAt = DateTime.Now;
|
||||
item.CompletedAt = DateTime.Now;
|
||||
await _db.Updateable(item).ExecuteCommandAsync();
|
||||
return;
|
||||
}
|
||||
var cookie = await _db.Queryable<DouyinCookie>().InSingleAsync(snapshot.CookieId);
|
||||
var roots = StorageMigrationPathPolicy.GetLocalRoots(cookie);
|
||||
var candidates = BuildLocalCleanupCandidates(snapshot);
|
||||
var localVideos = await _db.Queryable<DouyinVideo>().Where(x => x.StorageType == MediaStorageType.Local).ToListAsync();
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(candidate.Path) || !File.Exists(candidate.Path)) continue;
|
||||
if (!SafeLocalMigrationFile.TryResolve(candidate.Path, roots, out var safePath, out var error))
|
||||
throw new InvalidOperationException($"拒绝清理不安全路径 {candidate.Path}:{error}");
|
||||
if (IsLocalPathReferenced(candidate.Path, localVideos, snapshot.Id, candidate.Shared)) continue;
|
||||
File.Delete(safePath);
|
||||
}
|
||||
DeleteEmptyParents(snapshot.VideoSavePath, roots);
|
||||
item.Stage = StorageMigrationItemStage.Cleaned;
|
||||
item.ErrorMessage = null;
|
||||
item.UpdatedAt = DateTime.Now;
|
||||
item.CompletedAt = DateTime.Now;
|
||||
await _db.Updateable(item).ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
private static List<CleanupCandidate> BuildLocalCleanupCandidates(DouyinVideo snapshot)
|
||||
{
|
||||
var result = new List<CleanupCandidate>
|
||||
{
|
||||
new(snapshot.VideoSavePath, false),
|
||||
new(snapshot.VideoCoverSavePath, snapshot.ViedoType is VideoTypeEnum.dy_mix or VideoTypeEnum.dy_series),
|
||||
new(snapshot.AuthorAvatar, true)
|
||||
};
|
||||
if (!string.IsNullOrWhiteSpace(snapshot.VideoSavePath))
|
||||
{
|
||||
var directory = Path.GetDirectoryName(snapshot.VideoSavePath);
|
||||
var basename = Path.GetFileNameWithoutExtension(snapshot.VideoSavePath);
|
||||
result.Add(new CleanupCandidate(Path.Combine(directory ?? string.Empty, basename + ".nfo"), false));
|
||||
if (snapshot.ViedoType is VideoTypeEnum.dy_mix or VideoTypeEnum.dy_series)
|
||||
result.Add(new CleanupCandidate(Path.Combine(directory ?? string.Empty, "tvshow.nfo"), true));
|
||||
}
|
||||
if (!string.IsNullOrWhiteSpace(snapshot.DynamicVideos))
|
||||
{
|
||||
try
|
||||
{
|
||||
foreach (var attachment in JsonConvert.DeserializeObject<List<DouyinMergeVideoDto>>(snapshot.DynamicVideos) ?? new())
|
||||
result.Add(new CleanupCandidate(attachment.Path, false));
|
||||
}
|
||||
catch (JsonException) { }
|
||||
}
|
||||
return result.Where(x => !string.IsNullOrWhiteSpace(x.Path)).DistinctBy(x => x.Path).ToList();
|
||||
}
|
||||
|
||||
private static bool IsLocalPathReferenced(string path, IEnumerable<DouyinVideo> localVideos, string excludedVideoId, bool shared)
|
||||
{
|
||||
foreach (var video in localVideos.Where(x => x.Id != excludedVideoId))
|
||||
{
|
||||
if (path == video.VideoSavePath || path == video.VideoCoverSavePath || path == video.AuthorAvatar) return true;
|
||||
if (shared && !string.IsNullOrWhiteSpace(video.VideoSavePath)
|
||||
&& string.Equals(Path.GetDirectoryName(video.VideoSavePath), Path.GetDirectoryName(path), StringComparison.Ordinal)) return true;
|
||||
if (!string.IsNullOrWhiteSpace(video.DynamicVideos) && video.DynamicVideos.Contains(path, StringComparison.Ordinal)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private async Task<bool> IsRemotePathReferencedAsync(string path, CancellationToken cancellationToken)
|
||||
{
|
||||
var videos = await _db.Queryable<DouyinVideo>().Where(x => x.StorageType.IsRemote()).ToListAsync();
|
||||
foreach (var video in videos)
|
||||
{
|
||||
if (video.VideoSavePath == path || video.VideoCoverSavePath == path || video.AuthorAvatar == path) return true;
|
||||
if (!string.IsNullOrWhiteSpace(video.DynamicVideos) && video.DynamicVideos.Contains(path, StringComparison.Ordinal)) return true;
|
||||
if (NfoContentBuilder.Build(video).Keys.Contains(path)) return true;
|
||||
if (Path.GetFileName(path).Equals("tvshow.nfo", StringComparison.OrdinalIgnoreCase)
|
||||
&& StoragePath.DirectoryName(video.VideoSavePath) == StoragePath.DirectoryName(path)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static void DeleteEmptyParents(string filePath, IReadOnlyList<string> roots)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(filePath)) return;
|
||||
var directory = Path.GetDirectoryName(Path.GetFullPath(filePath));
|
||||
var root = roots.Where(x => SafeLocalMigrationFile.IsWithin(filePath, x)).OrderByDescending(x => x.Length).FirstOrDefault();
|
||||
while (!string.IsNullOrWhiteSpace(directory) && !string.Equals(directory, root, StringComparison.Ordinal)
|
||||
&& Directory.Exists(directory) && !Directory.EnumerateFileSystemEntries(directory).Any())
|
||||
{
|
||||
Directory.Delete(directory, false);
|
||||
directory = Path.GetDirectoryName(directory);
|
||||
}
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> DeserializePaths(string json)
|
||||
{
|
||||
try { return JsonConvert.DeserializeObject<List<string>>(json) ?? new(); }
|
||||
catch { return Array.Empty<string>(); }
|
||||
}
|
||||
|
||||
public async Task RefreshCountsAsync(string taskId)
|
||||
{
|
||||
var items = await _db.Queryable<StorageMigrationItem>().Where(x => x.TaskId == taskId).ToListAsync();
|
||||
var pending = items.Count(i => i.Stage == StorageMigrationItemStage.Pending || i.Stage == StorageMigrationItemStage.Uploading
|
||||
|| i.Stage == StorageMigrationItemStage.Verifying || i.Stage == StorageMigrationItemStage.Committing);
|
||||
var success = items.Count(i => i.Stage == StorageMigrationItemStage.Succeeded || i.Stage == StorageMigrationItemStage.SucceededWithWarnings
|
||||
|| i.Stage == StorageMigrationItemStage.Cleaned);
|
||||
var warning = items.Count(i => i.Stage == StorageMigrationItemStage.SucceededWithWarnings);
|
||||
var failed = items.Count(i => i.Stage == StorageMigrationItemStage.Failed);
|
||||
var removed = items.Count(i => i.Stage == StorageMigrationItemStage.RecordRemoved);
|
||||
var now = DateTime.Now;
|
||||
await _db.Updateable<StorageMigrationTask>()
|
||||
.SetColumns(x => new StorageMigrationTask
|
||||
{
|
||||
PendingCount = pending,
|
||||
SuccessCount = success,
|
||||
WarningCount = warning,
|
||||
FailedCount = failed,
|
||||
RemovedCount = removed,
|
||||
UpdatedAt = now
|
||||
}).Where(x => x.Id == taskId).ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
private async Task<FailedRecordRemovalPlan> BuildFailedRecordRemovalPlanAsync()
|
||||
{
|
||||
var activeMigrationCount = await _db.Queryable<StorageMigrationTask>().Where(x =>
|
||||
x.Status == StorageMigrationTaskStatus.Queued || x.Status == StorageMigrationTaskStatus.Running
|
||||
|| x.Status == StorageMigrationTaskStatus.Paused || x.Status == StorageMigrationTaskStatus.Cleaning).CountAsync();
|
||||
var terminalTaskIds = await _db.Queryable<StorageMigrationTask>().Where(x =>
|
||||
x.Status == StorageMigrationTaskStatus.Completed || x.Status == StorageMigrationTaskStatus.PartiallyFailed
|
||||
|| x.Status == StorageMigrationTaskStatus.Cancelled || x.Status == StorageMigrationTaskStatus.Cleaned
|
||||
|| x.Status == StorageMigrationTaskStatus.RolledBack).Select(x => x.Id).ToListAsync();
|
||||
var failedItems = new List<StorageMigrationItem>();
|
||||
foreach (var taskIdBatch in terminalTaskIds.Chunk(500))
|
||||
failedItems.AddRange(await _db.Queryable<StorageMigrationItem>().Where(x => taskIdBatch.Contains(x.TaskId)
|
||||
&& x.Stage == StorageMigrationItemStage.Failed).ToListAsync());
|
||||
var groups = failedItems.GroupBy(x => x.VideoId).OrderBy(x => x.Key, StringComparer.Ordinal)
|
||||
.Select(x => new FailedRecordGroup { VideoId = x.Key, Items = x.OrderBy(i => i.Id, StringComparer.Ordinal).ToList() })
|
||||
.ToList();
|
||||
|
||||
foreach (var group in groups)
|
||||
{
|
||||
foreach (var item in group.Items)
|
||||
{
|
||||
try
|
||||
{
|
||||
var snapshot = JsonConvert.DeserializeObject<DouyinVideo>(item.OldSnapshotJson);
|
||||
if (snapshot != null) group.Snapshots.Add(snapshot);
|
||||
}
|
||||
catch (JsonException) { }
|
||||
}
|
||||
}
|
||||
|
||||
var videoIds = groups.Select(x => x.VideoId).Where(x => !string.IsNullOrWhiteSpace(x)).Distinct().ToList();
|
||||
var currentVideosById = new List<DouyinVideo>();
|
||||
foreach (var videoIdBatch in videoIds.Chunk(500))
|
||||
currentVideosById.AddRange(await _db.Queryable<DouyinVideo>().Where(x => videoIdBatch.Contains(x.Id)).ToListAsync());
|
||||
var currentById = currentVideosById.ToDictionary(x => x.Id, StringComparer.Ordinal);
|
||||
var snapshotAwemeIds = groups.SelectMany(x => x.Snapshots).Select(x => x.AwemeId)
|
||||
.Where(x => !string.IsNullOrWhiteSpace(x)).Distinct().ToList();
|
||||
var currentVideosByAweme = new List<DouyinVideo>();
|
||||
var excludedAwemeIds = new HashSet<string>(StringComparer.Ordinal);
|
||||
foreach (var awemeIdBatch in snapshotAwemeIds.Chunk(500))
|
||||
{
|
||||
currentVideosByAweme.AddRange(await _db.Queryable<DouyinVideo>()
|
||||
.Where(x => awemeIdBatch.Contains(x.AwemeId)).ToListAsync());
|
||||
excludedAwemeIds.UnionWith(await _db.Queryable<DouyinVideoDelete>()
|
||||
.Where(x => awemeIdBatch.Contains(x.ViedoId)).Select(x => x.ViedoId).ToListAsync());
|
||||
}
|
||||
var currentByAweme = currentVideosByAweme.GroupBy(x => x.AwemeId)
|
||||
.ToDictionary(x => x.Key, x => x.ToList(), StringComparer.Ordinal);
|
||||
|
||||
foreach (var group in groups)
|
||||
{
|
||||
var awemeIds = group.Snapshots.Select(x => x.AwemeId).Where(x => !string.IsNullOrWhiteSpace(x))
|
||||
.Distinct(StringComparer.Ordinal).ToList();
|
||||
if (group.Snapshots.Count == 0 || awemeIds.Count != 1
|
||||
|| group.Snapshots.All(x => !string.Equals(x.Id, group.VideoId, StringComparison.Ordinal)))
|
||||
{
|
||||
group.Disposition = FailedRecordDisposition.InvalidSnapshot;
|
||||
continue;
|
||||
}
|
||||
|
||||
group.AwemeId = awemeIds[0];
|
||||
if (excludedAwemeIds.Contains(group.AwemeId))
|
||||
{
|
||||
group.Disposition = FailedRecordDisposition.PermanentlyExcluded;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!currentById.TryGetValue(group.VideoId, out var current))
|
||||
{
|
||||
group.Disposition = currentByAweme.TryGetValue(group.AwemeId, out var replacements)
|
||||
&& replacements.Any(x => x.Id != group.VideoId)
|
||||
? FailedRecordDisposition.Changed
|
||||
: FailedRecordDisposition.AlreadyMissing;
|
||||
continue;
|
||||
}
|
||||
|
||||
group.Current = current;
|
||||
var matchesSnapshot = group.Snapshots.Any(snapshot => current.StorageType == snapshot.StorageType
|
||||
&& string.Equals(current.AwemeId, group.AwemeId, StringComparison.Ordinal)
|
||||
&& string.Equals(snapshot.Id, current.Id, StringComparison.Ordinal)
|
||||
&& string.Equals(snapshot.AwemeId, current.AwemeId, StringComparison.Ordinal)
|
||||
&& string.Equals(snapshot.VideoSavePath, current.VideoSavePath, StringComparison.Ordinal));
|
||||
group.Disposition = matchesSnapshot ? FailedRecordDisposition.Eligible : FailedRecordDisposition.Changed;
|
||||
}
|
||||
|
||||
var tokenSource = string.Join("\n", groups.SelectMany(group => group.Items.Select(item =>
|
||||
$"{item.Id}|{item.TaskId}|{item.VideoId}|{item.UpdatedAt.Ticks}|{(int)group.Disposition}|{group.AwemeId}|{group.Current?.Id}|{(int?)(group.Current?.StorageType)}|{group.Current?.VideoSavePath}")));
|
||||
var token = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(tokenSource))).ToLowerInvariant();
|
||||
return new FailedRecordRemovalPlan
|
||||
{
|
||||
ActiveMigrationCount = activeMigrationCount,
|
||||
Groups = groups,
|
||||
ConfirmationToken = token
|
||||
};
|
||||
}
|
||||
|
||||
private async Task ChangeStatusAsync(string id, StorageMigrationTaskStatus[] allowed, StorageMigrationTaskStatus status, string error)
|
||||
{
|
||||
var task = await RequireTaskAsync(id);
|
||||
if (!allowed.Contains(task.Status)) throw new InvalidOperationException($"任务状态 {task.Status} 不允许此操作。");
|
||||
await _db.Updateable<StorageMigrationTask>()
|
||||
.SetColumns(x => new StorageMigrationTask { Status = status, ErrorMessage = error, UpdatedAt = DateTime.Now })
|
||||
.Where(x => x.Id == id).ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
private async Task<StorageMigrationTask> RequireTaskAsync(string id) =>
|
||||
await _db.Queryable<StorageMigrationTask>().InSingleAsync(id) ?? throw new KeyNotFoundException("迁移任务不存在");
|
||||
|
||||
private async Task<IMediaStorage> BindTaskTargetAsync(StorageMigrationTask task)
|
||||
{
|
||||
if ((task.TargetStorageType ?? MediaStorageType.WebDav) != MediaStorageType.OpenList)
|
||||
throw new InvalidOperationException("旧 WebDAV 迁移任务只保留审计信息,不能再执行写入、清理或回滚。");
|
||||
var settings = await _settingsService.GetAsync();
|
||||
if (!string.Equals(StorageConfigurationFingerprint.Create(settings), task.ConfigurationFingerprint, StringComparison.Ordinal))
|
||||
throw new InvalidOperationException("OpenList 地址、源挂载、基础目录或账号已变化,请恢复任务原目标后再操作。");
|
||||
return _openList.Bind(settings);
|
||||
}
|
||||
|
||||
private async Task<List<PlanningEntry>> BuildPlanningAsync()
|
||||
{
|
||||
var videos = await _db.Queryable<DouyinVideo>()
|
||||
.Where(x => x.StorageType != MediaStorageType.OpenList)
|
||||
.OrderBy(x => x.CreateTime).ToListAsync();
|
||||
var cookies = (await _db.Queryable<DouyinCookie>().ToListAsync()).ToDictionary(x => x.Id ?? string.Empty, StringComparer.Ordinal);
|
||||
var categories = (await _db.Queryable<DouyinCollectCate>().ToListAsync()).ToDictionary(x => x.Id ?? string.Empty, StringComparer.Ordinal);
|
||||
var followed = await _db.Queryable<DouyinFollowed>().ToListAsync();
|
||||
var config = _commonService.GetConfig() ?? new AppConfig();
|
||||
var episodes = videos.Where(x => x.ViedoType is VideoTypeEnum.dy_mix or VideoTypeEnum.dy_series)
|
||||
.GroupBy(x => x.CateId ?? x.CateXId ?? string.Empty)
|
||||
.SelectMany(group => group.OrderBy(x => x.CreateTime).ThenBy(x => x.Id).Select((video, index) => new { video.Id, Episode = index + 1 }))
|
||||
.ToDictionary(x => x.Id, x => x.Episode);
|
||||
var result = new List<PlanningEntry>(videos.Count);
|
||||
var openListSettings = await _settingsService.GetAsync();
|
||||
var targetStorage = _openList.Bind(openListSettings);
|
||||
foreach (var video in videos)
|
||||
{
|
||||
cookies.TryGetValue(video.CookieId ?? string.Empty, out var cookie);
|
||||
categories.TryGetValue(video.CateId ?? string.Empty, out var category);
|
||||
var follow = followed.FirstOrDefault(x => x.UperId == video.AuthorId);
|
||||
StorageMigrationPathPlan plan = null;
|
||||
string planError = null;
|
||||
try
|
||||
{
|
||||
plan = video.StorageType == MediaStorageType.WebDav
|
||||
? new StorageMigrationPathPlan
|
||||
{
|
||||
VideoPath = StoragePath.NormalizeRemote(video.VideoSavePath),
|
||||
CoverPath = string.IsNullOrWhiteSpace(video.VideoCoverSavePath)
|
||||
? string.Empty : StoragePath.NormalizeRemote(video.VideoCoverSavePath),
|
||||
AvatarPath = string.IsNullOrWhiteSpace(video.AuthorAvatar)
|
||||
? string.Empty : StoragePath.NormalizeRemote(video.AuthorAvatar)
|
||||
}
|
||||
: StorageMigrationPathPolicy.Build(video, cookie, category, follow, config,
|
||||
episodes.GetValueOrDefault(video.Id, 1));
|
||||
}
|
||||
catch (Exception ex) { planError = $"视频 {video.AwemeId ?? video.Id}:{ex.Message}"; }
|
||||
|
||||
var roots = StorageMigrationPathPolicy.GetLocalRoots(cookie);
|
||||
var readable = SafeLocalMigrationFile.TryResolve(video.VideoSavePath, roots, out var fullPath, out _)
|
||||
&& new FileInfo(fullPath).Length > 0;
|
||||
long? existingTargetLength = null;
|
||||
if (plan != null && video.StorageType == MediaStorageType.WebDav)
|
||||
{
|
||||
try { existingTargetLength = await targetStorage.GetLengthAsync(plan.VideoPath); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
planError = $"视频 {video.AwemeId ?? video.Id} OpenList 接管检测失败:{ex.GetBaseException().Message}";
|
||||
}
|
||||
}
|
||||
result.Add(new PlanningEntry
|
||||
{
|
||||
Video = video,
|
||||
Cookie = cookie,
|
||||
Category = category,
|
||||
Followed = follow,
|
||||
Plan = plan,
|
||||
PlanError = planError,
|
||||
LocalRoots = roots,
|
||||
LocalReadable = readable,
|
||||
LocalLength = readable ? new FileInfo(fullPath).Length : 0,
|
||||
AdoptExistingTarget = existingTargetLength is > 0
|
||||
&& (video.FileSize <= 0 || existingTargetLength.Value == video.FileSize)
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static List<MigrationAttachmentPlan> BuildAttachments(PlanningEntry entry)
|
||||
{
|
||||
var attachments = new List<MigrationAttachmentPlan>();
|
||||
Add(attachments, "cover", entry.Video.VideoCoverSavePath, entry.Video.VideoCoverUrl, entry.Plan.CoverPath, false);
|
||||
Add(attachments, "avatar", entry.Video.AuthorAvatar, entry.Video.AuthorAvatarUrl, entry.Plan.AvatarPath, true);
|
||||
if (!string.IsNullOrWhiteSpace(entry.Video.DynamicVideos))
|
||||
{
|
||||
try
|
||||
{
|
||||
var dynamicItems = JsonConvert.DeserializeObject<List<DouyinMergeVideoDto>>(entry.Video.DynamicVideos) ?? new();
|
||||
var index = 0;
|
||||
foreach (var item in dynamicItems)
|
||||
{
|
||||
index++;
|
||||
var oldName = Path.GetFileName(item.Path);
|
||||
var name = string.IsNullOrWhiteSpace(oldName) ? $"attachment-{index:D3}.bin" : oldName;
|
||||
Add(attachments, "dynamic", item.Path, Uri.IsWellFormedUriString(item.Path, UriKind.Absolute) ? item.Path : null,
|
||||
StoragePath.CombineRemote(entry.Plan.DirectoryPath, name), false, item.Height, item.Width);
|
||||
}
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
Serilog.Log.Warning(ex, "解析迁移附件失败:{VideoId}", entry.Video.Id);
|
||||
}
|
||||
}
|
||||
return attachments;
|
||||
}
|
||||
|
||||
private static void Add(List<MigrationAttachmentPlan> list, string kind, string source, string sourceUrl, string target, bool shared, int height = 0, int width = 0)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(target)) return;
|
||||
list.Add(new MigrationAttachmentPlan { Kind = kind, SourcePath = source, SourceUrl = sourceUrl, TargetPath = target, Shared = shared, Height = height, Width = width });
|
||||
}
|
||||
|
||||
private static StorageMigrationTaskDetail ToDetail(StorageMigrationTask task)
|
||||
{
|
||||
var isOpenListTask = task.TargetStorageType == MediaStorageType.OpenList;
|
||||
return new StorageMigrationTaskDetail
|
||||
{
|
||||
Task = task,
|
||||
StatusText = task.Status switch
|
||||
{
|
||||
StorageMigrationTaskStatus.Queued => "排队",
|
||||
StorageMigrationTaskStatus.Running => "运行中",
|
||||
StorageMigrationTaskStatus.Paused => "已暂停",
|
||||
StorageMigrationTaskStatus.Completed => "已完成",
|
||||
StorageMigrationTaskStatus.PartiallyFailed => "部分失败",
|
||||
StorageMigrationTaskStatus.Cancelled => "已取消",
|
||||
StorageMigrationTaskStatus.Cleaning => "清理中",
|
||||
StorageMigrationTaskStatus.Cleaned => "已清理",
|
||||
StorageMigrationTaskStatus.RolledBack => "已回滚",
|
||||
_ => task.Status.ToString()
|
||||
},
|
||||
CurrentFile = task.CurrentFile,
|
||||
CanPause = isOpenListTask && task.Status is (StorageMigrationTaskStatus.Queued or StorageMigrationTaskStatus.Running),
|
||||
CanResume = isOpenListTask && task.Status == StorageMigrationTaskStatus.Paused,
|
||||
CanCancel = isOpenListTask && task.Status is (StorageMigrationTaskStatus.Queued or StorageMigrationTaskStatus.Running or StorageMigrationTaskStatus.Paused),
|
||||
CanRetryFailed = isOpenListTask && task.FailedCount > 0
|
||||
&& task.Status is (StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled or StorageMigrationTaskStatus.Paused),
|
||||
CanCleanup = isOpenListTask && task.SuccessCount > 0
|
||||
&& task.Status is (StorageMigrationTaskStatus.Completed or StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled),
|
||||
CanRollback = isOpenListTask && task.SuccessCount > 0 && !task.CleanedAt.HasValue
|
||||
&& task.Status is (StorageMigrationTaskStatus.Completed or StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled),
|
||||
CanArchive = CanArchive(task)
|
||||
};
|
||||
}
|
||||
|
||||
private static bool CanArchive(StorageMigrationTask task) => task != null && !task.IsArchived
|
||||
&& IsTerminal(task.Status)
|
||||
&& (task.TargetStorageType != MediaStorageType.OpenList
|
||||
|| task.Status is StorageMigrationTaskStatus.Cleaned or StorageMigrationTaskStatus.RolledBack
|
||||
|| task.SuccessCount == 0);
|
||||
|
||||
private static bool IsTerminal(StorageMigrationTaskStatus status) => status is
|
||||
StorageMigrationTaskStatus.Completed or StorageMigrationTaskStatus.PartiallyFailed
|
||||
or StorageMigrationTaskStatus.Cancelled or StorageMigrationTaskStatus.Cleaned
|
||||
or StorageMigrationTaskStatus.RolledBack;
|
||||
|
||||
private static string FormatBytes(long value)
|
||||
{
|
||||
string[] units = { "B", "KB", "MB", "GB", "TB" };
|
||||
var size = (double)value;
|
||||
var unit = 0;
|
||||
while (size >= 1024 && unit < units.Length - 1) { size /= 1024; unit++; }
|
||||
return $"{size:0.##} {units[unit]}";
|
||||
}
|
||||
|
||||
private sealed class PlanningEntry
|
||||
{
|
||||
public DouyinVideo Video { get; set; }
|
||||
public DouyinCookie Cookie { get; set; }
|
||||
public DouyinCollectCate Category { get; set; }
|
||||
public DouyinFollowed Followed { get; set; }
|
||||
public StorageMigrationPathPlan Plan { get; set; }
|
||||
public string PlanError { get; set; }
|
||||
public IReadOnlyList<string> LocalRoots { get; set; }
|
||||
public bool LocalReadable { get; set; }
|
||||
public long LocalLength { get; set; }
|
||||
public bool AdoptExistingTarget { get; set; }
|
||||
}
|
||||
|
||||
internal sealed class MigrationAttachmentPlan
|
||||
{
|
||||
public string Kind { get; set; }
|
||||
public string SourcePath { get; set; }
|
||||
public string SourceUrl { get; set; }
|
||||
public string TargetPath { get; set; }
|
||||
public bool Shared { get; set; }
|
||||
public int Height { get; set; }
|
||||
public int Width { get; set; }
|
||||
}
|
||||
|
||||
private enum FailedRecordDisposition
|
||||
{
|
||||
Eligible,
|
||||
AlreadyMissing,
|
||||
Changed,
|
||||
PermanentlyExcluded,
|
||||
InvalidSnapshot
|
||||
}
|
||||
|
||||
private sealed class FailedRecordRemovalPlan
|
||||
{
|
||||
public int ActiveMigrationCount { get; set; }
|
||||
public List<FailedRecordGroup> Groups { get; set; } = new();
|
||||
public string ConfirmationToken { get; set; }
|
||||
}
|
||||
|
||||
private sealed class FailedRecordGroup
|
||||
{
|
||||
public string VideoId { get; set; }
|
||||
public string AwemeId { get; set; }
|
||||
public DouyinVideo Current { get; set; }
|
||||
public List<DouyinVideo> Snapshots { get; set; } = new();
|
||||
public List<StorageMigrationItem> Items { get; set; } = new();
|
||||
public FailedRecordDisposition Disposition { get; set; }
|
||||
}
|
||||
|
||||
private sealed record CleanupCandidate(string Path, bool Shared);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
using dy.net.model.dto;
|
||||
using dy.net.model.entity;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using SqlSugar;
|
||||
using MediaStorageType = dy.net.model.dto.StorageType;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
public class StorageMigrationWorker : BackgroundService
|
||||
{
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
private readonly ILogger<StorageMigrationWorker> _logger;
|
||||
|
||||
public StorageMigrationWorker(IServiceScopeFactory scopeFactory, ILogger<StorageMigrationWorker> logger)
|
||||
{
|
||||
_scopeFactory = scopeFactory;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
await RecoverInterruptedAsync();
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
var worked = await RunOneBatchAsync(stoppingToken);
|
||||
if (!worked) await Task.Delay(TimeSpan.FromSeconds(2), stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { }
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "存储迁移后台服务发生错误");
|
||||
await Task.Delay(TimeSpan.FromSeconds(3), stoppingToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RecoverInterruptedAsync()
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<ISqlSugarClient>();
|
||||
await db.Updateable<StorageMigrationTask>()
|
||||
.SetColumns(x => new StorageMigrationTask
|
||||
{
|
||||
Status = StorageMigrationTaskStatus.Cancelled,
|
||||
ErrorMessage = "旧 WebDAV 迁移任务已停止,仅保留审计信息;请创建新的 OpenList 接管任务。",
|
||||
CurrentFile = null,
|
||||
CurrentVideoId = null,
|
||||
CompletedAt = DateTime.Now,
|
||||
UpdatedAt = DateTime.Now
|
||||
})
|
||||
.Where(x => x.TargetStorageType == null
|
||||
&& (x.Status == StorageMigrationTaskStatus.Queued
|
||||
|| x.Status == StorageMigrationTaskStatus.Running
|
||||
|| x.Status == StorageMigrationTaskStatus.Paused
|
||||
|| x.Status == StorageMigrationTaskStatus.Cleaning))
|
||||
.ExecuteCommandAsync();
|
||||
|
||||
var running = await db.Queryable<StorageMigrationTask>().Where(x =>
|
||||
x.TargetStorageType == MediaStorageType.OpenList
|
||||
&& x.Status == StorageMigrationTaskStatus.Running).ToListAsync();
|
||||
foreach (var task in running)
|
||||
{
|
||||
await db.Updateable<StorageMigrationItem>()
|
||||
.SetColumns(x => new StorageMigrationItem { Stage = StorageMigrationItemStage.Pending, UpdatedAt = DateTime.Now })
|
||||
.Where(x => x.TaskId == task.Id && (x.Stage == StorageMigrationItemStage.Uploading
|
||||
|| x.Stage == StorageMigrationItemStage.Verifying || x.Stage == StorageMigrationItemStage.Committing))
|
||||
.ExecuteCommandAsync();
|
||||
await db.Updateable<StorageMigrationTask>()
|
||||
.SetColumns(x => new StorageMigrationTask { Status = StorageMigrationTaskStatus.Queued, CurrentFile = null, CurrentVideoId = null, UpdatedAt = DateTime.Now })
|
||||
.Where(x => x.Id == task.Id).ExecuteCommandAsync();
|
||||
}
|
||||
await db.Updateable<StorageMigrationTask>()
|
||||
.SetColumns(x => new StorageMigrationTask
|
||||
{
|
||||
Status = StorageMigrationTaskStatus.PartiallyFailed,
|
||||
ErrorMessage = "旧文件清理被应用重启中断,请确认后重试清理。",
|
||||
UpdatedAt = DateTime.Now
|
||||
})
|
||||
.Where(x => x.Status == StorageMigrationTaskStatus.Cleaning)
|
||||
.ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
private async Task<bool> RunOneBatchAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
string taskId;
|
||||
int concurrency;
|
||||
List<string> itemIds;
|
||||
using (var scope = _scopeFactory.CreateScope())
|
||||
{
|
||||
var db = scope.ServiceProvider.GetRequiredService<ISqlSugarClient>();
|
||||
var task = await db.Queryable<StorageMigrationTask>()
|
||||
.Where(x => x.TargetStorageType == MediaStorageType.OpenList
|
||||
&& (x.Status == StorageMigrationTaskStatus.Queued || x.Status == StorageMigrationTaskStatus.Running))
|
||||
.OrderBy(x => x.CreatedAt).FirstAsync();
|
||||
if (task == null) return false;
|
||||
taskId = task.Id;
|
||||
concurrency = Math.Clamp(task.Concurrency, 1, 3);
|
||||
if (task.Status == StorageMigrationTaskStatus.Queued)
|
||||
{
|
||||
await db.Updateable<StorageMigrationTask>()
|
||||
.SetColumns(x => new StorageMigrationTask { Status = StorageMigrationTaskStatus.Running, StartedAt = DateTime.Now, UpdatedAt = DateTime.Now })
|
||||
.Where(x => x.Id == task.Id && x.Status == StorageMigrationTaskStatus.Queued).ExecuteCommandAsync();
|
||||
}
|
||||
itemIds = await db.Queryable<StorageMigrationItem>().Where(x => x.TaskId == task.Id && x.Stage == StorageMigrationItemStage.Pending)
|
||||
.OrderBy(x => x.CreatedAt).Take(concurrency).Select(x => x.Id).ToListAsync();
|
||||
}
|
||||
|
||||
if (itemIds.Count == 0)
|
||||
{
|
||||
await CompleteTaskAsync(taskId);
|
||||
return true;
|
||||
}
|
||||
|
||||
await Task.WhenAll(itemIds.Select(itemId => ProcessInScopeAsync(taskId, itemId, cancellationToken)));
|
||||
using (var scope = _scopeFactory.CreateScope())
|
||||
await scope.ServiceProvider.GetRequiredService<StorageMigrationService>().RefreshCountsAsync(taskId);
|
||||
return true;
|
||||
}
|
||||
|
||||
private async Task ProcessInScopeAsync(string taskId, string itemId, CancellationToken cancellationToken)
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<ISqlSugarClient>();
|
||||
var item = await db.Queryable<StorageMigrationItem>().InSingleAsync(itemId);
|
||||
var videoId = item == null ? null : item.VideoId;
|
||||
var oldVideoPath = item == null ? null : item.OldVideoPath;
|
||||
var now = DateTime.Now;
|
||||
await db.Updateable<StorageMigrationTask>()
|
||||
.SetColumns(x => new StorageMigrationTask { CurrentVideoId = videoId, CurrentFile = oldVideoPath, UpdatedAt = now })
|
||||
.Where(x => x.Id == taskId && x.Status == StorageMigrationTaskStatus.Running).ExecuteCommandAsync();
|
||||
await scope.ServiceProvider.GetRequiredService<StorageMigrationItemProcessor>().ProcessAsync(itemId, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task CompleteTaskAsync(string taskId)
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<ISqlSugarClient>();
|
||||
var task = await db.Queryable<StorageMigrationTask>().InSingleAsync(taskId);
|
||||
if (task?.Status != StorageMigrationTaskStatus.Running) return;
|
||||
var failed = await db.Queryable<StorageMigrationItem>().Where(x => x.TaskId == taskId && x.Stage == StorageMigrationItemStage.Failed).CountAsync();
|
||||
await db.Updateable<StorageMigrationTask>()
|
||||
.SetColumns(x => new StorageMigrationTask
|
||||
{
|
||||
Status = failed == 0 ? StorageMigrationTaskStatus.Completed : StorageMigrationTaskStatus.PartiallyFailed,
|
||||
CurrentVideoId = null,
|
||||
CurrentFile = null,
|
||||
CompletedAt = DateTime.Now,
|
||||
UpdatedAt = DateTime.Now
|
||||
}).Where(x => x.Id == taskId && x.Status == StorageMigrationTaskStatus.Running).ExecuteCommandAsync();
|
||||
await scope.ServiceProvider.GetRequiredService<StorageMigrationService>().RefreshCountsAsync(taskId);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
using dy.net.model.dto;
|
||||
using dy.net.model.entity;
|
||||
using Newtonsoft.Json;
|
||||
using SqlSugar;
|
||||
using MediaStorageType = dy.net.model.dto.StorageType;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
public class VideoExclusionService
|
||||
{
|
||||
private readonly ISqlSugarClient _db;
|
||||
private readonly VideoTaskService _tasks;
|
||||
|
||||
public VideoExclusionService(ISqlSugarClient db, VideoTaskService tasks)
|
||||
{
|
||||
_db = db;
|
||||
_tasks = tasks;
|
||||
}
|
||||
|
||||
public async Task<VideoExclusionPage> GetPageAsync(VideoExclusionPageRequest request)
|
||||
{
|
||||
request ??= new VideoExclusionPageRequest();
|
||||
var page = Math.Max(1, request.PageIndex);
|
||||
var size = Math.Clamp(request.PageSize, 1, 100);
|
||||
var query = _db.Queryable<DouyinVideoDelete>()
|
||||
.WhereIF(!string.IsNullOrWhiteSpace(request.Keyword), x => x.VideoTitle.Contains(request.Keyword)
|
||||
|| x.ViedoId.Contains(request.Keyword) || x.Author.Contains(request.Keyword));
|
||||
return new VideoExclusionPage
|
||||
{
|
||||
TotalCount = await query.CountAsync(),
|
||||
Items = await query.OrderByDescending(x => x.DeleteTime).Skip((page - 1) * size).Take(size).ToListAsync()
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<UnexcludeVideosResult> UnexcludeAsync(UnexcludeVideosRequest request)
|
||||
{
|
||||
if (request?.Ids == null || request.Ids.Count == 0) throw new InvalidOperationException("请选择要取消排除的视频。");
|
||||
var exclusions = await _db.Queryable<DouyinVideoDelete>().Where(x => request.Ids.Contains(x.Id)).ToListAsync();
|
||||
if (exclusions.Count == 0) throw new KeyNotFoundException("未找到永久排除记录。");
|
||||
var awemeIds = exclusions.Select(x => x.ViedoId).Where(x => !string.IsNullOrWhiteSpace(x)).Distinct().ToList();
|
||||
var allDuplicates = awemeIds.Count == 0
|
||||
? exclusions
|
||||
: await _db.Queryable<DouyinVideoDelete>().Where(x => awemeIds.Contains(x.ViedoId)).ToListAsync();
|
||||
var snapshots = request.CreateDownloadTask
|
||||
? allDuplicates.Select(TryGetSnapshot).Where(x => x != null).GroupBy(x => x.AwemeId).Select(x => x.First()).ToList()
|
||||
: new List<DouyinVideo>();
|
||||
VideoDownloadTask task = null;
|
||||
var result = new UnexcludeVideosResult();
|
||||
var transaction = await _db.Ado.UseTranAsync(async () =>
|
||||
{
|
||||
if (request.CreateDownloadTask && snapshots.Count > 0)
|
||||
{
|
||||
task = await _tasks.CreateTaskAsync(VideoTaskType.ExclusionRestore, VideoTaskTrigger.UserAction,
|
||||
$"取消排除恢复({snapshots.Count} 条)", storageType: snapshots[0].StorageType);
|
||||
foreach (var video in snapshots)
|
||||
{
|
||||
var exclusion = allDuplicates.First(x => x.ViedoId == video.AwemeId);
|
||||
await _tasks.AddItemAsync(task.Id, video.AwemeId, video.CookieId, null, video.ViedoType,
|
||||
video.VideoTitle, video.Author, video.VideoSavePath,
|
||||
new[] { video.VideoUrl }, video, video.FileSize, workerManaged: true);
|
||||
}
|
||||
}
|
||||
await _db.Deleteable<DouyinVideoDelete>()
|
||||
.Where(x => request.Ids.Contains(x.Id) || awemeIds.Contains(x.ViedoId)).ExecuteCommandAsync();
|
||||
var now = DateTime.Now;
|
||||
if (awemeIds.Count > 0)
|
||||
{
|
||||
var relatedTaskId = task?.Id;
|
||||
await _db.Updateable<VideoDownloadTaskItem>()
|
||||
.SetColumns(x => new VideoDownloadTaskItem
|
||||
{
|
||||
ExclusionReleasedAt = now,
|
||||
RelatedTaskId = relatedTaskId,
|
||||
UpdatedAt = now
|
||||
}).Where(x => awemeIds.Contains(x.AwemeId) && x.SkipReason == VideoTaskSkipReason.PermanentlyExcluded)
|
||||
.ExecuteCommandAsync();
|
||||
}
|
||||
});
|
||||
if (!transaction.IsSuccess) throw new InvalidOperationException("取消排除失败:" + transaction.ErrorMessage);
|
||||
|
||||
result.ReleasedCount = awemeIds.Count;
|
||||
result.QueuedCount = snapshots.Count;
|
||||
result.TaskId = task?.Id;
|
||||
result.CannotQueueIds = request.CreateDownloadTask
|
||||
? awemeIds.Where(id => snapshots.All(x => x.AwemeId != id)).ToList()
|
||||
: new List<string>();
|
||||
result.Message = result.CannotQueueIds.Count > 0
|
||||
? $"已取消排除 {result.ReleasedCount} 条,其中 {result.CannotQueueIds.Count} 条旧记录资料不足,将在下次正常同步时重新发现。"
|
||||
: task == null
|
||||
? $"已取消排除 {result.ReleasedCount} 条,将在下次正常同步时重新发现。"
|
||||
: $"已取消排除并创建 {result.QueuedCount} 条恢复下载任务。";
|
||||
return result;
|
||||
}
|
||||
|
||||
private static DouyinVideo TryGetSnapshot(DouyinVideoDelete exclusion)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(exclusion.RestoreSnapshotJson))
|
||||
{
|
||||
try
|
||||
{
|
||||
var snapshot = JsonConvert.DeserializeObject<DouyinVideo>(exclusion.RestoreSnapshotJson);
|
||||
if (snapshot != null && !string.IsNullOrWhiteSpace(snapshot.AwemeId) && !string.IsNullOrWhiteSpace(snapshot.CookieId)) return snapshot;
|
||||
}
|
||||
catch { }
|
||||
}
|
||||
if (string.IsNullOrWhiteSpace(exclusion.ViedoId) || string.IsNullOrWhiteSpace(exclusion.CookieId)
|
||||
|| !exclusion.VideoType.HasValue || string.IsNullOrWhiteSpace(exclusion.VideoUrl)) return null;
|
||||
return new DouyinVideo
|
||||
{
|
||||
Id = Guid.NewGuid().ToString("N"),
|
||||
AwemeId = exclusion.ViedoId,
|
||||
CookieId = exclusion.CookieId,
|
||||
ViedoType = exclusion.VideoType.Value,
|
||||
VideoTitle = exclusion.VideoTitle,
|
||||
VideoSavePath = exclusion.VideoSavePath,
|
||||
VideoUrl = exclusion.VideoUrl,
|
||||
AuthorId = exclusion.AuthorId,
|
||||
Author = exclusion.Author,
|
||||
StorageType = MediaStorageType.Local,
|
||||
SyncTime = DateTime.Now
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
using dy.net.model.dto;
|
||||
using dy.net.model.entity;
|
||||
using dy.net.storage;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using SqlSugar;
|
||||
using MediaStorageType = dy.net.model.dto.StorageType;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
/// <summary>
|
||||
/// Replaces only the main media after a complete new download. The old row and file remain valid on failure.
|
||||
/// </summary>
|
||||
public class VideoRedownloadWorker : BackgroundService
|
||||
{
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
|
||||
public VideoRedownloadWorker(IServiceScopeFactory scopeFactory) => _scopeFactory = scopeFactory;
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<ISqlSugarClient>();
|
||||
var waitingTaskIds = await db.Queryable<VideoDownloadTask>()
|
||||
.Where(x => x.Status == VideoTaskStatus.WaitingForStorage)
|
||||
.Select(x => x.Id).ToListAsync();
|
||||
var pendingJobs = await db.Queryable<DouyinReDownload>()
|
||||
.Where(x => x.Status == 0).OrderBy(x => x.CreateTime).ToListAsync();
|
||||
var job = pendingJobs.FirstOrDefault(x => string.IsNullOrWhiteSpace(x.TaskId)
|
||||
|| !waitingTaskIds.Contains(x.TaskId));
|
||||
if (job == null)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken);
|
||||
continue;
|
||||
}
|
||||
await ProcessAsync(scope.ServiceProvider, job, stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { }
|
||||
catch (Exception ex)
|
||||
{
|
||||
Serilog.Log.Error(ex, "安全重新下载后台任务异常");
|
||||
await Task.Delay(TimeSpan.FromSeconds(3), stoppingToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task ProcessAsync(IServiceProvider services, DouyinReDownload job, CancellationToken cancellationToken)
|
||||
{
|
||||
var db = services.GetRequiredService<ISqlSugarClient>();
|
||||
var tasks = services.GetRequiredService<VideoTaskService>();
|
||||
if (!string.IsNullOrWhiteSpace(job.TaskId))
|
||||
{
|
||||
var parent = await db.Queryable<VideoDownloadTask>().InSingleAsync(job.TaskId);
|
||||
if (parent?.Status is VideoTaskStatus.WaitingForStorage or VideoTaskStatus.WaitingForSource)
|
||||
{
|
||||
if (parent.Status == VideoTaskStatus.WaitingForStorage) return;
|
||||
}
|
||||
try
|
||||
{
|
||||
await tasks.EnsureTaskStorageAvailableAsync(job.TaskId);
|
||||
}
|
||||
catch (InvalidOperationException ex)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(job.TaskItemId))
|
||||
{
|
||||
var blockedItem = await db.Queryable<VideoDownloadTaskItem>().InSingleAsync(job.TaskItemId);
|
||||
if (blockedItem != null)
|
||||
{
|
||||
blockedItem.Stage = VideoTaskItemStage.WaitingForStorage;
|
||||
blockedItem.ErrorType = VideoTaskErrorType.StorageUnavailable;
|
||||
blockedItem.ErrorMessage = ex.Message;
|
||||
blockedItem.UpdatedAt = DateTime.Now;
|
||||
await db.Updateable(blockedItem).ExecuteCommandAsync();
|
||||
}
|
||||
}
|
||||
await tasks.BlockTaskForStorageAsync(job.TaskId, ex.Message);
|
||||
await tasks.RefreshCountsAsync(job.TaskId);
|
||||
return;
|
||||
}
|
||||
}
|
||||
var http = services.GetRequiredService<DouyinHttpClientService>();
|
||||
var resolver = services.GetRequiredService<DouyinMigrationSourceResolver>();
|
||||
var video = !string.IsNullOrWhiteSpace(job.VideoRecordId)
|
||||
? await db.Queryable<DouyinVideo>().InSingleAsync(job.VideoRecordId)
|
||||
: await db.Queryable<DouyinVideo>().Where(x => x.AwemeId == job.ViedoId).FirstAsync();
|
||||
if (video == null)
|
||||
{
|
||||
await FinishAsync(db, job, false, "原视频记录不存在");
|
||||
if (!string.IsNullOrWhiteSpace(job.TaskItemId))
|
||||
await tasks.MarkFailedAsync(job.TaskItemId, new InvalidOperationException("原视频记录不存在"), VideoTaskErrorType.SourceUnavailable);
|
||||
await CompleteTaskWhenIdleAsync(db, tasks, job.TaskId);
|
||||
return;
|
||||
}
|
||||
var cookie = await db.Queryable<DouyinCookie>().InSingleAsync(video.CookieId);
|
||||
if (!string.IsNullOrWhiteSpace(job.TaskItemId))
|
||||
{
|
||||
var tracked = await db.Queryable<VideoDownloadTaskItem>().InSingleAsync(job.TaskItemId);
|
||||
var access = tracked?.Stage == VideoTaskItemStage.WaitingForSource
|
||||
? await tasks.TryAcquireSourceAsync(video.CookieId, tracked.Id)
|
||||
: await tasks.GetSourceAccessAsync(video.CookieId);
|
||||
if (!access.Allowed)
|
||||
{
|
||||
await tasks.WaitItemForSourceAsync(job.TaskItemId, video.CookieId, access.Message);
|
||||
return;
|
||||
}
|
||||
if (tracked?.Stage == VideoTaskItemStage.WaitingForSource)
|
||||
{
|
||||
tracked.Stage = VideoTaskItemStage.Pending;
|
||||
tracked.ErrorMessage = null;
|
||||
tracked.ErrorType = VideoTaskErrorType.None;
|
||||
tracked.UpdatedAt = DateTime.Now;
|
||||
await db.Updateable(tracked).ExecuteCommandAsync();
|
||||
}
|
||||
}
|
||||
if (!string.IsNullOrWhiteSpace(job.TaskId)) await tasks.StartTaskAsync(job.TaskId);
|
||||
if (!string.IsNullOrWhiteSpace(job.TaskItemId))
|
||||
await tasks.SetItemStageAsync(job.TaskItemId, VideoTaskItemStage.Downloading, job.SavePath);
|
||||
var urls = new List<string>();
|
||||
if (!string.IsNullOrWhiteSpace(video.VideoUrl)) urls.Add(video.VideoUrl);
|
||||
foreach (var url in await resolver.ResolveAsync(video, cookie, cancellationToken))
|
||||
if (!urls.Contains(url)) urls.Add(url);
|
||||
if (urls.Count == 0)
|
||||
{
|
||||
await FinishAsync(db, job, false, "记录 URL 与抖音全量列表均无可用地址");
|
||||
if (!string.IsNullOrWhiteSpace(job.TaskItemId))
|
||||
await tasks.MarkFailedAsync(job.TaskItemId, new InvalidOperationException("记录 URL 与抖音全量列表均无可用地址"), VideoTaskErrorType.SourceUnavailable);
|
||||
await CompleteTaskWhenIdleAsync(db, tasks, job.TaskId);
|
||||
return;
|
||||
}
|
||||
|
||||
job.Attempts++;
|
||||
job.UpdateTime = DateTime.UtcNow;
|
||||
await db.Updateable(job).ExecuteCommandAsync();
|
||||
try
|
||||
{
|
||||
long length;
|
||||
if (video.StorageType == MediaStorageType.Local)
|
||||
{
|
||||
var target = Path.GetFullPath(video.VideoSavePath);
|
||||
var directory = Path.GetDirectoryName(target) ?? throw new InvalidOperationException("原视频目录无效");
|
||||
Directory.CreateDirectory(directory);
|
||||
var temporary = Path.Combine(directory, "." + Path.GetFileName(target) + ".redownload-" + Guid.NewGuid().ToString("N"));
|
||||
try
|
||||
{
|
||||
var downloaded = await http.DownloadAsync(urls[0], temporary, cookie?.Cookies, urls.Skip(1).ToList(), cancellationToken, maxRetryCount: Math.Min(12, urls.Count));
|
||||
if (!downloaded.Success)
|
||||
{
|
||||
var disposition = SourceFailureDisposition.Continue;
|
||||
if (!string.IsNullOrWhiteSpace(job.TaskItemId))
|
||||
disposition = await tasks.MarkDownloadFailedAsync(job.TaskItemId, video.CookieId, downloaded);
|
||||
if (disposition == SourceFailureDisposition.Continue)
|
||||
await FinishAsync(db, job, false, downloaded.Message);
|
||||
await CompleteTaskWhenIdleAsync(db, tasks, job.TaskId);
|
||||
return;
|
||||
}
|
||||
await tasks.RecordSourceSuccessAsync(video.CookieId);
|
||||
if (!File.Exists(downloaded.ActualSavePath)) throw new MediaStorageException("重新下载完成后临时文件不存在。");
|
||||
length = new FileInfo(downloaded.ActualSavePath).Length;
|
||||
if (length <= 0) throw new IOException("重新下载文件为空");
|
||||
File.Move(downloaded.ActualSavePath, target, true);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(temporary)) File.Delete(temporary);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
var downloaded = await http.DownloadToStorageAsync(video.StorageType, urls[0], video.VideoSavePath,
|
||||
cookie?.Cookies, urls.Skip(1).ToList(), cancellationToken, Math.Min(12, urls.Count));
|
||||
if (!downloaded.Success)
|
||||
{
|
||||
var disposition = SourceFailureDisposition.Continue;
|
||||
if (!string.IsNullOrWhiteSpace(job.TaskItemId))
|
||||
disposition = await tasks.MarkDownloadFailedAsync(job.TaskItemId, video.CookieId, downloaded);
|
||||
if (disposition == SourceFailureDisposition.Continue)
|
||||
await FinishAsync(db, job, false, downloaded.Message);
|
||||
await CompleteTaskWhenIdleAsync(db, tasks, job.TaskId);
|
||||
return;
|
||||
}
|
||||
await tasks.RecordSourceSuccessAsync(video.CookieId);
|
||||
length = await services.GetRequiredService<MediaStorageRouter>()
|
||||
.Resolve(video.StorageType).GetLengthAsync(video.VideoSavePath, cancellationToken) ?? 0;
|
||||
if (length <= 0) throw new IOException($"{video.StorageType} 替换后的文件为空");
|
||||
}
|
||||
if (!string.IsNullOrWhiteSpace(job.TaskItemId))
|
||||
await tasks.SetItemStageAsync(job.TaskItemId, VideoTaskItemStage.Verifying, video.VideoSavePath);
|
||||
if (length <= 0) throw new IOException("重新下载后的文件为空");
|
||||
if (!string.IsNullOrWhiteSpace(job.TaskItemId))
|
||||
await tasks.SetItemStageAsync(job.TaskItemId, VideoTaskItemStage.Committing, video.VideoSavePath);
|
||||
video.FileSize = length;
|
||||
video.SyncTime = DateTime.Now;
|
||||
await db.Updateable(video).ExecuteCommandAsync();
|
||||
await FinishAsync(db, job, true, null);
|
||||
if (!string.IsNullOrWhiteSpace(job.TaskItemId)) await tasks.MarkSucceededAsync(job.TaskItemId, length, video.Id);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Serilog.Log.Warning(ex, "安全重新下载失败,旧记录和旧文件保持不变:{VideoId}", video.Id);
|
||||
await FinishAsync(db, job, false, ex.Message);
|
||||
if (!string.IsNullOrWhiteSpace(job.TaskItemId)) await tasks.MarkFailedAsync(job.TaskItemId, ex);
|
||||
}
|
||||
await CompleteTaskWhenIdleAsync(db, tasks, job.TaskId);
|
||||
}
|
||||
|
||||
private static async Task CompleteTaskWhenIdleAsync(ISqlSugarClient db, VideoTaskService tasks, string taskId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(taskId)) return;
|
||||
var pending = await db.Queryable<DouyinReDownload>().Where(x => x.TaskId == taskId && x.Status == 0).CountAsync();
|
||||
if (pending == 0) await tasks.CompleteTaskAsync(taskId);
|
||||
}
|
||||
|
||||
private static async Task FinishAsync(ISqlSugarClient db, DouyinReDownload job, bool success, string error)
|
||||
{
|
||||
job.Status = success ? 1 : 2;
|
||||
job.ErrorMessage = error;
|
||||
job.UpdateTime = DateTime.UtcNow;
|
||||
if (success) job.DownTime = DateTime.UtcNow;
|
||||
await db.Updateable(job).ExecuteCommandAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,222 @@
|
||||
using dy.net.model.dto;
|
||||
using dy.net.model.entity;
|
||||
using dy.net.storage;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Newtonsoft.Json;
|
||||
using SqlSugar;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
public class VideoTaskWorker : BackgroundService
|
||||
{
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
|
||||
public VideoTaskWorker(IServiceScopeFactory scopeFactory) => _scopeFactory = scopeFactory;
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
using (var scope = _scopeFactory.CreateScope())
|
||||
{
|
||||
var tasks = scope.ServiceProvider.GetRequiredService<VideoTaskService>();
|
||||
await tasks.RecoverInterruptedAsync();
|
||||
await BackfillLegacyRedownloadsAsync(scope.ServiceProvider);
|
||||
await tasks.CleanupHistoryAsync();
|
||||
}
|
||||
var lastCleanup = DateTime.UtcNow;
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
var worked = await ProcessOneAsync(stoppingToken);
|
||||
if (DateTime.UtcNow - lastCleanup > TimeSpan.FromDays(1))
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
await scope.ServiceProvider.GetRequiredService<VideoTaskService>().CleanupHistoryAsync();
|
||||
lastCleanup = DateTime.UtcNow;
|
||||
}
|
||||
if (!worked) await Task.Delay(TimeSpan.FromSeconds(2), stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { }
|
||||
catch (Exception ex)
|
||||
{
|
||||
Serilog.Log.Error(ex, "视频任务后台服务异常");
|
||||
await Task.Delay(TimeSpan.FromSeconds(3), stoppingToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<bool> ProcessOneAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<ISqlSugarClient>();
|
||||
var tasks = scope.ServiceProvider.GetRequiredService<VideoTaskService>();
|
||||
var candidates = await db.Queryable<VideoDownloadTaskItem>()
|
||||
.Where(x => x.WorkerManaged && (x.Stage == VideoTaskItemStage.Pending
|
||||
|| x.Stage == VideoTaskItemStage.WaitingForSource))
|
||||
.OrderBy(x => x.CreatedAt).Take(50).ToListAsync();
|
||||
VideoDownloadTaskItem item = null;
|
||||
var isSourceProbe = false;
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
var access = candidate.Stage == VideoTaskItemStage.WaitingForSource
|
||||
? await tasks.TryAcquireSourceAsync(candidate.CookieId, candidate.Id)
|
||||
: await tasks.GetSourceAccessAsync(candidate.CookieId);
|
||||
if (access.Allowed)
|
||||
{
|
||||
item = candidate;
|
||||
isSourceProbe = access.IsProbe;
|
||||
if (item.Stage == VideoTaskItemStage.WaitingForSource)
|
||||
{
|
||||
item.Stage = VideoTaskItemStage.Pending;
|
||||
item.ErrorMessage = null;
|
||||
item.ErrorType = VideoTaskErrorType.None;
|
||||
item.RetryAfter = null;
|
||||
item.UpdatedAt = DateTime.Now;
|
||||
await db.Updateable(item).ExecuteCommandAsync();
|
||||
}
|
||||
break;
|
||||
}
|
||||
if (candidate.Stage == VideoTaskItemStage.Pending)
|
||||
await tasks.WaitItemForSourceAsync(candidate.Id, candidate.CookieId, access.Message);
|
||||
}
|
||||
if (item == null) return false;
|
||||
var task = await db.Queryable<VideoDownloadTask>().InSingleAsync(item.TaskId);
|
||||
if (task == null || task.Type == VideoTaskType.Redownload) return false;
|
||||
try
|
||||
{
|
||||
await tasks.EnsureTaskStorageAvailableAsync(task.Id);
|
||||
}
|
||||
catch (InvalidOperationException ex)
|
||||
{
|
||||
item.Stage = VideoTaskItemStage.WaitingForStorage;
|
||||
item.ErrorType = VideoTaskErrorType.StorageUnavailable;
|
||||
item.ErrorMessage = ex.Message;
|
||||
item.UpdatedAt = DateTime.Now;
|
||||
await db.Updateable(item).ExecuteCommandAsync();
|
||||
await tasks.BlockTaskForStorageAsync(task.Id, ex.Message);
|
||||
await tasks.RefreshCountsAsync(task.Id);
|
||||
return true;
|
||||
}
|
||||
await tasks.StartTaskAsync(task.Id);
|
||||
try
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(item.RetrySnapshotJson))
|
||||
throw new InvalidOperationException("该失败条目缺少可恢复快照,请重新执行对应的同步任务。");
|
||||
var video = JsonConvert.DeserializeObject<DouyinVideo>(item.RetrySnapshotJson)
|
||||
?? throw new InvalidOperationException("任务缺少视频恢复快照。");
|
||||
var cookie = await db.Queryable<DouyinCookie>().InSingleAsync(video.CookieId)
|
||||
?? throw new InvalidOperationException("任务所属抖音授权不存在。");
|
||||
if (string.IsNullOrWhiteSpace(cookie.Cookies)) throw new InvalidOperationException("Cookie 无效,无法恢复下载。");
|
||||
var storedUrls = DeserializeUrls(item.SourceUrlsJson);
|
||||
if (!string.IsNullOrWhiteSpace(video.VideoUrl) && !storedUrls.Contains(video.VideoUrl)) storedUrls.Insert(0, video.VideoUrl);
|
||||
var resolvedUrls = (await scope.ServiceProvider.GetRequiredService<DouyinMigrationSourceResolver>()
|
||||
.ResolveAsync(video, cookie, cancellationToken)).ToList();
|
||||
var urls = (isSourceProbe ? resolvedUrls.Concat(storedUrls) : storedUrls.Concat(resolvedUrls))
|
||||
.Where(x => !string.IsNullOrWhiteSpace(x)).Distinct(StringComparer.Ordinal).Take(12).ToList();
|
||||
if (urls.Count == 0) throw new InvalidOperationException("作品没有可用下载地址,可能已下架或设为私密。");
|
||||
await tasks.SetItemStageAsync(item.Id, VideoTaskItemStage.Downloading, video.VideoSavePath);
|
||||
var storage = scope.ServiceProvider.GetRequiredService<MediaStorageRouter>().Resolve(video.StorageType);
|
||||
var existingLength = await storage.GetLengthAsync(video.VideoSavePath, cancellationToken);
|
||||
if (!existingLength.HasValue || existingLength <= 0)
|
||||
{
|
||||
var downloaded = await scope.ServiceProvider.GetRequiredService<DouyinHttpClientService>()
|
||||
.DownloadToStorageAsync(video.StorageType, urls[0], video.VideoSavePath, cookie.Cookies,
|
||||
urls.Skip(1).ToList(), cancellationToken, Math.Min(12, urls.Count));
|
||||
if (!downloaded.Success)
|
||||
{
|
||||
await tasks.MarkDownloadFailedAsync(item.Id, cookie.Id, downloaded);
|
||||
await CompleteWhenIdleAsync(db, tasks, task.Id);
|
||||
return true;
|
||||
}
|
||||
video.VideoSavePath = downloaded.ActualSavePath;
|
||||
await tasks.RecordSourceSuccessAsync(cookie.Id);
|
||||
}
|
||||
await tasks.SetItemStageAsync(item.Id, VideoTaskItemStage.Verifying, video.VideoSavePath);
|
||||
var length = await storage.GetLengthAsync(video.VideoSavePath, cancellationToken) ?? 0;
|
||||
if (length <= 0) throw new IOException("下载后的主媒体不存在或为空。");
|
||||
if (item.ExpectedLength > 0 && length != item.ExpectedLength)
|
||||
throw new IOException($"下载长度不一致:期望 {item.ExpectedLength},实际 {length}。");
|
||||
await tasks.SetItemStageAsync(item.Id, VideoTaskItemStage.Committing, video.VideoSavePath);
|
||||
video.FileSize = length;
|
||||
video.SyncTime = DateTime.Now;
|
||||
if (string.IsNullOrWhiteSpace(video.Id)) video.Id = Guid.NewGuid().ToString("N");
|
||||
await db.Storageable(video).ExecuteCommandAsync();
|
||||
var warning = "恢复任务仅保证主媒体,封面、头像或 NFO 将在后续同步补齐。";
|
||||
var cleanupPending = false;
|
||||
string cleanupError = null;
|
||||
if (video.SupersededStorageSnapshot != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
await scope.ServiceProvider.GetRequiredService<DouyinVideoService>()
|
||||
.CleanupSupersededLocalArtifactsAsync(video.SupersededStorageSnapshot, video.VideoSavePath);
|
||||
}
|
||||
catch (Exception cleanupException)
|
||||
{
|
||||
cleanupPending = true;
|
||||
cleanupError = cleanupException.GetBaseException().Message;
|
||||
warning += $" 旧本地文件清理失败,可在确认远端文件后稍后重试:{cleanupError}";
|
||||
}
|
||||
}
|
||||
await tasks.MarkSucceededAsync(item.Id, length, video.Id, warning, cleanupPending, cleanupError);
|
||||
await CompleteWhenIdleAsync(db, tasks, task.Id);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await tasks.MarkFailedAsync(item.Id, ex);
|
||||
await CompleteWhenIdleAsync(db, tasks, task.Id);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private static async Task CompleteWhenIdleAsync(ISqlSugarClient db, VideoTaskService tasks, string taskId)
|
||||
{
|
||||
var remaining = await db.Queryable<VideoDownloadTaskItem>().Where(x => x.TaskId == taskId
|
||||
&& (x.Stage == VideoTaskItemStage.Pending || x.Stage == VideoTaskItemStage.Downloading
|
||||
|| x.Stage == VideoTaskItemStage.Verifying || x.Stage == VideoTaskItemStage.Committing
|
||||
|| x.Stage == VideoTaskItemStage.WaitingForSource)).CountAsync();
|
||||
if (remaining == 0) await tasks.CompleteTaskAsync(taskId);
|
||||
}
|
||||
|
||||
private static List<string> DeserializeUrls(string json)
|
||||
{
|
||||
try { return JsonConvert.DeserializeObject<List<string>>(json) ?? new(); }
|
||||
catch { return new(); }
|
||||
}
|
||||
|
||||
private static async Task BackfillLegacyRedownloadsAsync(IServiceProvider services)
|
||||
{
|
||||
var db = services.GetRequiredService<ISqlSugarClient>();
|
||||
var legacy = await db.Queryable<DouyinReDownload>().Where(x => string.IsNullOrEmpty(x.TaskId)).ToListAsync();
|
||||
if (legacy.Count == 0) return;
|
||||
var tasks = services.GetRequiredService<VideoTaskService>();
|
||||
var task = await tasks.CreateTaskAsync(VideoTaskType.Redownload, VideoTaskTrigger.UpgradeRecovery,
|
||||
$"升级前重新下载({legacy.Count} 条)");
|
||||
foreach (var job in legacy)
|
||||
{
|
||||
var video = !string.IsNullOrWhiteSpace(job.VideoRecordId)
|
||||
? await db.Queryable<DouyinVideo>().InSingleAsync(job.VideoRecordId)
|
||||
: await db.Queryable<DouyinVideo>().Where(x => x.AwemeId == job.ViedoId).FirstAsync();
|
||||
var item = await tasks.AddItemAsync(task.Id, job.ViedoId, job.CookieId, null, video?.ViedoType,
|
||||
video?.VideoTitle, video?.Author, job.SavePath, new[] { video?.VideoUrl }, video, video?.FileSize ?? 0);
|
||||
item.Stage = job.Status switch
|
||||
{
|
||||
1 => VideoTaskItemStage.Succeeded,
|
||||
2 => VideoTaskItemStage.Failed,
|
||||
_ => VideoTaskItemStage.Pending
|
||||
};
|
||||
item.ErrorMessage = job.ErrorMessage;
|
||||
item.ErrorType = job.Status == 2 ? VideoTaskErrorType.Unknown : VideoTaskErrorType.None;
|
||||
item.CompletedAt = job.Status == 0 ? null : job.UpdateTime;
|
||||
item.UpdatedAt = job.UpdateTime == default ? DateTime.Now : job.UpdateTime;
|
||||
await db.Updateable(item).ExecuteCommandAsync();
|
||||
job.TaskId = task.Id;
|
||||
job.TaskItemId = item.Id;
|
||||
}
|
||||
await db.Updateable(legacy).ExecuteCommandAsync();
|
||||
await tasks.RefreshCountsAsync(task.Id);
|
||||
var pending = legacy.Count(x => x.Status == 0);
|
||||
if (pending == 0) await tasks.CompleteTaskAsync(task.Id);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
using dy.net.model.dto;
|
||||
using dy.net.model.entity;
|
||||
using Microsoft.AspNetCore.DataProtection;
|
||||
using SqlSugar;
|
||||
using dy.net.utils;
|
||||
using MediaStorageType = dy.net.model.dto.StorageType;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
public class WebDavSettingsService
|
||||
{
|
||||
private const string SettingsId = "default";
|
||||
private readonly ISqlSugarClient _db;
|
||||
private readonly IDataProtector _protector;
|
||||
|
||||
public WebDavSettingsService(ISqlSugarClient db, IDataProtectionProvider provider)
|
||||
{
|
||||
_db = db;
|
||||
_protector = provider.CreateProtector("dysync.webdav.password.v1");
|
||||
}
|
||||
|
||||
public async Task<WebDavSettings> GetAsync()
|
||||
{
|
||||
return await _db.Queryable<WebDavSettings>().InSingleAsync(SettingsId)
|
||||
?? new WebDavSettings { Id = SettingsId, BasePath = "/dysync" };
|
||||
}
|
||||
|
||||
public async Task<string> GetPasswordAsync(WebDavSettings settings = null)
|
||||
{
|
||||
settings ??= await GetAsync();
|
||||
if (string.IsNullOrWhiteSpace(settings.ProtectedPassword)) return string.Empty;
|
||||
try
|
||||
{
|
||||
return _protector.Unprotect(settings.ProtectedPassword);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Serilog.Log.Error(ex, "WebDAV 密码解密失败,请重新输入密码");
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<WebDavSettings> BuildCandidateAsync(WebDavTestRequest request)
|
||||
{
|
||||
request ??= new WebDavTestRequest();
|
||||
var existing = await GetAsync();
|
||||
var password = string.IsNullOrWhiteSpace(request.Password)
|
||||
? await GetPasswordAsync(existing)
|
||||
: request.Password;
|
||||
|
||||
return new WebDavSettings
|
||||
{
|
||||
Id = SettingsId,
|
||||
Endpoint = string.IsNullOrWhiteSpace(request.Endpoint) ? existing.Endpoint : request.Endpoint.Trim(),
|
||||
BasePath = string.IsNullOrWhiteSpace(request.BasePath) ? existing.BasePath : request.BasePath.Trim(),
|
||||
UserName = string.IsNullOrWhiteSpace(request.UserName) ? existing.UserName : request.UserName.Trim(),
|
||||
ProtectedPassword = string.IsNullOrWhiteSpace(password) ? null : _protector.Protect(password),
|
||||
AllowInvalidCertificate = request.AllowInvalidCertificate
|
||||
};
|
||||
}
|
||||
|
||||
public async Task SaveAsync(WebDavSettings settings, bool tested, string testMessage)
|
||||
{
|
||||
settings.Id = SettingsId;
|
||||
settings.LastTestedAt = tested ? DateTime.Now : null;
|
||||
settings.LastTestMessage = testMessage;
|
||||
await _db.Storageable(settings).ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
public async Task<WebDavSettingsDto> ToDtoAsync(MediaStorageType storageType)
|
||||
{
|
||||
var settings = await GetAsync();
|
||||
return new WebDavSettingsDto
|
||||
{
|
||||
StorageType = storageType,
|
||||
Endpoint = settings.Endpoint,
|
||||
BasePath = settings.BasePath,
|
||||
UserName = settings.UserName,
|
||||
HasPassword = !string.IsNullOrWhiteSpace(settings.ProtectedPassword),
|
||||
AllowInvalidCertificate = settings.AllowInvalidCertificate,
|
||||
LastTestedAt = settings.LastTestedAt,
|
||||
LastTestMessage = settings.LastTestMessage
|
||||
};
|
||||
}
|
||||
|
||||
public async Task InitializeCookiePathsAsync()
|
||||
{
|
||||
var cookies = await _db.Queryable<DouyinCookie>().ToListAsync();
|
||||
foreach (var cookie in cookies)
|
||||
{
|
||||
ApplyDefaultCookiePaths(cookie);
|
||||
}
|
||||
if (cookies.Count > 0) await _db.Updateable(cookies).ExecuteCommandAsync();
|
||||
}
|
||||
|
||||
public void ApplyDefaultCookiePaths(DouyinCookie cookie)
|
||||
{
|
||||
if (cookie == null) return;
|
||||
var root = DouyinFileNameHelper.SanitizeLinuxFileName(cookie.UserName, cookie.Id, true);
|
||||
if (string.IsNullOrWhiteSpace(cookie.WebDavCollectPath)) cookie.WebDavCollectPath = $"/{root}/collect";
|
||||
if (string.IsNullOrWhiteSpace(cookie.WebDavFavoritePath)) cookie.WebDavFavoritePath = $"/{root}/favorite";
|
||||
if (string.IsNullOrWhiteSpace(cookie.WebDavFollowPath)) cookie.WebDavFollowPath = $"/{root}/follow";
|
||||
if (string.IsNullOrWhiteSpace(cookie.WebDavMixPath)) cookie.WebDavMixPath = $"/{root}/mix";
|
||||
if (string.IsNullOrWhiteSpace(cookie.WebDavSeriesPath)) cookie.WebDavSeriesPath = $"/{root}/series";
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user