feat: add fnOS packaging, storage workflows and release pipeline

This commit is contained in:
2026-08-11 18:05:49 +08:00
parent c5922f9b08
commit 95932f0199
181 changed files with 24024 additions and 1164 deletions
+77
View File
@@ -0,0 +1,77 @@
using System.Reflection;
namespace dy.net.service
{
public sealed class DatabaseUpgradeBackup
{
private readonly string _databaseRoot;
private readonly string _version;
private readonly string _markerPath;
private DatabaseUpgradeBackup(string databaseRoot, string version)
{
_databaseRoot = databaseRoot;
_version = version;
_markerPath = Path.Combine(databaseRoot, ".schema-version");
}
public static DatabaseUpgradeBackup Prepare(string dbPath)
{
var root = string.IsNullOrWhiteSpace(dbPath)
? Path.Combine(Environment.CurrentDirectory, "db")
: Path.Combine(dbPath, "db");
Directory.CreateDirectory(root);
var version = Assembly.GetExecutingAssembly().GetName().Version?.ToString(3) ?? "0.2.24";
var backup = new DatabaseUpgradeBackup(root, version);
backup.CreateIfNeeded();
return backup;
}
public void MarkSchemaReady() => File.WriteAllText(_markerPath, _version);
private void CreateIfNeeded()
{
var database = Path.Combine(_databaseRoot, "dy.sqlite");
if (!File.Exists(database)) return;
if (File.Exists(_markerPath) && string.Equals(File.ReadAllText(_markerPath).Trim(), _version, StringComparison.Ordinal)) return;
var backupRoot = Path.Combine(_databaseRoot, "upgrade-backups");
Directory.CreateDirectory(backupRoot);
var versionPrefix = $"pre-{_version}-";
if (!Directory.EnumerateDirectories(backupRoot, versionPrefix + "*").Any())
{
var destination = Path.Combine(backupRoot, versionPrefix + DateTime.Now.ToString("yyyyMMdd-HHmmss"));
Directory.CreateDirectory(destination);
CopyIfPresent(database, Path.Combine(destination, "dy.sqlite"));
CopyIfPresent(database + "-wal", Path.Combine(destination, "dy.sqlite-wal"));
CopyIfPresent(database + "-shm", Path.Combine(destination, "dy.sqlite-shm"));
CopyDirectory(Path.Combine(_databaseRoot, "keys"), Path.Combine(destination, "keys"));
}
foreach (var old in new DirectoryInfo(backupRoot).GetDirectories()
.OrderByDescending(x => x.CreationTimeUtc).Skip(3))
{
try { old.Delete(true); }
catch (Exception ex) { Serilog.Log.Warning(ex, "清理旧升级备份失败:{Path}", old.FullName); }
}
}
private static void CopyIfPresent(string source, string destination)
{
if (File.Exists(source)) File.Copy(source, destination, false);
}
private static void CopyDirectory(string source, string destination)
{
if (!Directory.Exists(source)) return;
Directory.CreateDirectory(destination);
foreach (var file in Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories))
{
var relative = Path.GetRelativePath(source, file);
var target = Path.Combine(destination, relative);
Directory.CreateDirectory(Path.GetDirectoryName(target)!);
File.Copy(file, target, false);
}
}
}
}
+7
View File
@@ -135,6 +135,13 @@ namespace dy.net.service
return count > 0;
}
public async Task<DouyinVideoDelete> GetDeleteVideoAsync(string videoId)
{
if (string.IsNullOrWhiteSpace(videoId)) return null;
return await sqlSugarClient.Queryable<DouyinVideoDelete>()
.Where(x => x.ViedoId == videoId).OrderByDescending(x => x.DeleteTime).FirstAsync();
}
/// <summary>
/// 新增要删除的视频
/// </summary>
+9 -4
View File
@@ -48,10 +48,15 @@ namespace dy.net.service
var d= await _cookieRepository.FastResetCookie(id, cookie);
if (d)
{
var cookies =await _cookieRepository.GetByIdAsync(id);
cookies.StatusCode = 0;
cookies.StatusMsg = "正常";
await _cookieRepository.UpdateAsync(cookies);
var cookies = string.IsNullOrWhiteSpace(id)
? await _cookieRepository.GetAllAsync()
: new List<DouyinCookie> { await _cookieRepository.GetByIdAsync(id) };
foreach (var item in cookies.Where(x => x != null))
{
item.StatusCode = 0;
item.StatusMsg = "正常";
await _cookieRepository.UpdateAsync(item);
}
}
return d;
}
+16 -2
View File
@@ -7,6 +7,13 @@ using SqlSugar;
namespace dy.net.service
{
public enum AddFollowResult
{
Added,
AlreadyExists,
Failed
}
public class DouyinFollowService
{
@@ -30,16 +37,23 @@ namespace dy.net.service
/// <param name="followed"></param>
/// <returns></returns>
public async Task<bool> AddAsync(DouyinFollowed followed)
{
return await TryAddAsync(followed) == AddFollowResult.Added;
}
public async Task<AddFollowResult> TryAddAsync(DouyinFollowed followed)
{
var foll = await _followRepository.GetFirstAsync(x => x.SecUid == followed.SecUid && x.mySelfId == followed.mySelfId);
if (foll != null)
{
return false;
return AddFollowResult.AlreadyExists;
}
followed.Id = IdGener.GetLong().ToString();
followed.LastSyncTime = DateTime.UtcNow;
followed.IsNoFollowed = true;
return await _followRepository.InsertAsync(followed);
return await _followRepository.InsertAsync(followed)
? AddFollowResult.Added
: AddFollowResult.Failed;
}
/// <summary>
+457 -162
View File
@@ -11,12 +11,17 @@ using System.Net;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using dy.net.storage;
namespace dy.net.service
{
public class DouyinHttpClientService : IDisposable
{
private readonly IHttpClientFactory _clientFactory;
private readonly MediaStorageRouter _storageRouter;
private readonly Func<TimeSpan, CancellationToken, Task> _delayAsync;
private const int TransientRequestRetryCount = 2;
private const int MediaNetworkRetryCount = 2;
private readonly JsonSerializerSettings _jsonSettings=new JsonSerializerSettings
{
NullValueHandling = NullValueHandling.Ignore,
@@ -24,9 +29,19 @@ namespace dy.net.service
};
private bool _disposedValue;
public DouyinHttpClientService(IHttpClientFactory clientFactory)
public DouyinHttpClientService(IHttpClientFactory clientFactory, MediaStorageRouter storageRouter)
: this(clientFactory, storageRouter, Task.Delay)
{
}
internal DouyinHttpClientService(
IHttpClientFactory clientFactory,
MediaStorageRouter storageRouter,
Func<TimeSpan, CancellationToken, Task> delayAsync)
{
_clientFactory = clientFactory;
_storageRouter = storageRouter;
_delayAsync = delayAsync ?? Task.Delay;
}
/// <summary>
@@ -51,21 +66,46 @@ namespace dy.net.service
);
}
using var requestMessage = new HttpRequestMessage(httpMethod, fullUrl);
if (!string.IsNullOrEmpty(refererValue) && Uri.IsWellFormedUriString(refererValue, UriKind.Absolute))
{
requestMessage.Headers.Referrer = new Uri(refererValue);
}
if (!string.IsNullOrEmpty(cookie))
{
requestMessage.Headers.TryAddWithoutValidation("Cookie", cookie);
}
// 优化:移除using,由IHttpClientFactory管理生命周期
var httpClient = _clientFactory.CreateClient(DouyinRequestParamManager.DY_HTTP_CLIENT);
return await httpClient.SendAsync(requestMessage);
for (var attempt = 0; ; attempt++)
{
using var requestMessage = new HttpRequestMessage(httpMethod, fullUrl);
if (!string.IsNullOrEmpty(refererValue) && Uri.IsWellFormedUriString(refererValue, UriKind.Absolute))
requestMessage.Headers.Referrer = new Uri(refererValue);
if (!string.IsNullOrEmpty(cookie))
requestMessage.Headers.TryAddWithoutValidation("Cookie", cookie);
try
{
var response = await httpClient.SendAsync(requestMessage);
if (!IsTransientStatus(response.StatusCode) || attempt >= TransientRequestRetryCount)
return response;
var status = (int)response.StatusCode;
response.Dispose();
await DelayTransientRetryAsync(attempt + 1, CancellationToken.None);
Log.Warning("抖音接口暂时返回 HTTP {Status},正在进行第 {Retry}/{MaxRetry} 次有限重试",
status, attempt + 1, TransientRequestRetryCount);
}
catch (Exception ex) when (IsTransientRequestFailure(ex) && attempt < TransientRequestRetryCount)
{
Log.Warning("抖音接口连接失败,正在进行第 {Retry}/{MaxRetry} 次有限重试:{Reason}",
attempt + 1, TransientRequestRetryCount, ex.GetBaseException().Message);
await DelayTransientRetryAsync(attempt + 1, CancellationToken.None);
}
}
}
private static bool IsTransientStatus(HttpStatusCode status) =>
status == HttpStatusCode.RequestTimeout || (int)status >= 500;
private static bool IsTransientRequestFailure(Exception error) =>
error is HttpRequestException or TimeoutException or TaskCanceledException;
private Task DelayTransientRetryAsync(int retryNumber, CancellationToken cancellationToken)
{
var exponential = retryNumber <= 1 ? 1 : 3;
var jitter = Random.Shared.Next(0, 401);
return _delayAsync(TimeSpan.FromMilliseconds(exponential * 1000 + jitter), cancellationToken);
}
#region
@@ -450,19 +490,21 @@ namespace dy.net.service
using var jsonReader = new JsonTextReader(reader);
var model = JsonSerializer.Create(_jsonSettings).Deserialize<DouyinVideoInfoResponse>(jsonReader);
if (model == null)
Log.Error($"SyncUpderPostVideos fail, data is null");
throw new InvalidDataException("获取博主作品列表失败:响应内容为空或无法解析");
return model;
}
else
{
Log.Error($"SyncUpderPostVideos StatusCode fail: {response.StatusCode}");
return null;
throw new HttpRequestException(
$"获取博主作品列表失败:HTTP {(int)response.StatusCode} {response.StatusCode}",
null,
response.StatusCode);
}
}
catch (Exception ex)
{
Log.Error($"SyncUpderPostVideos error: {ex.Message}", ex);
return null;
throw;
}
}
@@ -538,185 +580,438 @@ namespace dy.net.service
}
#endregion
#region
public async Task<(bool Success, string ActualSavePath)> DownloadAsync(
#region
public Task<MediaDownloadResult> DownloadAsync(
string videoUrl,
string savePath,
string cookie,
List<string> otherUrls = null,
CancellationToken cancellationToken = default,
TimeSpan? streamTimeout = null,
int maxRetryCount = 3,
int maxRetryCount = 12,
TimeSpan? initialRetryDelay = null)
{
// 优化1:隔离外部列表,避免引用泄漏
var retryUrls = new List<string> { videoUrl };
if (otherUrls != null && otherUrls.Any())
{
retryUrls.AddRange(otherUrls);
maxRetryCount = Math.Min(maxRetryCount, retryUrls.Count); // 防止溢出
}
int retryCount = 0;
var retryDelay = initialRetryDelay ?? TimeSpan.FromSeconds(1);
streamTimeout ??= TimeSpan.FromSeconds(60);
while (retryCount < maxRetryCount)
{
try
var urls = BuildCandidateUrls(videoUrl, otherUrls, maxRetryCount);
return DownloadCandidatesAsync(
urls,
savePath,
cookie,
cancellationToken,
streamTimeout ?? TimeSpan.FromSeconds(60),
initialRetryDelay ?? TimeSpan.FromSeconds(1),
async (response, actualPath, timeout, token) =>
{
string currentUrl = retryUrls[retryCount];
return await TryDownloadOnceAsync(currentUrl, savePath, cookie, cancellationToken, streamTimeout.Value);
}
catch (Exception ex) when (IsRetryableException(ex) && retryCount < maxRetryCount - 1)
{
retryCount++;
var delay = TimeSpan.FromMilliseconds(retryDelay.TotalMilliseconds * Math.Pow(2, retryCount - 1));
Log.Warning(ex, $"下载失败(第{retryCount}/{maxRetryCount}次重试):{videoUrl},将在{delay.TotalSeconds:F1}秒后重试");
try
{
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
Log.Information($"重试等待被取消:{videoUrl}");
return (false, savePath);
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
Log.Information($"下载被取消:{videoUrl}");
return (false, savePath);
}
catch (Exception ex)
{
Log.Error(ex, $"下载失败(不可重试):{videoUrl}");
CleanupIncompleteFile(savePath);
return (false, savePath);
}
}
Log.Error($"下载失败:已达最大重试次数 {maxRetryCount}URL={videoUrl}");
return (false, savePath);
await WriteLocalAsync(response, actualPath, timeout, token);
});
}
private async Task<(bool Success, string ActualSavePath)> TryDownloadOnceAsync(
string videoUrl,
/// <summary>
/// 将抖音媒体写入指定存储。远端存储负责安全中转和最终提交,
/// 不会先落入永久本地媒体目录。
/// </summary>
public async Task<MediaDownloadResult> DownloadToStorageAsync(
StorageType storageType,
string mediaUrl,
string savePath,
string cookie,
List<string> otherUrls = null,
CancellationToken cancellationToken = default,
int maxRetryCount = 12,
IMediaStorage storageOverride = null)
{
if (storageType == StorageType.Local)
return await DownloadAsync(mediaUrl, savePath, cookie, otherUrls, cancellationToken, maxRetryCount: maxRetryCount);
var storage = storageOverride ?? _storageRouter.Resolve(storageType);
if (storage.StorageType != storageType)
throw new InvalidOperationException("指定的媒体存储与目标存储类型不一致");
if (storage is ICanonicalMediaStorage canonicalStorage)
savePath = await canonicalStorage.CanonicalizePathAsync(savePath, false, cancellationToken);
var urls = BuildCandidateUrls(mediaUrl, otherUrls, maxRetryCount);
return await DownloadCandidatesAsync(
urls,
savePath,
cookie,
cancellationToken,
TimeSpan.FromSeconds(60),
TimeSpan.FromSeconds(1),
async (response, actualPath, _, token) =>
{
try
{
await using var responseStream = await response.Content.ReadAsStreamAsync(token);
await storage.WriteAsync(
actualPath,
responseStream,
response.Content.Headers.ContentLength,
response.Content.Headers.ContentType?.MediaType,
token);
}
catch (OperationCanceledException) when (token.IsCancellationRequested) { throw; }
catch (MediaIntegrityException) { throw; }
catch (Exception ex)
{
throw new MediaStorageException($"{storageType} 媒体写入失败:{actualPath}", ex);
}
});
}
private async Task<MediaDownloadResult> DownloadCandidatesAsync(
IReadOnlyList<Uri> urls,
string savePath,
string cookie,
CancellationToken cancellationToken,
TimeSpan streamTimeout)
TimeSpan streamTimeout,
TimeSpan retryDelay,
Func<HttpResponseMessage, string, TimeSpan, CancellationToken, Task> writeAsync)
{
DateTime lastStreamActivity = DateTime.UtcNow;
string actualSavePath = savePath;
string detectedExtension = string.Empty;
if (urls.Count == 0)
return Failure(MediaDownloadFailureKind.SourceUnavailable, savePath, null, null, 0, null, "没有有效的媒体下载地址。");
// 确保目录存在
var directory = Path.GetDirectoryName(savePath);
if (!Directory.Exists(directory))
var client = _clientFactory.CreateClient(DouyinRequestParamManager.DY_HTTP_CLIENT_DOWN);
MediaDownloadResult lastFailure = null;
var forbiddenCount = 0;
var attempted = 0;
foreach (var source in urls)
{
Directory.CreateDirectory(directory);
attempted++;
var networkRetry = 0;
var rateLimitRetry = 0;
while (true)
{
HttpResponseMessage response = null;
Uri finalUri = source;
try
{
(response, finalUri) = await SendWithSafeRedirectsAsync(client, source, cookie, cancellationToken);
var status = (int)response.StatusCode;
var host = SafeHost(finalUri);
if (response.StatusCode == HttpStatusCode.Unauthorized)
return Failure(MediaDownloadFailureKind.SourceUnauthorized, savePath, status, host, attempted, null,
"抖音媒体请求返回 401,需要重新授权。");
if (response.StatusCode == HttpStatusCode.Forbidden)
{
forbiddenCount++;
lastFailure = Failure(MediaDownloadFailureKind.SourceForbidden, savePath, status, host, attempted, null,
"抖音媒体来源拒绝访问(403)。");
break;
}
if (response.StatusCode is HttpStatusCode.NotFound or HttpStatusCode.Gone)
{
lastFailure = Failure(MediaDownloadFailureKind.SourceNotFound, savePath, status, host, attempted, null,
$"媒体来源返回 {status},作品可能已下架或链接已失效。");
break;
}
if ((int)response.StatusCode == 429)
{
var retryAt = GetRetryAfter(response);
if (rateLimitRetry++ == 0)
{
var wait = retryAt.HasValue ? retryAt.Value - DateTime.UtcNow : TimeSpan.FromSeconds(5);
if (wait < TimeSpan.Zero) wait = TimeSpan.Zero;
if (wait > TimeSpan.FromSeconds(60)) wait = TimeSpan.FromSeconds(60);
await _delayAsync(wait, cancellationToken);
continue;
}
return Failure(MediaDownloadFailureKind.SourceRateLimited, savePath, status, host, attempted, retryAt,
"抖音媒体来源请求过于频繁(429),已进入冷却等待。");
}
if ((int)response.StatusCode >= 500)
{
if (networkRetry < MediaNetworkRetryCount)
{
networkRetry++;
await DelayMediaRetryAsync(retryDelay, networkRetry, cancellationToken);
continue;
}
lastFailure = Failure(MediaDownloadFailureKind.SourceUnavailable, savePath, status, host, attempted, null,
$"媒体来源暂时不可用({status})。");
break;
}
if (!response.IsSuccessStatusCode)
{
lastFailure = Failure(MediaDownloadFailureKind.SourceUnavailable, savePath, status, host, attempted, null,
$"媒体来源请求失败({status})。");
break;
}
var mediaType = response.Content.Headers.ContentType?.MediaType;
if (IsObviousNonMedia(mediaType))
{
lastFailure = Failure(MediaDownloadFailureKind.SourceInvalidContent, savePath, status, host, attempted, null,
$"媒体来源返回了非媒体内容({mediaType})。");
break;
}
if (response.Content.Headers.ContentLength == 0)
{
lastFailure = Failure(MediaDownloadFailureKind.IntegrityCheckFailed, savePath, status, host, attempted, null,
"媒体来源返回空内容。");
break;
}
var actualPath = DetectActualSavePath(savePath, mediaType);
await writeAsync(response, actualPath, streamTimeout, cancellationToken);
return MediaDownloadResult.Succeeded(actualPath, host, attempted);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
CleanupIncompleteFile(savePath);
return Failure(MediaDownloadFailureKind.Cancelled, savePath, null, SafeHost(finalUri), attempted, null, "媒体下载已取消。");
}
catch (MediaStorageException ex)
{
CleanupIncompleteFile(savePath);
Log.Error(ex, "媒体存储写入失败:{Path}", savePath);
return Failure(MediaDownloadFailureKind.StorageUnavailable, savePath, null, SafeHost(finalUri), attempted, null,
DescribeStorageFailure(ex));
}
catch (MediaIntegrityException ex)
{
CleanupIncompleteFile(savePath);
lastFailure = Failure(MediaDownloadFailureKind.IntegrityCheckFailed, savePath, null, SafeHost(finalUri), attempted, null, ex.Message);
break;
}
catch (MediaSourceException ex)
{
lastFailure = Failure(ex.FailureKind, savePath, ex.StatusCode, ex.SourceHost, attempted, ex.RetryAfter, ex.Message);
break;
}
catch (Exception ex) when (ex is HttpRequestException or TimeoutException or TaskCanceledException or IOException)
{
if (networkRetry < MediaNetworkRetryCount)
{
networkRetry++;
await DelayMediaRetryAsync(retryDelay, networkRetry, cancellationToken);
continue;
}
lastFailure = Failure(MediaDownloadFailureKind.SourceUnavailable, savePath, null, SafeHost(finalUri), attempted, null,
"连接媒体来源失败。");
break;
}
finally
{
response?.Dispose();
}
}
}
// 清理已存在的文件
CleanupIncompleteFile(savePath);
if (forbiddenCount == attempted && attempted > 0)
lastFailure = Failure(MediaDownloadFailureKind.SourceForbidden, savePath, 403, lastFailure?.SourceHost,
attempted, null, $"全部 {attempted} 个候选媒体地址均返回 403。");
else if (lastFailure?.FailureKind == MediaDownloadFailureKind.SourceForbidden)
lastFailure = Failure(MediaDownloadFailureKind.SourceUnavailable, savePath, null, lastFailure.SourceHost,
attempted, null, "候选媒体地址返回了多种来源错误。");
// 优化2:使用using包裹下载专用HttpClient(工厂创建的Client仍可using,释放内部handler
using var httpClient = _clientFactory.CreateClient(DouyinRequestParamManager.DY_HTTP_CLIENT_DOWN);
lastFailure ??= Failure(MediaDownloadFailureKind.SourceUnavailable, savePath, null, null, attempted, null, "所有候选媒体地址均不可用。");
if (lastFailure.FailureKind == MediaDownloadFailureKind.SourceForbidden)
Log.Warning("抖音媒体来源被拒绝:Host={SourceHost}, Status={Status}, Candidates={Candidates}",
lastFailure.SourceHost, lastFailure.HttpStatusCode, lastFailure.AttemptedUrlCount);
else
Log.Warning("媒体来源下载失败:Host={SourceHost}, Status={Status}, Kind={Kind}, Candidates={Candidates}",
lastFailure.SourceHost, lastFailure.HttpStatusCode, lastFailure.FailureKind, lastFailure.AttemptedUrlCount);
return lastFailure;
}
private Task DelayMediaRetryAsync(TimeSpan baseDelay, int retryNumber, CancellationToken cancellationToken)
{
var multiplier = Math.Pow(2, Math.Max(0, retryNumber - 1));
var milliseconds = Math.Min(5000, Math.Max(100, baseDelay.TotalMilliseconds * multiplier));
milliseconds += Random.Shared.Next(0, 251);
return _delayAsync(TimeSpan.FromMilliseconds(milliseconds), cancellationToken);
}
private static List<Uri> BuildCandidateUrls(string primary, IEnumerable<string> alternatives, int limit)
{
var result = new List<Uri>();
var seen = new HashSet<string>(StringComparer.Ordinal);
foreach (var value in new[] { primary }.Concat(alternatives ?? Enumerable.Empty<string>()))
{
if (result.Count >= Math.Clamp(limit, 1, 12)) break;
if (!Uri.TryCreate(value, UriKind.Absolute, out var uri) || (uri.Scheme != Uri.UriSchemeHttp && uri.Scheme != Uri.UriSchemeHttps)) continue;
if (seen.Add(uri.AbsoluteUri)) result.Add(uri);
}
return result;
}
internal static string DescribeStorageFailure(MediaStorageException error)
{
if (error == null) return "媒体存储写入失败。";
var message = error.Message?.Trim();
var reason = error.GetBaseException()?.Message?.Trim();
if (string.IsNullOrWhiteSpace(message)) message = "媒体存储写入失败。";
if (string.IsNullOrWhiteSpace(reason) || string.Equals(message, reason, StringComparison.Ordinal))
return message;
return $"{message};原因:{reason}";
}
private static async Task<(HttpResponseMessage Response, Uri FinalUri)> SendWithSafeRedirectsAsync(
HttpClient client,
Uri initialUri,
string cookie,
CancellationToken cancellationToken)
{
var current = initialUri;
for (var redirect = 0; redirect <= 5; redirect++)
{
using var request = new HttpRequestMessage(HttpMethod.Get, current);
if (!string.IsNullOrWhiteSpace(cookie) && IsDouyinHost(current.Host))
request.Headers.TryAddWithoutValidation("Cookie", cookie);
var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
if (!IsRedirect(response.StatusCode)) return (response, current);
var location = response.Headers.Location;
response.Dispose();
if (location == null)
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable, "媒体来源返回了无目标地址的跳转。",
(int)response.StatusCode, SafeHost(current));
if (redirect == 5)
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable, "媒体来源跳转次数超过 5 次。",
(int)response.StatusCode, SafeHost(current));
current = location.IsAbsoluteUri ? location : new Uri(current, location);
if (current.Scheme != Uri.UriSchemeHttp && current.Scheme != Uri.UriSchemeHttps)
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable, "媒体来源返回了不安全的跳转协议。",
sourceHost: SafeHost(current));
}
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable, "媒体来源跳转失败。", sourceHost: SafeHost(current));
}
private static async Task WriteLocalAsync(
HttpResponseMessage response,
string actualSavePath,
TimeSpan streamTimeout,
CancellationToken cancellationToken)
{
var directory = Path.GetDirectoryName(actualSavePath);
FileStream fileStream = null;
try
{
// 配置请求头
httpClient.DefaultRequestHeaders.Remove("Cookie"); // 先移除再添加,避免重复
httpClient.DefaultRequestHeaders.Add("Cookie", cookie);
httpClient.Timeout = TimeSpan.FromMinutes(5);
// 优化3:使用HttpCompletionOption.ResponseHeadersRead,不缓存整个响应
using var response = await httpClient.GetAsync(
videoUrl,
HttpCompletionOption.ResponseHeadersRead,
cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
long? totalBytes = response.Content.Headers.ContentLength;
// 检测文件类型
var ext = Path.GetExtension(savePath);
if (ext == ".mp3")
{
var contentType = response.Content.Headers.ContentType?.MediaType;
if (!string.IsNullOrEmpty(contentType))
{
if (contentType.Contains("audio/mp4") || contentType.Contains("audio/m4a"))
detectedExtension = "m4a";
else if (contentType.Contains("audio/mpeg") || contentType.Contains("audio/mp3"))
detectedExtension = "mp3";
else if (contentType.Contains("video/mp4"))
detectedExtension = "mp4";
}
}
// 修正保存路径
if (!string.IsNullOrEmpty(detectedExtension))
{
actualSavePath = Path.ChangeExtension(actualSavePath, detectedExtension.Trim('.'));
CleanupIncompleteFile(actualSavePath);
}
// 优化4:流式写入文件,且使用ArrayPool复用缓冲区(减少大对象分配)
using var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
using var fileStream = new FileStream(
actualSavePath,
FileMode.CreateNew,
FileAccess.Write,
FileShare.None,
bufferSize: 8192,
FileOptions.Asynchronous | FileOptions.SequentialScan);
var buffer = System.Buffers.ArrayPool<byte>.Shared.Rent(8192); // 复用缓冲区
try
{
int bytesRead;
long totalRead = 0;
if (!string.IsNullOrWhiteSpace(directory)) Directory.CreateDirectory(directory);
CleanupIncompleteFile(actualSavePath);
fileStream = new FileStream(actualSavePath, FileMode.CreateNew, FileAccess.Write, FileShare.None,
81920, FileOptions.Asynchronous | FileOptions.SequentialScan);
}
catch (Exception ex) when (ex is UnauthorizedAccessException or IOException)
{
throw new MediaStorageException($"本地媒体写入失败:{actualSavePath}", ex);
}
while ((bytesRead = await responseStream.ReadAsync(buffer, 0, buffer.Length, cancellationToken).ConfigureAwait(false)) > 0)
Stream responseStream;
try { responseStream = await response.Content.ReadAsStreamAsync(cancellationToken); }
catch (Exception ex) when (ex is HttpRequestException or IOException)
{
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable, "读取媒体来源响应失败。",
sourceHost: SafeHost(response.RequestMessage?.RequestUri), innerException: ex);
}
await using var ownedResponseStream = responseStream;
var buffer = System.Buffers.ArrayPool<byte>.Shared.Rent(81920);
long totalRead = 0;
var lastActivity = DateTime.UtcNow;
try
{
int read;
while (true)
{
if (DateTime.UtcNow - lastStreamActivity > streamTimeout)
throw new TimeoutException($"流读取超时({streamTimeout.TotalSeconds}秒无数据)");
await fileStream.WriteAsync(buffer, 0, bytesRead, cancellationToken).ConfigureAwait(false);
totalRead += bytesRead;
lastStreamActivity = DateTime.UtcNow;
try { read = await responseStream.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken); }
catch (Exception ex) when (ex is HttpRequestException or IOException)
{
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable, "读取媒体来源响应失败。",
sourceHost: SafeHost(response.RequestMessage?.RequestUri), innerException: ex);
}
if (read <= 0) break;
if (DateTime.UtcNow - lastActivity > streamTimeout)
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable,
$"媒体流超过 {streamTimeout.TotalSeconds:0} 秒没有数据。");
try { await fileStream.WriteAsync(buffer.AsMemory(0, read), cancellationToken); }
catch (Exception ex) when (ex is UnauthorizedAccessException or IOException)
{
throw new MediaStorageException($"本地媒体写入失败:{actualSavePath}", ex);
}
totalRead += read;
lastActivity = DateTime.UtcNow;
}
try { await fileStream.FlushAsync(cancellationToken); }
catch (Exception ex) when (ex is UnauthorizedAccessException or IOException)
{
throw new MediaStorageException($"本地媒体写入失败:{actualSavePath}", ex);
}
await fileStream.FlushAsync(cancellationToken).ConfigureAwait(false);
}
finally
{
System.Buffers.ArrayPool<byte>.Shared.Return(buffer); // 归还缓冲区
System.Buffers.ArrayPool<byte>.Shared.Return(buffer);
}
return (true, actualSavePath);
}
catch
{
CleanupIncompleteFile(actualSavePath);
throw;
}
finally
{
//httpClient.DefaultRequestHeaders.Clear(); // 清空请求头,帮助GC
if (totalRead <= 0) throw new MediaIntegrityException("下载内容为空。");
var expected = response.Content.Headers.ContentLength;
if (expected.HasValue && expected.Value != totalRead)
throw new MediaIntegrityException($"下载长度不一致:期望 {expected.Value},实际 {totalRead}。");
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
catch (MediaSourceException) { CleanupIncompleteFile(actualSavePath); throw; }
catch (MediaIntegrityException) { CleanupIncompleteFile(actualSavePath); throw; }
catch (MediaStorageException) { CleanupIncompleteFile(actualSavePath); throw; }
finally { if (fileStream != null) await fileStream.DisposeAsync(); }
}
private bool IsRetryableException(Exception ex)
private static string DetectActualSavePath(string savePath, string mediaType)
{
return ex is HttpRequestException
|| ex is TimeoutException
|| ex is IOException
|| (ex is AggregateException aggEx && aggEx.InnerExceptions.Any(IsRetryableException));
if (!Path.GetExtension(savePath).Equals(".mp3", StringComparison.OrdinalIgnoreCase)) return savePath;
if (mediaType?.Contains("audio/mp4", StringComparison.OrdinalIgnoreCase) == true
|| mediaType?.Contains("audio/m4a", StringComparison.OrdinalIgnoreCase) == true)
return Path.ChangeExtension(savePath, ".m4a");
if (mediaType?.Contains("video/mp4", StringComparison.OrdinalIgnoreCase) == true)
return Path.ChangeExtension(savePath, ".mp4");
return savePath;
}
private static bool IsObviousNonMedia(string mediaType) =>
!string.IsNullOrWhiteSpace(mediaType)
&& (mediaType.StartsWith("text/html", StringComparison.OrdinalIgnoreCase)
|| mediaType.Contains("json", StringComparison.OrdinalIgnoreCase));
private static bool IsDouyinHost(string host) =>
string.Equals(host, "douyin.com", StringComparison.OrdinalIgnoreCase)
|| host.EndsWith(".douyin.com", StringComparison.OrdinalIgnoreCase);
private static bool IsRedirect(HttpStatusCode code) => code is
HttpStatusCode.MovedPermanently or HttpStatusCode.Redirect or HttpStatusCode.RedirectMethod
or HttpStatusCode.TemporaryRedirect or HttpStatusCode.PermanentRedirect;
private static string SafeHost(Uri uri) => uri?.IsAbsoluteUri == true ? uri.IdnHost : null;
private static DateTime? GetRetryAfter(HttpResponseMessage response)
{
var value = response.Headers.RetryAfter;
if (value?.Date.HasValue == true) return value.Date.Value.UtcDateTime;
if (value?.Delta.HasValue == true) return DateTime.UtcNow.Add(value.Delta.Value);
return DateTime.UtcNow.AddMinutes(15);
}
private static MediaDownloadResult Failure(
MediaDownloadFailureKind kind,
string savePath,
int? status,
string host,
int attempted,
DateTime? retryAfter,
string message) => new()
{
Success = false,
ActualSavePath = savePath,
FailureKind = kind,
HttpStatusCode = status,
SourceHost = host,
AttemptedUrlCount = attempted,
RetryAfter = retryAfter,
Message = message
};
private static void CleanupIncompleteFile(string savePath)
{
if (File.Exists(savePath))
@@ -757,4 +1052,4 @@ namespace dy.net.service
}
#endregion
}
}
}
+210
View File
@@ -0,0 +1,210 @@
using System.Net;
using dy.net.model.dto;
using dy.net.utils;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
namespace dy.net.service
{
public interface IDouyinLiveStatusClient
{
Task<DouyinLiveStatusProbe> ProbeAsync(
string secUid,
string cookie,
CancellationToken cancellationToken = default);
}
public sealed class DouyinLiveStatusRequestException : InvalidOperationException
{
public DouyinLiveStatusRequestException(
string message,
HttpStatusCode? statusCode = null,
bool requiresAccountCooldown = false,
Exception innerException = null)
: base(message, innerException)
{
StatusCode = statusCode;
RequiresAccountCooldown = requiresAccountCooldown;
}
public HttpStatusCode? StatusCode { get; }
public bool RequiresAccountCooldown { get; }
}
public sealed class DouyinLiveStatusClient : IDouyinLiveStatusClient
{
private const string ProfilePath = "/aweme/v1/web/user/profile/other/";
private readonly IHttpClientFactory _clientFactory;
private readonly DouyinABogusSigner _signer = new();
public DouyinLiveStatusClient(IHttpClientFactory clientFactory) => _clientFactory = clientFactory;
public async Task<DouyinLiveStatusProbe> ProbeAsync(
string secUid,
string cookie,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(secUid))
throw new InvalidOperationException("博主 SecUid 为空,无法检查直播状态");
if (string.IsNullOrWhiteSpace(cookie))
throw new InvalidOperationException("授权 Cookie 为空,无法检查直播状态");
var parameters = BuildParameters(secUid.Trim(), cookie);
var unsignedQuery = BuildQueryString(parameters);
var requestUri = ProfilePath + "?" + unsignedQuery + "&a_bogus=" + Encode(_signer.Sign(unsignedQuery));
using var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
request.Headers.Referrer = new Uri("https://www.douyin.com/user/" + Encode(secUid.Trim()));
request.Headers.TryAddWithoutValidation("Accept", "application/json, text/plain, */*");
request.Headers.TryAddWithoutValidation("Cookie", cookie);
var client = _clientFactory.CreateClient(DouyinRequestParamManager.DY_HTTP_CLIENT);
try
{
using var response = await client.SendAsync(
request,
HttpCompletionOption.ResponseHeadersRead,
cancellationToken);
if (!response.IsSuccessStatusCode)
{
var shouldCooldown = response.StatusCode is HttpStatusCode.Forbidden
or HttpStatusCode.TooManyRequests
or HttpStatusCode.Unauthorized;
throw new DouyinLiveStatusRequestException(
$"抖音直播状态请求失败(HTTP {(int)response.StatusCode}",
response.StatusCode,
shouldCooldown);
}
var content = await response.Content.ReadAsStringAsync(cancellationToken);
UserProfileResponse payload;
try
{
payload = JsonConvert.DeserializeObject<UserProfileResponse>(content);
}
catch (JsonException ex)
{
throw new DouyinLiveStatusRequestException(
"抖音返回了无法识别的直播状态响应,可能需要验证授权",
requiresAccountCooldown: true,
innerException: ex);
}
if (payload?.User == null)
throw new DouyinLiveStatusRequestException(
"抖音没有返回博主资料,可能需要验证授权",
requiresAccountCooldown: true);
if (payload.StatusCode != 0)
throw new DouyinLiveStatusRequestException(
$"抖音拒绝了直播状态查询(状态码 {payload.StatusCode}",
requiresAccountCooldown: true);
return DouyinLiveStatusParser.Parse(
payload.User.LiveStatus,
payload.User.RoomId,
payload.User.RoomIdStr,
payload.User.RoomData);
}
catch (DouyinLiveStatusRequestException)
{
throw;
}
catch (HttpRequestException ex)
{
throw new DouyinLiveStatusRequestException("抖音直播状态网络请求失败,请稍后重试", innerException: ex);
}
catch (OperationCanceledException ex) when (!cancellationToken.IsCancellationRequested)
{
throw new DouyinLiveStatusRequestException("抖音直播状态查询超时,请稍后重试", innerException: ex);
}
}
private static List<KeyValuePair<string, string>> BuildParameters(string secUid, string cookie)
{
var cookies = ParseCookie(cookie);
return new List<KeyValuePair<string, string>>
{
new("device_platform", "webapp"),
new("aid", "6383"),
new("channel", "channel_pc_web"),
new("update_version_code", "170400"),
new("pc_client_type", "1"),
new("pc_libra_divert", "Windows"),
new("support_h265", "1"),
new("support_dash", "1"),
new("version_code", "170400"),
new("version_name", "17.4.0"),
new("cookie_enabled", "true"),
new("screen_width", "1536"),
new("screen_height", "864"),
new("browser_language", "zh-CN"),
new("browser_platform", "Win32"),
new("browser_name", "Chrome"),
new("browser_version", "119.0.0.0"),
new("browser_online", "true"),
new("engine_name", "Blink"),
new("engine_version", "119.0.0.0"),
new("os_name", "Windows"),
new("os_version", "10"),
new("cpu_core_num", "16"),
new("device_memory", "8"),
new("platform", "PC"),
new("downlink", "10"),
new("effective_type", "4g"),
new("round_trip_time", "200"),
new("uifid", CookieValue(cookies, "UIFID", "UIFID_TEMP")),
new("msToken", CookieValue(cookies, "msToken")),
new("sec_user_id", secUid),
new("publish_video_strategy_type", "2"),
new("personal_center_strategy", "1")
};
}
private static Dictionary<string, string> ParseCookie(string cookie)
{
var values = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var part in (cookie ?? string.Empty).Split(';', StringSplitOptions.RemoveEmptyEntries))
{
var separator = part.IndexOf('=');
if (separator <= 0) continue;
values[part[..separator].Trim()] = part[(separator + 1)..].Trim();
}
return values;
}
private static string CookieValue(Dictionary<string, string> cookies, params string[] names)
{
foreach (var name in names)
if (cookies.TryGetValue(name, out var value)) return value;
return string.Empty;
}
private static string BuildQueryString(IEnumerable<KeyValuePair<string, string>> parameters) =>
string.Join("&", parameters.Select(x => Encode(x.Key) + "=" + Encode(x.Value ?? string.Empty)));
private static string Encode(string value) => Uri.EscapeDataString(value ?? string.Empty);
private sealed class UserProfileResponse
{
[JsonProperty("status_code")]
public int StatusCode { get; set; }
[JsonProperty("user")]
public UserProfile User { get; set; }
}
private sealed class UserProfile
{
[JsonProperty("live_status")]
public int? LiveStatus { get; set; }
[JsonProperty("room_id")]
public string RoomId { get; set; }
[JsonProperty("room_id_str")]
public string RoomIdStr { get; set; }
[JsonProperty("room_data")]
public JToken RoomData { get; set; }
}
}
}
+482
View File
@@ -0,0 +1,482 @@
using System.Collections.Concurrent;
using System.Net;
using dy.net.model.dto;
using dy.net.model.entity;
using dy.net.model.response;
using dy.net.utils;
using Newtonsoft.Json.Linq;
using Serilog;
using SqlSugar;
namespace dy.net.service
{
public sealed class DouyinLiveStatusService
{
private static readonly TimeSpan RefreshInterval = TimeSpan.FromMinutes(5);
private static readonly TimeSpan ManualRefreshFloor = TimeSpan.FromSeconds(30);
private static readonly TimeSpan StaleAfter = TimeSpan.FromMinutes(10);
private static readonly ConcurrentDictionary<string, SemaphoreSlim> NotificationGates = new();
private readonly ISqlSugarClient _db;
private readonly IDouyinLiveStatusClient _client;
private readonly LiveEmailNotificationService _emailNotifications;
public DouyinLiveStatusService(
ISqlSugarClient db,
IDouyinLiveStatusClient client,
LiveEmailNotificationService emailNotifications)
{
_db = db;
_client = client;
_emailNotifications = emailNotifications;
}
public async Task<FollowLiveStatusDto> SetMonitorAsync(
FollowLiveMonitorUpdateDto request,
CancellationToken cancellationToken = default)
{
if (request == null || string.IsNullOrWhiteSpace(request.Id))
throw new InvalidOperationException("博主记录不能为空");
var follow = await RequireFollowAsync(request.Id);
if (request.Enabled && string.IsNullOrWhiteSpace(follow.SecUid))
throw new InvalidOperationException("该博主缺少 SecUid,无法开启直播监测");
follow.LiveMonitorEnabled = request.Enabled;
if (!request.Enabled)
{
follow.LiveStatus = DouyinLiveStatusState.Unknown;
follow.LiveRoomId = null;
follow.LiveWebRid = null;
follow.LiveTitle = null;
follow.LiveStartedAt = null;
follow.LiveCheckedAt = null;
follow.LiveStatusUpdatedAt = null;
follow.LiveCheckError = null;
follow.LiveEmailNotificationEnabled = false;
follow.LastLiveNotificationKey = null;
follow.LastLiveNotificationAttemptAt = null;
follow.LastLiveNotificationError = null;
}
else if (!follow.LiveCheckedAt.HasValue)
{
follow.LiveStatus = DouyinLiveStatusState.Unknown;
follow.LiveCheckError = null;
}
await UpdateFollowLiveColumnsAsync(follow);
if (!request.Enabled) await UpdateFollowNotificationColumnsAsync(follow);
if (request.Enabled)
follow = await ProbeFollowAsync(follow, ignoreRecentCheck: true, cancellationToken);
return ToDto(follow);
}
public async Task<FollowLiveStatusDto> SetEmailNotificationAsync(FollowLiveEmailUpdateDto request)
{
if (request == null || string.IsNullOrWhiteSpace(request.Id))
throw new InvalidOperationException("博主记录不能为空");
var follow = await RequireFollowAsync(request.Id);
if (request.Enabled && !follow.LiveMonitorEnabled)
throw new InvalidOperationException("请先开启该博主的直播监测");
if (request.Enabled)
{
var readinessError = await _emailNotifications.GetReadinessErrorAsync();
if (!string.IsNullOrWhiteSpace(readinessError)) throw new InvalidOperationException(readinessError);
}
follow.LiveEmailNotificationEnabled = request.Enabled;
follow.LastLiveNotificationError = null;
if (!request.Enabled)
{
follow.LastLiveNotificationKey = null;
follow.LastLiveNotificationAttemptAt = null;
}
await UpdateFollowNotificationColumnsAsync(follow);
if (request.Enabled && follow.LiveStatus == DouyinLiveStatusState.Live)
{
await HandleLiveNotificationAsync(follow, DouyinLiveStatusState.Unknown, CancellationToken.None);
follow = await RequireFollowAsync(follow.Id);
}
return ToDto(follow);
}
public async Task<FollowLiveStatusDto> RefreshAsync(
string id,
CancellationToken cancellationToken = default)
{
var follow = await RequireFollowAsync(id);
if (!follow.LiveMonitorEnabled)
throw new InvalidOperationException("请先开启该博主的直播监测");
follow = await ProbeFollowAsync(follow, ignoreRecentCheck: false, cancellationToken);
return ToDto(follow);
}
public async Task<List<FollowLiveStatusDto>> QueryAsync(IEnumerable<string> ids)
{
var normalized = (ids ?? Array.Empty<string>())
.Where(x => !string.IsNullOrWhiteSpace(x))
.Select(x => x.Trim())
.Distinct(StringComparer.Ordinal)
.Take(100)
.ToList();
if (normalized.Count == 0) return new List<FollowLiveStatusDto>();
var follows = await _db.Queryable<DouyinFollowed>()
.Where(x => normalized.Contains(x.Id))
.ToListAsync();
return follows.Select(ToDto).ToList();
}
public async Task RefreshDueAsync(CancellationToken cancellationToken = default)
{
var cutoff = DateTime.Now.Subtract(RefreshInterval);
var follows = await _db.Queryable<DouyinFollowed>()
.Where(x => x.LiveMonitorEnabled && (x.LiveCheckedAt == null || x.LiveCheckedAt <= cutoff))
.OrderBy(x => x.LiveCheckedAt)
.ToListAsync();
if (follows.Count == 0) return;
var myUserIds = follows.Select(x => x.mySelfId).Where(x => !string.IsNullOrWhiteSpace(x)).Distinct().ToList();
var cookies = await _db.Queryable<DouyinCookie>()
.Where(x => myUserIds.Contains(x.MyUserId))
.ToListAsync();
foreach (var group in follows.GroupBy(x => x.mySelfId, StringComparer.Ordinal))
{
cancellationToken.ThrowIfCancellationRequested();
var cookie = SelectCookie(cookies, group.Key);
if (cookie == null || cookie.Status != 1 || cookie.StatusCode != 0 || string.IsNullOrWhiteSpace(cookie.Cookies))
{
await MarkGroupUnavailableAsync(group, "授权账号无效或未启用,直播状态暂时无法检查");
continue;
}
if (cookie.LiveCheckCooldownUntil.HasValue && cookie.LiveCheckCooldownUntil > DateTime.Now)
{
await MarkGroupUnavailableAsync(group,
$"直播状态检查已冷却至 {cookie.LiveCheckCooldownUntil.Value.ToLocalTime():MM-dd HH:mm}");
continue;
}
var items = group.ToList();
for (var index = 0; index < items.Count; index++)
{
cancellationToken.ThrowIfCancellationRequested();
var result = await ProbeWithCookieAsync(items[index], cookie, cancellationToken);
if (result.ShouldStopAccount) break;
if (index < items.Count - 1)
await Task.Delay(Random.Shared.Next(750, 1501), cancellationToken);
}
}
}
public async Task ApplyFollowListStatusesAsync(
IEnumerable<FollowingsItem> followings,
DouyinCookie cookie)
{
if (cookie == null || string.IsNullOrWhiteSpace(cookie.MyUserId)) return;
var source = (followings ?? Array.Empty<FollowingsItem>())
.Where(x => x != null && !string.IsNullOrWhiteSpace(x.SecUid))
.GroupBy(x => x.SecUid, StringComparer.Ordinal)
.ToDictionary(x => x.Key, x => x.First(), StringComparer.Ordinal);
if (source.Count == 0) return;
var secUids = source.Keys.ToList();
var monitored = await _db.Queryable<DouyinFollowed>()
.Where(x => x.mySelfId == cookie.MyUserId && x.LiveMonitorEnabled && secUids.Contains(x.SecUid))
.ToListAsync();
foreach (var follow in monitored)
{
if (!source.TryGetValue(follow.SecUid, out var item)) continue;
var probe = DouyinLiveStatusParser.Parse(
item.LiveStatus,
item.RoomId,
item.RoomIdStr,
item.RoomData);
if (probe.Status == DouyinLiveStatusState.Unknown) continue;
var previousStatus = follow.LiveStatus;
ApplySuccess(follow, probe, DateTime.Now);
await UpdateFollowLiveColumnsAsync(follow);
await HandleLiveNotificationAsync(follow, previousStatus, CancellationToken.None);
}
}
private async Task<DouyinFollowed> ProbeFollowAsync(
DouyinFollowed follow,
bool ignoreRecentCheck,
CancellationToken cancellationToken)
{
if (!ignoreRecentCheck && follow.LiveCheckedAt.HasValue &&
DateTime.Now - follow.LiveCheckedAt.Value < ManualRefreshFloor)
return follow;
var cookies = await _db.Queryable<DouyinCookie>()
.Where(x => x.MyUserId == follow.mySelfId)
.ToListAsync();
var cookie = SelectCookie(cookies, follow.mySelfId);
if (cookie == null || cookie.Status != 1 || cookie.StatusCode != 0 || string.IsNullOrWhiteSpace(cookie.Cookies))
return await MarkFailedAsync(follow, "授权账号无效或未启用,直播状态暂时无法检查");
if (cookie.LiveCheckCooldownUntil.HasValue && cookie.LiveCheckCooldownUntil > DateTime.Now)
return await MarkFailedAsync(follow,
$"直播状态检查已冷却至 {cookie.LiveCheckCooldownUntil.Value.ToLocalTime():MM-dd HH:mm}");
await ProbeWithCookieAsync(follow, cookie, cancellationToken);
return await RequireFollowAsync(follow.Id);
}
private async Task<ProbeResult> ProbeWithCookieAsync(
DouyinFollowed follow,
DouyinCookie cookie,
CancellationToken cancellationToken)
{
try
{
var probe = await _client.ProbeAsync(follow.SecUid, cookie.Cookies, cancellationToken);
if (probe.Status == DouyinLiveStatusState.Unknown)
throw new DouyinLiveStatusRequestException("抖音响应中缺少可识别的直播状态");
var previousStatus = follow.LiveStatus;
ApplySuccess(follow, probe, DateTime.Now);
await UpdateFollowLiveColumnsAsync(follow);
await HandleLiveNotificationAsync(follow, previousStatus, cancellationToken);
await ResetAccountHealthAsync(cookie);
return new ProbeResult(false);
}
catch (DouyinLiveStatusRequestException ex)
{
var message = Limit(ex.Message, 1000);
await MarkFailedAsync(follow, message);
if (ex.RequiresAccountCooldown)
{
await ApplyAccountCooldownAsync(cookie, message);
Log.Warning("直播状态检查触发账号冷却:Account={Account}, HTTP={StatusCode}",
cookie.UserName, ex.StatusCode.HasValue ? (int)ex.StatusCode.Value : null);
}
else
{
Log.Warning("直播状态检查失败:Account={Account}, Blogger={Blogger}, Reason={Reason}",
cookie.UserName, follow.UperName, message);
}
return new ProbeResult(ex.RequiresAccountCooldown);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
var message = "直播状态检查失败,请稍后重试";
await MarkFailedAsync(follow, message);
Log.Warning(ex, "直播状态检查发生异常:Account={Account}, Blogger={Blogger}",
cookie.UserName, follow.UperName);
return new ProbeResult(false);
}
}
private async Task ApplyAccountCooldownAsync(DouyinCookie cookie, string error)
{
cookie.ConsecutiveLiveCheckFailures = Math.Max(0, cookie.ConsecutiveLiveCheckFailures) + 1;
var minutes = cookie.ConsecutiveLiveCheckFailures switch
{
1 => 15,
2 => 30,
3 => 60,
4 => 120,
_ => 360
};
cookie.LiveCheckCooldownUntil = DateTime.Now.AddMinutes(minutes);
cookie.LastLiveCheckError = Limit(error, 1000);
cookie.LiveCheckHealthUpdatedAt = DateTime.Now;
await _db.Updateable(cookie)
.UpdateColumns(x => new
{
x.ConsecutiveLiveCheckFailures,
x.LiveCheckCooldownUntil,
x.LastLiveCheckError,
x.LiveCheckHealthUpdatedAt
})
.ExecuteCommandAsync();
}
private async Task ResetAccountHealthAsync(DouyinCookie cookie)
{
if (cookie.ConsecutiveLiveCheckFailures == 0 && !cookie.LiveCheckCooldownUntil.HasValue &&
string.IsNullOrWhiteSpace(cookie.LastLiveCheckError)) return;
cookie.ConsecutiveLiveCheckFailures = 0;
cookie.LiveCheckCooldownUntil = null;
cookie.LastLiveCheckError = null;
cookie.LiveCheckHealthUpdatedAt = DateTime.Now;
await _db.Updateable(cookie)
.UpdateColumns(x => new
{
x.ConsecutiveLiveCheckFailures,
x.LiveCheckCooldownUntil,
x.LastLiveCheckError,
x.LiveCheckHealthUpdatedAt
})
.ExecuteCommandAsync();
}
private async Task MarkGroupUnavailableAsync(IEnumerable<DouyinFollowed> follows, string message)
{
foreach (var follow in follows)
await MarkFailedAsync(follow, message, updateCheckedAt: false);
}
private async Task<DouyinFollowed> MarkFailedAsync(
DouyinFollowed follow,
string message,
bool updateCheckedAt = true)
{
if (updateCheckedAt) follow.LiveCheckedAt = DateTime.Now;
follow.LiveCheckError = Limit(message, 1000);
await UpdateFollowLiveColumnsAsync(follow);
return follow;
}
private static void ApplySuccess(DouyinFollowed follow, DouyinLiveStatusProbe probe, DateTime now)
{
follow.LiveStatus = probe.Status;
follow.LiveCheckedAt = now;
follow.LiveStatusUpdatedAt = now;
follow.LiveCheckError = null;
if (probe.Status == DouyinLiveStatusState.Live)
{
follow.LiveRoomId = Limit(probe.RoomId, 100);
follow.LiveWebRid = Limit(probe.WebRid, 100);
follow.LiveTitle = Limit(probe.Title, 500);
follow.LiveStartedAt = probe.StartedAt;
}
else
{
follow.LiveRoomId = null;
follow.LiveWebRid = null;
follow.LiveTitle = null;
follow.LiveStartedAt = null;
}
}
private async Task UpdateFollowLiveColumnsAsync(DouyinFollowed follow)
{
await _db.Updateable(follow)
.UpdateColumns(x => new
{
x.LiveMonitorEnabled,
x.LiveStatus,
x.LiveRoomId,
x.LiveWebRid,
x.LiveTitle,
x.LiveCheckedAt,
x.LiveStatusUpdatedAt,
x.LiveStartedAt,
x.LiveCheckError,
x.LiveEmailNotificationEnabled
})
.ExecuteCommandAsync();
}
private async Task HandleLiveNotificationAsync(
DouyinFollowed follow,
DouyinLiveStatusState previousStatus,
CancellationToken cancellationToken)
{
var gate = NotificationGates.GetOrAdd(follow.Id, _ => new SemaphoreSlim(1, 1));
await gate.WaitAsync(cancellationToken);
try
{
follow = await RequireFollowAsync(follow.Id);
if (follow.LiveStatus != DouyinLiveStatusState.Live)
{
if (follow.LastLiveNotificationKey != null || follow.LastLiveNotificationError != null)
{
follow.LastLiveNotificationKey = null;
follow.LastLiveNotificationAttemptAt = null;
follow.LastLiveNotificationError = null;
await UpdateFollowNotificationColumnsAsync(follow);
}
return;
}
if (!follow.LiveEmailNotificationEnabled) return;
var notificationKey = !string.IsNullOrWhiteSpace(follow.LiveWebRid)
? "web:" + follow.LiveWebRid
: !string.IsNullOrWhiteSpace(follow.LiveRoomId)
? "room:" + follow.LiveRoomId
: "live";
if (string.Equals(follow.LastLiveNotificationKey, notificationKey, StringComparison.Ordinal)) return;
if (previousStatus == DouyinLiveStatusState.Live &&
follow.LastLiveNotificationAttemptAt.HasValue &&
DateTime.Now - follow.LastLiveNotificationAttemptAt.Value < TimeSpan.FromMinutes(15)) return;
follow.LastLiveNotificationAttemptAt = DateTime.Now;
var result = await _emailNotifications.SendLiveStartedAsync(follow, cancellationToken);
if (result.Sent)
{
follow.LastLiveNotificationKey = notificationKey;
follow.LastLiveNotifiedAt = DateTime.Now;
follow.LastLiveNotificationError = null;
}
else if (!string.IsNullOrWhiteSpace(result.Error))
{
follow.LastLiveNotificationError = Limit(result.Error, 1000);
}
await UpdateFollowNotificationColumnsAsync(follow);
}
finally
{
gate.Release();
}
}
private async Task UpdateFollowNotificationColumnsAsync(DouyinFollowed follow)
{
await _db.Updateable(follow)
.UpdateColumns(x => new
{
x.LiveEmailNotificationEnabled,
x.LastLiveNotificationKey,
x.LastLiveNotificationAttemptAt,
x.LastLiveNotifiedAt,
x.LastLiveNotificationError
})
.ExecuteCommandAsync();
}
private async Task<DouyinFollowed> RequireFollowAsync(string id) =>
await _db.Queryable<DouyinFollowed>().InSingleAsync(id)
?? throw new KeyNotFoundException("关注博主记录不存在");
private static DouyinCookie SelectCookie(IEnumerable<DouyinCookie> cookies, string myUserId) =>
cookies
.Where(x => string.Equals(x.MyUserId, myUserId, StringComparison.Ordinal))
.OrderByDescending(x => x.Status == 1 && x.StatusCode == 0 && !string.IsNullOrWhiteSpace(x.Cookies))
.FirstOrDefault();
private static FollowLiveStatusDto ToDto(DouyinFollowed follow)
{
var stale = follow.LiveMonitorEnabled &&
(!follow.LiveStatusUpdatedAt.HasValue ||
DateTime.Now - follow.LiveStatusUpdatedAt.Value > StaleAfter ||
!string.IsNullOrWhiteSpace(follow.LiveCheckError));
var roomUrl = follow.LiveStatus == DouyinLiveStatusState.Live && !string.IsNullOrWhiteSpace(follow.LiveWebRid)
? "https://live.douyin.com/" + Uri.EscapeDataString(follow.LiveWebRid)
: null;
return new FollowLiveStatusDto
{
Id = follow.Id,
LiveMonitorEnabled = follow.LiveMonitorEnabled,
LiveStatus = follow.LiveStatus,
LiveRoomId = follow.LiveRoomId,
LiveWebRid = follow.LiveWebRid,
LiveTitle = follow.LiveTitle,
LiveRoomUrl = roomUrl,
LiveCheckedAt = follow.LiveCheckedAt,
LiveStatusUpdatedAt = follow.LiveStatusUpdatedAt,
LiveStartedAt = follow.LiveStartedAt,
LiveCheckError = follow.LiveCheckError,
LiveStatusStale = stale,
LiveEmailNotificationEnabled = follow.LiveEmailNotificationEnabled,
LastLiveNotifiedAt = follow.LastLiveNotifiedAt,
LastLiveNotificationError = follow.LastLiveNotificationError
};
}
private static string Limit(string value, int maxLength)
{
value = value?.Trim();
return value != null && value.Length > maxLength ? value[..maxLength] : value;
}
private readonly record struct ProbeResult(bool ShouldStopAccount);
}
}
+47 -20
View File
@@ -102,12 +102,14 @@ namespace dy.net.service
/// 视频合成(优化后:解决 FFmpeg 进程冲突,支持容错重试)
/// </summary>
public async Task<bool> MergeToVideo(string cookie, string rootPath, MediaMergeRequest request,
string outputVideoPath, string fileNamefolder, bool mergeImg2Viedo, bool downImage = false, bool downMp3 = false)
string outputVideoPath, string fileNamefolder, bool mergeImg2Viedo, bool downImage = false,
bool downMp3 = false, Action<string> reportWarning = null, Action<string> reportError = null)
{
// 输入参数校验(避免无效执行)
if (request == null || request.ImageUrls == null || request.ImageUrls.Count == 0)
{
Log.Error("图片URL列表为空,无法合成视频");
reportError?.Invoke("图片 URL 列表为空");
return false;
}
//if (mergeImg2Viedo && (request.AudioUrls == null || request.AudioUrls.Count == 0))
@@ -128,7 +130,9 @@ namespace dy.net.service
request.ImageUrls, Path.Combine(tempDir, "raw-images"), "image_", "webp", cookie);
if (!string.IsNullOrEmpty(imageError) || rawImages == null || rawImages.Count == 0)
{
Log.Error($"图片下载失败:{imageError ?? ""}");
var reason = imageError ?? "未下载到任何图片";
Log.Error($"图片下载失败:{reason}");
reportError?.Invoke($"图片下载失败:{reason}");
return false;
}
// 保存下载的图片(如果需要)
@@ -138,19 +142,25 @@ namespace dy.net.service
}
// 2. 下载音频
List<DouyinMergeVideoDto> rawAudios =new List<DouyinMergeVideoDto>();
if (mergeImg2Viedo && request.AudioUrls != null && request.AudioUrls.Count > 0)
List<DouyinMergeVideoDto> rawAudios = new List<DouyinMergeVideoDto>();
var audioWarningReported = false;
if ((mergeImg2Viedo || downMp3) && request.AudioUrls != null && request.AudioUrls.Count > 0)
{
var (audios, audioError) = await DownloadMediaAsync(
request.AudioUrls.Select(x => new DouyinMergeVideoDto { Path = x })?.ToList(), Path.Combine(tempDir, "raw-audios"), "audio_", "mp3", cookie);
if (!string.IsNullOrEmpty(audioError) || audios == null || audios.Count == 0)
{
Log.Error($"音频下载失败:{audioError ?? ""}");
return false;
var warning = downMp3
? "作品音频不可用,已降级处理,未生成单独音频文件"
: "作品音频不可用,图文视频已降级为无声视频";
Log.Warning("音频下载失败:{Reason}{Fallback}",
audioError ?? "未下载到任何音频", warning);
reportWarning?.Invoke(warning);
audioWarningReported = true;
}
rawAudios = audios;
else rawAudios = audios;
// 保存下载的音频(如果需要)
if (downMp3)
if (downMp3 && rawAudios.Count > 0)
{
var ext = Path.GetExtension(rawAudios.FirstOrDefault()?.Path);
await SaveDownloadedFilesAsync(rawAudios, fileNamefolder, ext, Path.GetFileNameWithoutExtension(outputVideoPath));
@@ -161,7 +171,11 @@ namespace dy.net.service
if (!mergeImg2Viedo)
{
Log.Debug($"根据系统配置设置不下载图文视频-[{outputVideoPath}],{(downImage ? "" : "")},{(downMp3 ? "" : "")}");
return true;
// When only an audio attachment was requested, an unavailable audio source
// leaves no persistent artifact and must remain a real item-level failure.
var hasArtifact = downImage || !downMp3 || rawAudios.Count > 0;
if (!hasArtifact) reportError?.Invoke("音频下载失败,且当前任务未配置保存图片");
return hasArtifact;
}
// 3. 调整图片显示时长
@@ -194,14 +208,20 @@ namespace dy.net.service
if (rawAudios.Count == 0)
{
var musicPath = GetRandomMergeMusic();
rawAudios = new List<DouyinMergeVideoDto> { new DouyinMergeVideoDto { Path= musicPath } };
Log.Debug("版权原因无法下载音频,使用默认无声音频文件");
if (!audioWarningReported)
{
var warning = downMp3
? "作品没有可用音频,已生成无声视频,未生成单独音频文件"
: "作品没有可用音频,已生成无声视频";
reportWarning?.Invoke(warning);
audioWarningReported = true;
}
Log.Warning("图文作品没有可用音频,将直接生成无声视频");
}
string resultPath = await ffmpegHelper.CreateVideoFromImagesAndAudioAsync(
rawImages, // 所有下载的图片(确保无遗漏)
rawAudios.FirstOrDefault()?.Path, // 取第一个音频(可根据需求调整)
rawAudios.FirstOrDefault()?.Path, // 音频缺失时 FFmpegHelper 会生成无声视频
outputVideoPath,
ffmpegHelper.VideoWidth,
ffmpegHelper.VideoHeight,
@@ -218,11 +238,13 @@ namespace dy.net.service
}
});
if (!mergeSuccess) reportError?.Invoke("FFmpeg 未生成有效的视频文件");
return mergeSuccess;
}
catch (Exception ex)
{
Log.Error(ex, "视频合成过程中发生未处理异常");
reportError?.Invoke($"{ex.GetType().Name}{ex.GetBaseException().Message}");
return false;
}
finally
@@ -330,6 +352,7 @@ namespace dy.net.service
List<DouyinMergeVideoDto> urls, string saveDir, string prefix, string ext, string cookie)
{
var successPaths = new List<DouyinMergeVideoDto>();
var failures = new List<string>();
for (var i = 0; i < urls.Count; i++)
{
var url = urls[i];
@@ -339,12 +362,12 @@ namespace dy.net.service
try
{
var (Success, ActualSavePath) = await douyinHttpClientService.DownloadAsync(url.Path, savePath, cookie);
if (Success)
var result = await douyinHttpClientService.DownloadAsync(url.Path, savePath, cookie);
if (result.Success)
{
successPaths.Add(new DouyinMergeVideoDto
{
Path = ActualSavePath,
Path = result.ActualSavePath,
Height = url.Height,
Width =
url.Width
@@ -352,18 +375,22 @@ namespace dy.net.service
}
else
{
Serilog.Log.Error($"下载失败:{url} → {savePath}");
failures.Add($"{result.SourceHost ?? ""}/HTTP {result.HttpStatusCode?.ToString() ?? "-"}/{result.FailureKind}{result.Message}");
Serilog.Log.Warning("合成素材下载失败:Host={Host}, Status={Status}, Kind={Kind}, Target={Target}",
result.SourceHost, result.HttpStatusCode, result.FailureKind, savePath);
}
//Console.WriteLine($"下载成功:{url} → {savePath}");
}
catch (Exception ex)
{
var error = $"下载失败:{url},错误:{ex.Message}";
Serilog.Log.Error(error);
var error = $"合成素材下载失败:{ex.GetType().Name}{ex.GetBaseException().Message}";
Serilog.Log.Error("合成素材下载异常:Type={ExceptionType}, Target={Target}", ex.GetType().Name, savePath);
return (new List<DouyinMergeVideoDto>(), error);
}
}
return (successPaths, null);
return successPaths.Count == 0 && failures.Count > 0
? (successPaths, string.Join("", failures.Take(3)))
: (successPaths, null);
}
}
+75
View File
@@ -0,0 +1,75 @@
using dy.net.model.dto;
using dy.net.model.entity;
using dy.net.model.response;
using SqlSugar;
namespace dy.net.service
{
public class DouyinMigrationSourceResolver
{
private readonly DouyinHttpClientService _http;
private readonly ISqlSugarClient _db;
public DouyinMigrationSourceResolver(DouyinHttpClientService http, ISqlSugarClient db)
{
_http = http;
_db = db;
}
public async Task<IReadOnlyList<string>> ResolveAsync(DouyinVideo video, DouyinCookie cookie, CancellationToken cancellationToken)
{
if (video == null || cookie == null || string.IsNullOrWhiteSpace(cookie.Cookies)) return Array.Empty<string>();
var category = string.IsNullOrWhiteSpace(video.CateId)
? null
: await _db.Queryable<DouyinCollectCate>().InSingleAsync(video.CateId);
var followed = video.ViedoType == VideoTypeEnum.dy_follows
? await _db.Queryable<DouyinFollowed>().Where(x => x.UperId == video.AuthorId).FirstAsync()
: null;
var cursor = "0";
for (var page = 0; page < 500 && !cancellationToken.IsCancellationRequested; page++)
{
var response = await FetchAsync(video, cookie, category, followed, cursor);
if (response?.StatusCode != 0 || response.AwemeList == null) return Array.Empty<string>();
var match = response.AwemeList.FirstOrDefault(x => x.AwemeId == video.AwemeId);
if (match != null) return ExtractUrls(match);
if (response.HasMore != 1) break;
var next = response.Cursor ?? response.MaxCursor;
if (string.IsNullOrWhiteSpace(next) || next == cursor) break;
cursor = next;
}
return Array.Empty<string>();
}
private async Task<DouyinVideoInfoResponse> FetchAsync(
DouyinVideo video,
DouyinCookie cookie,
DouyinCollectCate category,
DouyinFollowed followed,
string cursor) => video.ViedoType switch
{
VideoTypeEnum.dy_favorite when !string.IsNullOrWhiteSpace(cookie.SecUserId) =>
await _http.SyncFavoriteVideos("18", cursor, cookie.SecUserId, cookie.Cookies),
VideoTypeEnum.dy_follows when !string.IsNullOrWhiteSpace(followed?.SecUid) =>
await _http.SyncUpderPostVideos("18", cursor, followed.SecUid, cookie.Cookies),
VideoTypeEnum.dy_custom_collect when !string.IsNullOrWhiteSpace(category?.XId) =>
await _http.SyncCollectVideosByCollectId(cursor, "18", cookie.Cookies, category.XId),
VideoTypeEnum.dy_mix when !string.IsNullOrWhiteSpace(category?.XId ?? video.CateXId) =>
await _http.SyncMixViedosByMixId(cursor, "18", cookie.Cookies, category?.XId ?? video.CateXId),
VideoTypeEnum.dy_series when !string.IsNullOrWhiteSpace(category?.XId ?? video.CateXId) =>
await _http.SyncSeriesViedosByMSeriesId(cursor, "18", cookie.Cookies, category?.XId ?? video.CateXId),
_ => await _http.SyncCollectVideos(cursor, "18", cookie.Cookies)
};
private static IReadOnlyList<string> ExtractUrls(Aweme aweme)
{
return (aweme.Video?.BitRate ?? new List<VideoBitRate>())
.Where(x => x?.PlayAddr?.UrlList != null)
.OrderByDescending(x => x.BitRateValue)
.SelectMany(x => x.PlayAddr.UrlList)
.Where(x => !string.IsNullOrWhiteSpace(x))
.Distinct()
.ToArray();
}
}
}
+121 -10
View File
@@ -14,6 +14,8 @@ namespace dy.net.service
{
private readonly ISchedulerFactory _schedulerFactory;
private readonly DouyinCookieService douyinCookieService;
private readonly DouyinCommonService _commonService;
private readonly VideoTaskService _videoTasks;
private const string DefaultJobGroup = "dysync.net";
private const int DefaultIntervalMinutes = 30;
private const int DefaultCronStartDelaySeconds = 30;
@@ -85,15 +87,91 @@ namespace dy.net.service
"dy.job.key.sync_follow_user_once",
"dy.trigger.key.sync_follow_user_once",
"抖音关注同步任务(单次执行)")
},
{
VideoTypeEnum.dy_live_monitor,
new JobConfig(
typeof(DouyinLiveStatusJob),
"dy.job.key.live_monitor",
"dy.trigger.key.live_monitor",
"抖音博主直播状态监测任务")
}
};
public DouyinQuartzJobService(ISchedulerFactory schedulerFactory,DouyinCookieService douyinCookieService)
public DouyinQuartzJobService(
ISchedulerFactory schedulerFactory,
DouyinCookieService douyinCookieService,
DouyinCommonService commonService,
VideoTaskService videoTasks)
{
_schedulerFactory = schedulerFactory ?? throw new ArgumentNullException(nameof(schedulerFactory));
this.douyinCookieService = douyinCookieService;
_commonService = commonService;
_videoTasks = videoTasks;
}
public async Task<List<VideoDownloadTask>> TriggerVideoJobsNowAsync(VideoTypeEnum? requestedType = null)
{
var cookies = await douyinCookieService.GetOpendCookiesAsync();
if (cookies == null || cookies.Count == 0) throw new InvalidOperationException("没有已启用的抖音授权账号。");
var storageType = _commonService.GetConfig()?.StorageType ?? StorageType.Local;
var enabled = GetTaskEnableConditions(cookies, storageType);
var types = new List<VideoTypeEnum>();
if (enabled.IsFavoriteEnabled) types.Add(VideoTypeEnum.dy_favorite);
if (enabled.IsCollectEnabled) types.Add(VideoTypeEnum.dy_collects);
if (enabled.IsFollowedEnabled) types.Add(VideoTypeEnum.dy_follows);
if (enabled.IsCustomCollectEnabled) types.Add(VideoTypeEnum.dy_custom_collect);
if (enabled.IsMixEnabled) types.Add(VideoTypeEnum.dy_mix);
if (enabled.IsSeriesEnabled) types.Add(VideoTypeEnum.dy_series);
if (types.Count == 0) throw new InvalidOperationException("没有已启用且路径完整的视频同步类型。");
if (requestedType.HasValue)
{
if (!IsVideoSyncType(requestedType.Value))
throw new InvalidOperationException("指定的任务类型不是可手动执行的视频同步类型。");
if (!types.Contains(requestedType.Value))
{
var message = requestedType.Value == VideoTypeEnum.dy_follows
? "关注视频同步未启用:请检查账号的关注下载开关、当前存储的关注路径,以及是否有博主开启同步。"
: $"{requestedType.Value.GetDesc()}同步未启用或当前存储路径未配置。";
throw new InvalidOperationException(message);
}
types = new List<VideoTypeEnum> { requestedType.Value };
}
var scheduler = await _schedulerFactory.GetScheduler();
var created = new List<VideoDownloadTask>();
foreach (var type in types)
{
var config = JobConfigs[type];
var jobKey = new JobKey(config.JobKey, DefaultJobGroup);
if (!await scheduler.CheckExists(jobKey))
await StartJobAsync(type, (_commonService.GetConfig()?.Cron ?? DefaultIntervalMinutes).ToString());
var task = await _videoTasks.CreateTaskAsync(VideoTaskType.Sync, VideoTaskTrigger.Manual,
$"{type.GetDesc()}手动同步", type, storageType);
try
{
await scheduler.TriggerJob(jobKey, new JobDataMap
{
["video-task-id"] = task.Id,
["video-task-trigger"] = "manual"
});
}
catch (Exception ex)
{
var message = $"{type.GetDesc()}手动同步未能进入调度,请稍后重试。";
await _videoTasks.CompleteTaskAsync(task.Id, message);
Log.Error(ex, "手动同步任务调度失败:{VideoType}, TaskId={TaskId}", type, task.Id);
throw new InvalidOperationException(message, ex);
}
created.Add(task);
}
return created;
}
internal static bool IsVideoSyncType(VideoTypeEnum type) => type is
VideoTypeEnum.dy_favorite or VideoTypeEnum.dy_collects or VideoTypeEnum.dy_follows or
VideoTypeEnum.dy_custom_collect or VideoTypeEnum.dy_mix or VideoTypeEnum.dy_series;
/// <summary>
/// 初始化或重启所有抖音定时任务
@@ -126,7 +204,8 @@ namespace dy.net.service
await RemoveAllExistingJobs(scheduler);
// 4. 检查各类型任务的启用条件
var taskEnableConditions = GetTaskEnableConditions(validCookies);
var storageType = _commonService.GetConfig()?.StorageType ?? StorageType.Local;
var taskEnableConditions = GetTaskEnableConditions(validCookies, storageType);
// 5. 启动符合条件的定时任务
int successfullyStartedJobs = 0;
@@ -144,6 +223,14 @@ namespace dy.net.service
continue;
}
// 直播监测完全独立于视频同步,固定每5分钟检查已单独开启的博主。
if (jobKey == VideoTypeEnum.dy_live_monitor)
{
bool startSuccess = await StartSingleJobAsync(jobKey, "5");
if (startSuccess) successfullyStartedJobs++;
continue;
}
// 根据不同任务类型和启用条件启动任务
bool isTaskEnabled = jobKey switch
{
@@ -196,16 +283,40 @@ namespace dy.net.service
/// </summary>
/// <param name="cookies">有效的抖音Cookie列表</param>
/// <returns>任务启用条件集合</returns>
private static TaskEnableConditions GetTaskEnableConditions(IEnumerable<DouyinCookie> cookies)
private static TaskEnableConditions GetTaskEnableConditions(IEnumerable<DouyinCookie> cookies, StorageType storageType)
{
bool HasPath(DouyinCookie cookie, VideoTypeEnum type)
{
if (storageType.IsRemote())
{
return type switch
{
VideoTypeEnum.dy_favorite => !string.IsNullOrWhiteSpace(cookie.WebDavFavoritePath),
VideoTypeEnum.dy_follows => !string.IsNullOrWhiteSpace(cookie.WebDavFollowPath),
VideoTypeEnum.dy_mix => !string.IsNullOrWhiteSpace(cookie.WebDavMixPath),
VideoTypeEnum.dy_series => !string.IsNullOrWhiteSpace(cookie.WebDavSeriesPath),
_ => !string.IsNullOrWhiteSpace(cookie.WebDavCollectPath)
};
}
return type switch
{
VideoTypeEnum.dy_favorite => !string.IsNullOrWhiteSpace(cookie.FavSavePath),
VideoTypeEnum.dy_follows => !string.IsNullOrWhiteSpace(cookie.UpSavePath),
VideoTypeEnum.dy_mix => !string.IsNullOrWhiteSpace(cookie.MixPath) || !string.IsNullOrWhiteSpace(cookie.SavePath),
VideoTypeEnum.dy_series => !string.IsNullOrWhiteSpace(cookie.SeriesPath) || !string.IsNullOrWhiteSpace(cookie.SavePath),
_ => !string.IsNullOrWhiteSpace(cookie.SavePath)
};
}
return new TaskEnableConditions
{
IsCollectEnabled = cookies.Any(x => x.DownCollect && !x.UseCollectFolder && !string.IsNullOrWhiteSpace(x.SavePath)),
IsFavoriteEnabled = cookies.Any(x => x.DownFavorite && !string.IsNullOrWhiteSpace(x.FavSavePath)),
IsFollowedEnabled = cookies.Any(x => x.DownFollowd && !string.IsNullOrWhiteSpace(x.UpSavePath)),
IsMixEnabled = cookies.Any(x => x.DownMix && !string.IsNullOrWhiteSpace(x.MixPath)),
IsSeriesEnabled = cookies.Any(x => x.DownSeries && !string.IsNullOrWhiteSpace(x.SeriesPath)),
IsCustomCollectEnabled = cookies.Any(x => x.UseCollectFolder && !string.IsNullOrWhiteSpace(x.SavePath))
IsCollectEnabled = cookies.Any(x => x.DownCollect && !x.UseCollectFolder && HasPath(x, VideoTypeEnum.dy_collects)),
IsFavoriteEnabled = cookies.Any(x => x.DownFavorite && HasPath(x, VideoTypeEnum.dy_favorite)),
IsFollowedEnabled = cookies.Any(x => x.DownFollowd && HasPath(x, VideoTypeEnum.dy_follows)),
IsMixEnabled = cookies.Any(x => x.DownMix && HasPath(x, VideoTypeEnum.dy_mix)),
IsSeriesEnabled = cookies.Any(x => x.DownSeries && HasPath(x, VideoTypeEnum.dy_series)),
IsCustomCollectEnabled = cookies.Any(x => x.UseCollectFolder && HasPath(x, VideoTypeEnum.dy_custom_collect))
};
}
@@ -423,4 +534,4 @@ namespace dy.net.service
public bool IsCustomCollectEnabled { get; set; }
}
}
}
}
+137
View File
@@ -0,0 +1,137 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using dy.net.model.dto;
using dy.net.model.entity;
using SqlSugar;
namespace dy.net.service
{
public sealed class DouyinUserLookupService
{
private static readonly TimeSpan CacheLifetime = TimeSpan.FromMinutes(5);
private static readonly SemaphoreSlim SearchGate = new(1, 1);
private static readonly ConcurrentDictionary<string, CacheEntry> Cache = new(StringComparer.Ordinal);
private static readonly Regex LabeledDouyinNo = new(
@"^(?:抖音号|douyin)\s*[:]\s*([^\s,;]+)",
RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex LabeledDouyinNoPrefix = new(
@"^(?:抖音号|douyin)\s*[:]",
RegexOptions.IgnoreCase | RegexOptions.Compiled);
private readonly ISqlSugarClient _db;
private readonly DouyinUserSearchClient _searchClient;
public DouyinUserLookupService(ISqlSugarClient db, DouyinUserSearchClient searchClient)
{
_db = db;
_searchClient = searchClient;
}
public async Task<DouyinFollowLookupResult> ResolveAsync(
DouyinFollowLookupRequest request,
CancellationToken cancellationToken = default)
{
if (request == null) throw new InvalidOperationException("查询参数不能为空");
var query = NormalizeDouyinNo(request.DouyinNo);
if (string.IsNullOrWhiteSpace(request.CookieId)) throw new InvalidOperationException("请选择抖音授权账号");
var cookie = await _db.Queryable<DouyinCookie>().InSingleAsync(request.CookieId);
if (cookie == null) throw new InvalidOperationException("选择的抖音授权账号不存在");
if (cookie.Status != 1 || cookie.StatusCode != 0 || string.IsNullOrWhiteSpace(cookie.Cookies))
throw new InvalidOperationException("当前账号 Cookie 无效或未启用,请先更新抖音授权");
if (string.IsNullOrWhiteSpace(cookie.MyUserId))
throw new InvalidOperationException("当前账号缺少用户 UID,请先完善抖音授权配置");
var cacheKey = cookie.Id + ":" + query.ToUpperInvariant();
var candidates = TryGetCached(cacheKey);
if (candidates == null)
{
await SearchGate.WaitAsync(cancellationToken);
try
{
candidates = TryGetCached(cacheKey);
if (candidates == null)
{
candidates = await _searchClient.SearchAsync(query, cookie.Cookies, cancellationToken);
Cache[cacheKey] = new CacheEntry(DateTimeOffset.UtcNow.Add(CacheLifetime), Clone(candidates));
}
}
finally
{
SearchGate.Release();
}
}
if (candidates.Count == 0) throw new InvalidOperationException("未找到对应博主,请检查抖音号或使用手动添加");
var secUids = candidates.Select(x => x.SecUid).Distinct().ToList();
var existing = await _db.Queryable<DouyinFollowed>()
.Where(x => x.mySelfId == cookie.MyUserId && secUids.Contains(x.SecUid))
.Select(x => x.SecUid)
.ToListAsync();
var existingSet = existing.ToHashSet(StringComparer.Ordinal);
foreach (var candidate in candidates)
{
candidate.ExactMatch = string.Equals(candidate.UniqueId, query, StringComparison.OrdinalIgnoreCase)
|| string.Equals(candidate.ShortId, query, StringComparison.OrdinalIgnoreCase);
candidate.AlreadyExists = existingSet.Contains(candidate.SecUid);
}
return new DouyinFollowLookupResult
{
Query = query,
Candidates = candidates
.OrderByDescending(x => x.ExactMatch)
.ThenByDescending(x => x.FollowerCount)
.Take(10)
.ToList()
};
}
public static string NormalizeDouyinNo(string input)
{
var value = (input ?? string.Empty).Trim();
var match = LabeledDouyinNo.Match(value);
if (match.Success)
value = match.Groups[1].Value;
else if (LabeledDouyinNoPrefix.IsMatch(value))
value = string.Empty;
value = value.Trim().TrimStart('@').Trim();
if (string.IsNullOrWhiteSpace(value)) throw new InvalidOperationException("请输入抖音号");
if (value.Length > 64) throw new InvalidOperationException("抖音号长度不能超过 64 个字符");
if (value.Any(char.IsWhiteSpace) || value.Any(char.IsControl))
throw new InvalidOperationException("抖音号格式不正确");
return value;
}
private static List<DouyinFollowCandidate> TryGetCached(string key)
{
if (!Cache.TryGetValue(key, out var entry)) return null;
if (entry.ExpiresAt <= DateTimeOffset.UtcNow)
{
Cache.TryRemove(key, out _);
return null;
}
return Clone(entry.Candidates);
}
private static List<DouyinFollowCandidate> Clone(IEnumerable<DouyinFollowCandidate> source) =>
source.Select(x => new DouyinFollowCandidate
{
SecUid = x.SecUid,
UperId = x.UperId,
DouyinNo = x.DouyinNo,
UniqueId = x.UniqueId,
ShortId = x.ShortId,
UperName = x.UperName,
UperAvatar = x.UperAvatar,
Signature = x.Signature,
Enterprise = x.Enterprise,
FollowerCount = x.FollowerCount,
ExactMatch = x.ExactMatch,
AlreadyExists = x.AlreadyExists
}).ToList();
private sealed record CacheEntry(DateTimeOffset ExpiresAt, List<DouyinFollowCandidate> Candidates);
}
}
+217
View File
@@ -0,0 +1,217 @@
using dy.net.model.dto;
using dy.net.utils;
using Newtonsoft.Json;
namespace dy.net.service
{
public sealed class DouyinUserSearchClient
{
private const string SearchPath = "/aweme/v1/web/discover/search/";
private readonly IHttpClientFactory _clientFactory;
private readonly DouyinABogusSigner _signer = new();
public DouyinUserSearchClient(IHttpClientFactory clientFactory) => _clientFactory = clientFactory;
public async Task<List<DouyinFollowCandidate>> SearchAsync(
string douyinNo,
string cookie,
CancellationToken cancellationToken = default)
{
var parameters = BuildParameters(douyinNo, cookie);
var unsignedQuery = BuildQueryString(parameters);
var signature = _signer.Sign(unsignedQuery);
var requestUri = SearchPath + "?" + unsignedQuery + "&a_bogus=" + Encode(signature);
using var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
request.Headers.Referrer = new Uri("https://www.douyin.com/root/search/" + Encode(douyinNo) + "?type=user");
request.Headers.TryAddWithoutValidation("Accept", "*/*");
request.Headers.TryAddWithoutValidation("Cookie", cookie);
var client = _clientFactory.CreateClient(DouyinRequestParamManager.DY_HTTP_CLIENT);
try
{
using var response = await client.SendAsync(
request,
HttpCompletionOption.ResponseHeadersRead,
cancellationToken);
if (!response.IsSuccessStatusCode)
throw new InvalidOperationException($"抖音用户搜索请求失败(HTTP {(int)response.StatusCode}");
var content = await response.Content.ReadAsStringAsync(cancellationToken);
SearchResponse payload;
try
{
payload = JsonConvert.DeserializeObject<SearchResponse>(content);
}
catch (JsonException)
{
throw new InvalidOperationException("抖音返回了风控验证页面,请稍后重试或使用手动添加");
}
if (payload == null)
throw new InvalidOperationException("抖音用户搜索没有返回有效数据");
if (payload.StatusCode != 0)
throw new InvalidOperationException("抖音用户搜索被拒绝,请确认 Cookie 有效后重试");
return (payload.Users ?? new List<SearchUserEntry>())
.Select(x => x.User)
.Where(x => x != null &&
!string.IsNullOrWhiteSpace(x.SecUid) &&
!string.IsNullOrWhiteSpace(x.Uid) &&
!string.IsNullOrWhiteSpace(x.Nickname))
.Select(x => new DouyinFollowCandidate
{
SecUid = x.SecUid,
UperId = x.Uid,
UniqueId = x.UniqueId,
ShortId = x.ShortId,
DouyinNo = !string.IsNullOrWhiteSpace(x.UniqueId) ? x.UniqueId : x.ShortId,
UperName = x.Nickname,
UperAvatar = x.Avatar?.Urls?.FirstOrDefault() ?? string.Empty,
Signature = x.Signature,
Enterprise = !string.IsNullOrWhiteSpace(x.Enterprise) ? x.Enterprise : x.CustomVerify,
FollowerCount = x.FollowerCount
})
.GroupBy(x => x.SecUid, StringComparer.Ordinal)
.Select(x => x.First())
.Take(10)
.ToList();
}
catch (HttpRequestException)
{
// 不向上抛出包含查询 URI 的原始异常,避免 msToken 出现在全局日志中。
throw new InvalidOperationException("抖音用户搜索网络请求失败,请稍后重试");
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
{
throw new InvalidOperationException("抖音用户搜索超时,请稍后重试");
}
}
private static List<KeyValuePair<string, string>> BuildParameters(string douyinNo, string cookie)
{
var cookies = ParseCookie(cookie);
return new List<KeyValuePair<string, string>>
{
new("device_platform", "webapp"),
new("aid", "6383"),
new("channel", "channel_pc_web"),
new("update_version_code", "170400"),
new("pc_client_type", "1"),
new("pc_libra_divert", "Windows"),
new("support_h265", "1"),
new("support_dash", "1"),
new("version_code", "170400"),
new("version_name", "17.4.0"),
new("cookie_enabled", "true"),
new("screen_width", "1536"),
new("screen_height", "864"),
new("browser_language", "zh-CN"),
new("browser_platform", "Win32"),
new("browser_name", "Chrome"),
new("browser_version", "119.0.0.0"),
new("browser_online", "true"),
new("engine_name", "Blink"),
new("engine_version", "119.0.0.0"),
new("os_name", "Windows"),
new("os_version", "10"),
new("cpu_core_num", "16"),
new("device_memory", "8"),
new("platform", "PC"),
new("downlink", "10"),
new("effective_type", "4g"),
new("round_trip_time", "200"),
new("uifid", CookieValue(cookies, "UIFID", "UIFID_TEMP")),
new("msToken", CookieValue(cookies, "msToken")),
new("pc_search_top_1_params", "{\"enable_ai_search_top_1\":1}"),
new("search_channel", "aweme_user_web"),
new("keyword", douyinNo),
new("search_source", "switch_tab"),
new("query_correct_type", "1"),
new("is_filter_search", "0"),
new("from_group_id", string.Empty),
new("disable_rs", "0"),
new("offset", "0"),
new("count", "10"),
new("need_filter_settings", "0"),
new("list_type", "single")
};
}
private static Dictionary<string, string> ParseCookie(string cookie)
{
var values = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var part in (cookie ?? string.Empty).Split(';', StringSplitOptions.RemoveEmptyEntries))
{
var separator = part.IndexOf('=');
if (separator <= 0) continue;
values[part[..separator].Trim()] = part[(separator + 1)..].Trim();
}
return values;
}
private static string CookieValue(Dictionary<string, string> cookies, params string[] names)
{
foreach (var name in names)
if (cookies.TryGetValue(name, out var value)) return value;
return string.Empty;
}
private static string BuildQueryString(IEnumerable<KeyValuePair<string, string>> parameters) =>
string.Join("&", parameters.Select(x => Encode(x.Key) + "=" + Encode(x.Value ?? string.Empty)));
private static string Encode(string value) => Uri.EscapeDataString(value ?? string.Empty);
private sealed class SearchResponse
{
[JsonProperty("status_code")]
public int StatusCode { get; set; }
[JsonProperty("user_list")]
public List<SearchUserEntry> Users { get; set; }
}
private sealed class SearchUserEntry
{
[JsonProperty("user_info")]
public SearchUser User { get; set; }
}
private sealed class SearchUser
{
[JsonProperty("sec_uid")]
public string SecUid { get; set; }
[JsonProperty("uid")]
public string Uid { get; set; }
[JsonProperty("unique_id")]
public string UniqueId { get; set; }
[JsonProperty("short_id")]
public string ShortId { get; set; }
[JsonProperty("nickname")]
public string Nickname { get; set; }
[JsonProperty("signature")]
public string Signature { get; set; }
[JsonProperty("enterprise_verify_reason")]
public string Enterprise { get; set; }
[JsonProperty("custom_verify")]
public string CustomVerify { get; set; }
[JsonProperty("follower_count")]
public long FollowerCount { get; set; }
[JsonProperty("avatar_thumb")]
public SearchAvatar Avatar { get; set; }
}
private sealed class SearchAvatar
{
[JsonProperty("url_list")]
public List<string> Urls { get; set; }
}
}
}
+305 -62
View File
@@ -5,6 +5,9 @@ using dy.net.repository;
using dy.net.utils;
using Serilog;
using SqlSugar;
using dy.net.storage;
using Newtonsoft.Json;
using MediaStorageType = dy.net.model.dto.StorageType;
namespace dy.net.service
{
@@ -15,12 +18,45 @@ namespace dy.net.service
private readonly DouyinVideoRepository _dyCollectVideoRepository;
private readonly DouyinCookieRepository douyinCookieRepository;
private readonly MediaStorageRouter _storageRouter;
private readonly VideoTaskService _videoTasks;
public DouyinVideoService(DouyinVideoRepository dyCollectVideoRepository, DouyinCookieRepository douyinCookieRepository, ISqlSugarClient sqlSugarClient)
public DouyinVideoService(DouyinVideoRepository dyCollectVideoRepository, DouyinCookieRepository douyinCookieRepository, ISqlSugarClient sqlSugarClient, MediaStorageRouter storageRouter, VideoTaskService videoTasks)
{
_dyCollectVideoRepository = dyCollectVideoRepository;
this.douyinCookieRepository = douyinCookieRepository;
this.sqlSugarClient = sqlSugarClient;
_storageRouter = storageRouter;
_videoTasks = videoTasks;
}
public async Task<StorageRecordInventory> GetStorageInventoryAsync(MediaStorageType currentStorageType)
{
var records = await sqlSugarClient.Queryable<DouyinVideo>()
.Select(x => new DouyinVideo { StorageType = x.StorageType, FileSize = x.FileSize })
.ToListAsync();
var local = records.Where(x => x.StorageType == MediaStorageType.Local).ToList();
var webDav = records.Where(x => x.StorageType == MediaStorageType.WebDav).ToList();
var openList = records.Where(x => x.StorageType == MediaStorageType.OpenList).ToList();
var currentCount = currentStorageType switch
{
MediaStorageType.WebDav => webDav.Count,
MediaStorageType.OpenList => openList.Count,
_ => local.Count
};
return new StorageRecordInventory
{
CurrentStorageType = currentStorageType,
TotalRecordCount = records.Count,
LocalRecordCount = local.Count,
WebDavRecordCount = webDav.Count,
OpenListRecordCount = openList.Count,
CurrentStorageRecordCount = currentCount,
OtherStorageRecordCount = records.Count - currentCount,
LocalDeclaredBytes = local.Sum(x => x.FileSize),
WebDavDeclaredBytes = webDav.Sum(x => x.FileSize),
OpenListDeclaredBytes = openList.Sum(x => x.FileSize)
};
}
@@ -76,9 +112,23 @@ namespace dy.net.service
{
if (updateMap.TryGetValue(existingVideo.AwemeId, out var updateData))
{
// 保留原记录主键,并把实际主键回写给任务条目使用。
updateData.Id = existingVideo.Id;
existingVideo.VideoSavePath = updateData.VideoSavePath;
existingVideo.VideoCoverSavePath = updateData.VideoCoverSavePath;
existingVideo.ViedoType = updateData.ViedoType;
existingVideo.StorageType = updateData.StorageType;
existingVideo.FileSize = updateData.FileSize;
existingVideo.FileHash = updateData.FileHash;
existingVideo.Resolution = updateData.Resolution;
existingVideo.DynamicVideos = updateData.DynamicVideos;
existingVideo.OnlyImgOrOnlyMp3 = updateData.OnlyImgOrOnlyMp3;
existingVideo.IsMergeVideo = updateData.IsMergeVideo;
existingVideo.VideoUrl = updateData.VideoUrl;
existingVideo.AuthorAvatar = updateData.AuthorAvatar;
existingVideo.CateId = updateData.CateId;
existingVideo.CateXId = updateData.CateXId;
existingVideo.SyncTime = updateData.SyncTime;
}
}
// 批量更新数据库
@@ -98,6 +148,167 @@ namespace dy.net.service
return transaction;
}
/// <summary>
/// 普通同步已经把同一作品提交到远端存储后,安全清理不再被本地记录引用的旧文件。
/// 数据库或远端主媒体未验证通过时不会删除任何本地文件。
/// </summary>
public async Task<int> CleanupSupersededLocalArtifactsAsync(DouyinVideo snapshot, string replacementPath)
{
if (snapshot == null || snapshot.StorageType != MediaStorageType.Local
|| string.IsNullOrWhiteSpace(snapshot.AwemeId) || string.IsNullOrWhiteSpace(replacementPath)) return 0;
var current = await sqlSugarClient.Queryable<DouyinVideo>()
.Where(x => x.AwemeId == snapshot.AwemeId).FirstAsync();
if (current == null || !current.StorageType.IsRemote()
|| !string.Equals(current.VideoSavePath, replacementPath, StringComparison.Ordinal))
throw new InvalidOperationException("新存储记录尚未提交,保留旧本地文件。");
var remoteLength = await _storageRouter.Resolve(current.StorageType).GetLengthAsync(replacementPath);
if (!remoteLength.HasValue || remoteLength.Value <= 0)
throw new InvalidOperationException("新存储主媒体不存在或为空,保留旧本地文件。");
var cookie = await sqlSugarClient.Queryable<DouyinCookie>().InSingleAsync(snapshot.CookieId);
var roots = StorageMigrationPathPolicy.GetLocalRoots(cookie);
if (roots.Count == 0) throw new InvalidOperationException("旧存储根目录不可用,保留旧本地文件。");
var localVideos = await sqlSugarClient.Queryable<DouyinVideo>()
.Where(x => x.StorageType == MediaStorageType.Local).ToListAsync();
var deleted = 0;
foreach (var candidate in BuildLocalCleanupCandidates(snapshot))
{
if (!File.Exists(candidate.Path)) continue;
if (!SafeLocalMigrationFile.TryResolve(candidate.Path, roots, out var safePath, out var error))
throw new InvalidOperationException($"拒绝清理不安全旧路径 {candidate.Path}{error}");
if (IsLocalPathReferenced(candidate.Path, localVideos, snapshot.Id, candidate.Shared)) continue;
File.Delete(safePath);
deleted++;
}
DeleteEmptyParents(snapshot.VideoSavePath, roots);
return deleted;
}
public async Task<int> RetryTaskItemLocalCleanupAsync(string taskId, string itemId)
{
var task = await sqlSugarClient.Queryable<VideoDownloadTask>().InSingleAsync(taskId)
?? throw new KeyNotFoundException("任务不存在。");
if (task.Type != VideoTaskType.Sync)
throw new InvalidOperationException("仅普通同步任务支持重试旧本地文件清理。");
var item = await sqlSugarClient.Queryable<VideoDownloadTaskItem>().InSingleAsync(itemId)
?? throw new KeyNotFoundException("任务条目不存在。");
if (item.TaskId != taskId || !VideoTaskService.CanRetryCleanup(item))
throw new InvalidOperationException("该条目当前没有可重试的旧本地文件清理。");
DouyinVideo replacement;
try
{
replacement = JsonConvert.DeserializeObject<DouyinVideo>(item.RetrySnapshotJson);
}
catch (JsonException ex)
{
throw new InvalidOperationException("旧本地记录快照损坏,无法安全重试清理。", ex);
}
var snapshot = replacement?.SupersededStorageSnapshot;
if (snapshot == null || snapshot.StorageType != MediaStorageType.Local)
throw new InvalidOperationException("任务没有可验证的旧本地记录快照,拒绝清理。");
var current = !string.IsNullOrWhiteSpace(item.VideoId)
? await sqlSugarClient.Queryable<DouyinVideo>().InSingleAsync(item.VideoId)
: null;
current ??= await sqlSugarClient.Queryable<DouyinVideo>()
.Where(x => x.AwemeId == item.AwemeId).FirstAsync();
if (current == null)
throw new InvalidOperationException("当前视频记录不存在,拒绝清理旧文件。");
try
{
var deleted = await CleanupSupersededLocalArtifactsAsync(snapshot, current.VideoSavePath);
item.CleanupPending = false;
item.CleanupError = null;
item.WarningMessage = RemoveCleanupWarning(item.WarningMessage);
item.Stage = string.IsNullOrWhiteSpace(item.WarningMessage)
? VideoTaskItemStage.Succeeded
: VideoTaskItemStage.SucceededWithWarnings;
item.UpdatedAt = DateTime.Now;
await sqlSugarClient.Updateable(item).ExecuteCommandAsync();
await _videoTasks.RefreshCountsAsync(taskId);
return deleted;
}
catch (Exception ex)
{
var reason = ex.GetBaseException().Message;
item.CleanupPending = true;
item.CleanupError = reason;
item.UpdatedAt = DateTime.Now;
await sqlSugarClient.Updateable(item).ExecuteCommandAsync();
await _videoTasks.RefreshCountsAsync(taskId);
throw new InvalidOperationException($"旧本地文件清理仍失败:{reason}", ex);
}
}
internal static string RemoveCleanupWarning(string warning)
{
if (string.IsNullOrWhiteSpace(warning)) return null;
var failureIndex = warning.IndexOf("旧本地文件清理失败", StringComparison.Ordinal);
if (failureIndex < 0) return warning.Trim();
var start = warning.LastIndexOf(";已切换到", failureIndex, StringComparison.Ordinal);
if (start >= 0) return warning[..start].Trim().TrimEnd('');
start = warning.LastIndexOf("已切换到", failureIndex, StringComparison.Ordinal);
if (start == 0) return null;
return warning[..failureIndex].Trim().TrimEnd('', '', ',');
}
private static List<LocalCleanupCandidate> BuildLocalCleanupCandidates(DouyinVideo snapshot)
{
var result = new List<LocalCleanupCandidate>
{
new(snapshot.VideoSavePath, false),
new(snapshot.VideoCoverSavePath, snapshot.ViedoType is VideoTypeEnum.dy_mix or VideoTypeEnum.dy_series),
new(snapshot.AuthorAvatar, true)
};
if (!string.IsNullOrWhiteSpace(snapshot.VideoSavePath))
{
var directory = Path.GetDirectoryName(snapshot.VideoSavePath);
var basename = Path.GetFileNameWithoutExtension(snapshot.VideoSavePath);
result.Add(new LocalCleanupCandidate(Path.Combine(directory ?? string.Empty, basename + ".nfo"), false));
if (snapshot.ViedoType is VideoTypeEnum.dy_mix or VideoTypeEnum.dy_series)
result.Add(new LocalCleanupCandidate(Path.Combine(directory ?? string.Empty, "tvshow.nfo"), true));
}
if (!string.IsNullOrWhiteSpace(snapshot.DynamicVideos))
{
try
{
foreach (var attachment in JsonConvert.DeserializeObject<List<DouyinMergeVideoDto>>(snapshot.DynamicVideos) ?? new())
result.Add(new LocalCleanupCandidate(attachment.Path, false));
}
catch (JsonException) { }
}
return result.Where(x => !string.IsNullOrWhiteSpace(x.Path)).DistinctBy(x => x.Path).ToList();
}
private static bool IsLocalPathReferenced(string path, IEnumerable<DouyinVideo> localVideos, string excludedVideoId, bool shared)
{
foreach (var video in localVideos.Where(x => x.Id != excludedVideoId))
{
if (path == video.VideoSavePath || path == video.VideoCoverSavePath || path == video.AuthorAvatar) return true;
if (shared && !string.IsNullOrWhiteSpace(video.VideoSavePath)
&& string.Equals(Path.GetDirectoryName(video.VideoSavePath), Path.GetDirectoryName(path), StringComparison.Ordinal)) return true;
if (!string.IsNullOrWhiteSpace(video.DynamicVideos) && video.DynamicVideos.Contains(path, StringComparison.Ordinal)) return true;
}
return false;
}
private static void DeleteEmptyParents(string filePath, IReadOnlyList<string> roots)
{
if (string.IsNullOrWhiteSpace(filePath)) return;
var directory = Path.GetDirectoryName(Path.GetFullPath(filePath));
var root = roots.Where(x => SafeLocalMigrationFile.IsWithin(filePath, x))
.OrderByDescending(x => x.Length).FirstOrDefault();
while (!string.IsNullOrWhiteSpace(directory) && !string.Equals(directory, root, StringComparison.Ordinal)
&& Directory.Exists(directory) && !Directory.EnumerateFileSystemEntries(directory).Any())
{
Directory.Delete(directory, false);
directory = Path.GetDirectoryName(directory);
}
}
private sealed record LocalCleanupCandidate(string Path, bool Shared);
public async Task<bool> UpdateOne(DouyinVideo video)
{
return await _dyCollectVideoRepository.UpdateAsync(video);
@@ -246,31 +457,57 @@ namespace dy.net.service
return false;
}
// Permanent deletion is an explicit, separate operation. A normal re-download never removes
// the database row or old media first; VideoRedownloadWorker replaces the main file atomically.
if (forever)
{
foreach (var video in videos)
{
try
{
var storage = _storageRouter.Resolve(video.StorageType);
if (video.StorageType.IsRemote())
await StorageArtifactCleaner.DeleteWebDavArtifactsAsync(storage, video);
else
{
if (!string.IsNullOrWhiteSpace(video.VideoSavePath)) await storage.DeleteAsync(video.VideoSavePath);
if (!string.IsNullOrWhiteSpace(video.VideoCoverSavePath)) await storage.DeleteAsync(video.VideoCoverSavePath);
}
if (!await _dyCollectVideoRepository.DeleteByIdAsync(video.Id)) return false;
}
catch (Exception ex)
{
Serilog.Log.Error(ex, "永久删除视频失败,数据库记录已保留:{VideoId}", video.Id);
return false;
}
}
return true;
}
// 3. 构建重新下载记录(提前准备数据,避免事务内耗时操作)
var reDownList = new List<DouyinReDownload>();
var filePathsToDelete = new List<(string path, bool onlyImgOrMp3)>(); // 收集待删除文件路径,统一处理
foreach (var video in videos)
{
// 跳过无保存路径的视频(避免无效文件操作
// 跳过无保存路径的视频(避免创建无效任务
if (string.IsNullOrWhiteSpace(video.VideoSavePath))
{
Serilog.Log.Debug("视频无保存路径,跳过文件删除VideoId={0}", video.Id);
Serilog.Log.Debug("视频无保存路径,跳过重新下载VideoId={0}", video.Id);
continue;
}
// 构建重新下载记录
reDownList.Add(new DouyinReDownload
{
Id = IdGener.GetLong().ToString(),
Id = Guid.NewGuid().ToString("N"),
CreateTime = DateTime.UtcNow, // 统一使用UTC时间,避免时区问题
Status = 0, // 0=待下载(建议用枚举替代魔法值)
SavePath = video.VideoSavePath,
ViedoId = video.AwemeId,
CookieId = video.CookieId
CookieId = video.CookieId,
VideoRecordId = video.Id,
UpdateTime = DateTime.UtcNow
});
filePathsToDelete.Add((video.VideoSavePath, video.OnlyImgOrOnlyMp3));
}
// 无有效重新下载记录时直接返回
@@ -282,52 +519,40 @@ namespace dy.net.service
try
{
// 4. 数据库操作(事务保证一致性:创建重新下载记录 + 删除原视频记录必须同时成功/失败)
var transactionResult = await _dyCollectVideoRepository.UseTranAsync(async () =>
var videosById = videos.ToDictionary(x => x.Id);
var groups = reDownList.GroupBy(x => videosById[x.VideoRecordId].StorageType).ToList();
var transaction = await sqlSugarClient.Ado.UseTranAsync(async () =>
{
// 4.1 批量插入重新下载记录(SqlSugar批量插入效率更高)
_dyCollectVideoRepository.InsertReDowns(reDownList);
// 4.2 批量删除原视频记录(使用视频实际存在的ID,避免无效删除)
var actualDeleteIds = videos.Select(v => v.Id).ToList();
var deleteCount = await _dyCollectVideoRepository.DeleteByIdsAsync(actualDeleteIds); // 建议仓储层提供异步删除方法
}, e =>
{
Serilog.Log.Error(e, "数据库事务执行失败:Ids={0}", string.Join(",", videoIds));
foreach (var group in groups)
{
var jobs = group.ToList();
var storageLabel = group.Key switch
{
MediaStorageType.WebDav => "WebDAV",
MediaStorageType.OpenList => "OpenList",
_ => "本地"
};
var title = groups.Count == 1
? $"重新下载({jobs.Count} 条)"
: $"重新下载 · {storageLabel}{jobs.Count} 条)";
var task = await _videoTasks.CreateTaskAsync(VideoTaskType.Redownload,
VideoTaskTrigger.UserAction, title, storageType: group.Key);
foreach (var job in jobs)
{
var video = videosById[job.VideoRecordId];
var item = await _videoTasks.AddItemAsync(task.Id, video.AwemeId, video.CookieId, null,
video.ViedoType, video.VideoTitle, video.Author, video.VideoSavePath,
new[] { video.VideoUrl }, video, video.FileSize);
job.TaskId = task.Id;
job.TaskItemId = item.Id;
}
if (await sqlSugarClient.Insertable(jobs).ExecuteCommandAsync() != jobs.Count)
throw new InvalidOperationException($"{storageLabel}重新下载队列写入不完整。");
}
});
// 5. 文件删除(非事务操作,失败不回滚数据库,可根据业务调整)
// 采用异步文件操作,避免同步IO阻塞线程(需.NET 5+支持)
foreach (var video in filePathsToDelete)
{
try
{
if (File.Exists(video.path))
{
File.Delete(video.path); // 异步删除,提升并发性能
Serilog.Log.Debug("视频文件删除成功:Path={0}", video.path);
if (!video.onlyImgOrMp3)//如果是纯图片或纯音频文件,则不删除所在文件夹
{
//检查这个路径所在文件夹是否还有其他视频文件,如果没有则删除这个文件夹
var dir = Path.GetDirectoryName(video.path);
bool hasMp4File = Directory.EnumerateFiles(dir, "*.mp4", SearchOption.TopDirectoryOnly).Any(); // 只要存在一个MP4文件就返回true;
if (!hasMp4File)
{
Directory.Delete(dir, true);
}
}
}
else
{
Serilog.Log.Error("视频文件不存在,跳过删除:Path={0}", video);
}
}
catch (IOException ex)
{
Serilog.Log.Error(ex, "视频文件删除失败:Path={0}", video);
}
}
if (!transaction.IsSuccess)
throw new InvalidOperationException("创建重新下载任务失败:" + transaction.ErrorMessage,
transaction.ErrorException);
//var CookieIds = reDownList.Select(x => x.CookieId).Distinct();
//foreach (var ck in CookieIds)
@@ -362,8 +587,7 @@ namespace dy.net.service
// await douyinCookieRepository.UpdateAsync(cookie);
//}
if (!forever)
Serilog.Log.Debug("重新下载视频流程执行完成:成功创建{0}条重新下载记录,删除{1}个文件,等待重新下载...", reDownList.Count, filePathsToDelete.Count);
Serilog.Log.Debug("安全重新下载任务已创建:{0} 条;旧记录和旧文件会保留到新主媒体完整写入。", reDownList.Count);
return true;
}
catch (Exception ex)
@@ -415,7 +639,8 @@ namespace dy.net.service
List<string> douyinVideoIds = new List<string>();
foreach (var v in videos)
{
if (!File.Exists(v.VideoSavePath))
if (v.OnlyImgOrOnlyMp3 && string.IsNullOrWhiteSpace(v.VideoSavePath)) continue;
if (!await _storageRouter.Resolve(v.StorageType).ExistsAsync(v.VideoSavePath))
{
douyinVideoIds.Add(v.Id);
vList.Add(new DeleteInvalidVideoDto { AwId = v.AwemeId, Title = v.VideoTitle, Path = v.VideoSavePath });
@@ -443,16 +668,26 @@ namespace dy.net.service
internal async Task<int> AddDeleteVideo(List<DouyinVideo> videos)
{
var deletes = videos.Select(video => new DouyinVideoDelete
var awemeIds = videos.Select(x => x.AwemeId).Where(x => !string.IsNullOrWhiteSpace(x)).Distinct().ToList();
var existing = await sqlSugarClient.Queryable<DouyinVideoDelete>()
.Where(x => awemeIds.Contains(x.ViedoId)).Select(x => x.ViedoId).ToListAsync();
var deletes = videos.Where(video => !existing.Contains(video.AwemeId)).Select(video => new DouyinVideoDelete
{
ViedoId = video.AwemeId,
VideoTitle = video.VideoTitle,
VideoSavePath = video.VideoSavePath,
Id = IdGener.GetLong().ToString(),
DeleteTime = DateTime.Now
DeleteTime = DateTime.Now,
CookieId = video.CookieId,
VideoType = video.ViedoType,
AuthorId = video.AuthorId,
Author = video.Author,
VideoUrl = video.VideoUrl,
RestoreSnapshotJson = JsonConvert.SerializeObject(video)
})?.ToList();
return await sqlSugarClient.Insertable<DouyinVideoDelete>(deletes).ExecuteCommandAsync();
if (deletes.Count > 0) await sqlSugarClient.Insertable<DouyinVideoDelete>(deletes).ExecuteCommandAsync();
return existing.Distinct().Count() + deletes.Count;
}
/// <summary>
@@ -470,11 +705,15 @@ namespace dy.net.service
{
if (videos.Count <= 30)
{
var deletes = await AddDeleteVideo(videos);
if (deletes < videos.Select(x => x.AwemeId).Distinct().Count())
{
Serilog.Log.Error("写入永久排除记录失败,已停止删除媒体");
return false;
}
var result = await ReDownloadViedoAsync(new ReDownViedoDto { Ids = videos.Select(x => x.Id)?.ToList() }, true);
if (result)
{
//加入删除逻辑
var deletes = await AddDeleteVideo(videos);
Serilog.Log.Debug($"批量永久删除博主{videos.FirstOrDefault()?.Author},共{deletes}条记录");
return true;
}
@@ -488,11 +727,15 @@ namespace dy.net.service
{
Task.Run(async () =>
{
var deletes = await AddDeleteVideo(videos);
if (deletes < videos.Select(x => x.AwemeId).Distinct().Count())
{
Serilog.Log.Error("写入永久排除记录失败,已停止后台删除媒体");
return;
}
var result = await ReDownloadViedoAsync(new ReDownViedoDto { Ids = videos.Select(x => x.Id)?.ToList() }, true);
if (result)
{
//加入删除逻辑
var deletes = await AddDeleteVideo(videos);
Serilog.Log.Debug($"批量永久删除博主{videos.FirstOrDefault()?.Author}{deletes}条记录");
}
else
+47
View File
@@ -0,0 +1,47 @@
using dy.net.model.entity;
using MailKit.Net.Smtp;
using MimeKit;
namespace dy.net.service
{
public interface IEmailNotificationSender
{
Task SendAsync(
EmailNotificationSettings settings,
string password,
string subject,
string htmlBody,
CancellationToken cancellationToken = default);
}
public sealed class EmailNotificationSender : IEmailNotificationSender
{
public async Task SendAsync(
EmailNotificationSettings settings,
string password,
string subject,
string htmlBody,
CancellationToken cancellationToken = default)
{
var message = new MimeMessage();
message.From.Add(new MailboxAddress(
string.IsNullOrWhiteSpace(settings.FromName) ? "dysync.net" : settings.FromName,
settings.FromAddress));
message.To.AddRange(EmailNotificationSettingsService.ParseRecipients(settings.Recipients));
message.Subject = subject;
message.Body = new BodyBuilder { HtmlBody = htmlBody }.ToMessageBody();
using var client = new SmtpClient();
client.Timeout = 30_000;
await client.ConnectAsync(
settings.Host,
settings.Port,
EmailNotificationSettingsService.ToSocketOptions(settings.SecurityMode),
cancellationToken);
if (!string.IsNullOrWhiteSpace(settings.UserName))
await client.AuthenticateAsync(settings.UserName, password ?? string.Empty, cancellationToken);
await client.SendAsync(message, cancellationToken);
await client.DisconnectAsync(true, cancellationToken);
}
}
}
+167
View File
@@ -0,0 +1,167 @@
using dy.net.model.dto;
using dy.net.model.entity;
using MailKit.Security;
using Microsoft.AspNetCore.DataProtection;
using MimeKit;
using SqlSugar;
namespace dy.net.service
{
public sealed class EmailNotificationSettingsService
{
private const string SettingsId = "default";
private readonly ISqlSugarClient _db;
private readonly IDataProtector _protector;
private readonly IEmailNotificationSender _sender;
public EmailNotificationSettingsService(
ISqlSugarClient db,
IDataProtectionProvider provider,
IEmailNotificationSender sender)
{
_db = db;
_protector = provider.CreateProtector("dysync.email.password.v1");
_sender = sender;
}
public async Task<EmailNotificationSettings> GetAsync() =>
await _db.Queryable<EmailNotificationSettings>().InSingleAsync(SettingsId)
?? new EmailNotificationSettings();
public async Task<string> GetPasswordAsync(EmailNotificationSettings settings = null)
{
settings ??= await GetAsync();
if (string.IsNullOrWhiteSpace(settings.ProtectedPassword)) return string.Empty;
try { return _protector.Unprotect(settings.ProtectedPassword); }
catch (Exception ex)
{
Serilog.Log.Error(ex, "SMTP 密码解密失败,请重新输入密码");
return string.Empty;
}
}
public async Task<EmailNotificationSettings> BuildCandidateAsync(EmailNotificationSettingsDto dto)
{
dto ??= new EmailNotificationSettingsDto();
var existing = await GetAsync();
var password = string.IsNullOrWhiteSpace(dto.Password)
? await GetPasswordAsync(existing)
: dto.Password;
var candidate = new EmailNotificationSettings
{
Id = SettingsId,
Enabled = dto.Enabled,
Host = dto.Host?.Trim(),
Port = dto.Port <= 0 ? 465 : dto.Port,
SecurityMode = dto.SecurityMode,
UserName = dto.UserName?.Trim(),
ProtectedPassword = string.IsNullOrWhiteSpace(password) ? null : _protector.Protect(password),
FromAddress = dto.FromAddress?.Trim(),
FromName = Limit(dto.FromName, 200),
Recipients = NormalizeRecipients(dto.Recipients),
LastTestedAt = existing.LastTestedAt,
LastTestMessage = existing.LastTestMessage
};
Validate(candidate, password, requireComplete: candidate.Enabled);
return candidate;
}
public async Task<EmailNotificationSettingsDto> SaveAsync(EmailNotificationSettingsDto dto)
{
var candidate = await BuildCandidateAsync(dto);
await _db.Storageable(candidate).ExecuteCommandAsync();
return ToDto(candidate);
}
public async Task<string> TestAsync(EmailNotificationSettingsDto dto, CancellationToken cancellationToken)
{
var candidate = await BuildCandidateAsync(dto);
var password = await GetPasswordAsync(candidate);
Validate(candidate, password, requireComplete: true);
var now = DateTime.Now;
await _sender.SendAsync(
candidate,
password,
"dysync.net 邮箱通知测试成功",
$"<h2>邮箱通知配置可用</h2><p>测试时间:{now:yyyy-MM-dd HH:mm:ss}</p><p>开播通知将只在博主首次进入直播状态时发送。</p>",
cancellationToken);
candidate.LastTestedAt = DateTime.Now;
candidate.LastTestMessage = "测试邮件发送成功";
await _db.Storageable(candidate).ExecuteCommandAsync();
return candidate.LastTestMessage;
}
public async Task<string> GetReadinessErrorAsync()
{
var settings = await GetAsync();
if (!settings.Enabled) return "请先在系统设置中启用邮箱通知";
var password = await GetPasswordAsync(settings);
try
{
Validate(settings, password, requireComplete: true);
return null;
}
catch (InvalidOperationException ex)
{
return "邮箱通知配置不完整:" + ex.Message;
}
}
public EmailNotificationSettingsDto ToDto(EmailNotificationSettings settings) => new()
{
Enabled = settings.Enabled,
Host = settings.Host,
Port = settings.Port,
SecurityMode = settings.SecurityMode,
UserName = settings.UserName,
HasPassword = !string.IsNullOrWhiteSpace(settings.ProtectedPassword),
FromAddress = settings.FromAddress,
FromName = settings.FromName,
Recipients = settings.Recipients,
LastTestedAt = settings.LastTestedAt,
LastTestMessage = settings.LastTestMessage
};
public static SecureSocketOptions ToSocketOptions(EmailSecurityMode mode) => mode switch
{
EmailSecurityMode.None => SecureSocketOptions.None,
EmailSecurityMode.StartTls => SecureSocketOptions.StartTls,
_ => SecureSocketOptions.SslOnConnect
};
private static void Validate(EmailNotificationSettings settings, string password, bool requireComplete)
{
if (settings.Port is < 1 or > 65535) throw new InvalidOperationException("SMTP 端口必须在 165535 之间");
if (!requireComplete) return;
if (string.IsNullOrWhiteSpace(settings.Host)) throw new InvalidOperationException("请填写 SMTP 服务器");
if (string.IsNullOrWhiteSpace(settings.FromAddress) || !MailboxAddress.TryParse(settings.FromAddress, out _))
throw new InvalidOperationException("发件邮箱格式不正确");
if (ParseRecipients(settings.Recipients).Count == 0) throw new InvalidOperationException("请至少填写一个收件邮箱");
if (!string.IsNullOrWhiteSpace(settings.UserName) && string.IsNullOrWhiteSpace(password))
throw new InvalidOperationException("请填写 SMTP 密码或授权码");
}
public static List<MailboxAddress> ParseRecipients(string recipients)
{
var result = new List<MailboxAddress>();
foreach (var value in (recipients ?? string.Empty)
.Split(new[] { ',', ';', '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)
.Select(x => x.Trim()).Distinct(StringComparer.OrdinalIgnoreCase).Take(20))
{
if (!MailboxAddress.TryParse(value, out var address))
throw new InvalidOperationException($"收件邮箱格式不正确:{value}");
result.Add(address);
}
return result;
}
private static string NormalizeRecipients(string recipients) =>
string.Join(";", ParseRecipients(recipients).Select(x => x.Address));
private static string Limit(string value, int maxLength)
{
value = value?.Trim();
return value != null && value.Length > maxLength ? value[..maxLength] : value;
}
}
}
+57
View File
@@ -0,0 +1,57 @@
using System.Net;
using dy.net.model.entity;
namespace dy.net.service
{
public sealed class LiveEmailNotificationService
{
private readonly EmailNotificationSettingsService _settingsService;
private readonly IEmailNotificationSender _sender;
public LiveEmailNotificationService(
EmailNotificationSettingsService settingsService,
IEmailNotificationSender sender)
{
_settingsService = settingsService;
_sender = sender;
}
public Task<string> GetReadinessErrorAsync() => _settingsService.GetReadinessErrorAsync();
public async Task<(bool Sent, string Error)> SendLiveStartedAsync(
DouyinFollowed follow,
CancellationToken cancellationToken = default)
{
try
{
var settings = await _settingsService.GetAsync();
if (!settings.Enabled || !follow.LiveEmailNotificationEnabled) return (false, null);
var password = await _settingsService.GetPasswordAsync(settings);
var roomUrl = !string.IsNullOrWhiteSpace(follow.LiveWebRid)
? "https://live.douyin.com/" + Uri.EscapeDataString(follow.LiveWebRid)
: "https://www.douyin.com/user/" + Uri.EscapeDataString(follow.SecUid ?? string.Empty);
var blogger = WebUtility.HtmlEncode(follow.UperName ?? "关注博主");
var title = WebUtility.HtmlEncode(follow.LiveTitle ?? "正在直播");
var safeUrl = WebUtility.HtmlEncode(roomUrl);
var subject = $"[dysync.net] {follow.UperName ?? ""} 开播了";
var body = $"<h2>{blogger} 正在直播</h2><p>{title}</p>" +
$"<p>检测时间:{DateTime.Now:yyyy-MM-dd HH:mm:ss}</p>" +
$"<p><a href=\"{safeUrl}\">进入直播间</a></p>";
await _sender.SendAsync(settings, password, subject, body, cancellationToken);
return (true, null);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
Serilog.Log.Warning(ex, "发送开播邮件失败:Blogger={Blogger}", follow.UperName);
return (false, SafeError(ex));
}
}
private static string SafeError(Exception ex)
{
var message = ex.GetBaseException().Message?.Trim();
if (string.IsNullOrWhiteSpace(message)) return "开播邮件发送失败,请检查邮箱设置";
return message.Length <= 1000 ? message : message[..1000];
}
}
}
+351
View File
@@ -0,0 +1,351 @@
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using dy.net.model.dto;
using dy.net.model.entity;
using dy.net.storage;
using SqlSugar;
using MediaStorageType = dy.net.model.dto.StorageType;
namespace dy.net.service
{
public sealed class OpenListDirectoryRepairService
{
private readonly ISqlSugarClient _db;
private readonly DouyinCommonService _common;
private readonly OpenListSettingsService _settingsService;
private readonly OpenListClient _client;
public OpenListDirectoryRepairService(
ISqlSugarClient db,
DouyinCommonService common,
OpenListSettingsService settingsService,
OpenListClient client)
{
_db = db;
_common = common;
_settingsService = settingsService;
_client = client;
}
public async Task<OpenListDirectoryRepairPreflightResult> PreflightAsync(
OpenListDirectoryRepairPreflightRequest request,
CancellationToken cancellationToken)
{
var plan = await BuildPlanAsync(request?.LogicalPath, cancellationToken);
return plan.Result;
}
public async Task<OpenListDirectoryRepairTaskDetail> CreateAsync(
CreateOpenListDirectoryRepairRequest request,
CancellationToken cancellationToken)
{
var plan = await BuildPlanAsync(request?.LogicalPath, cancellationToken);
if (!plan.Result.CanStart) throw new InvalidOperationException(string.Join("", plan.Result.Errors));
if (!string.Equals(request?.ConfigurationFingerprint, plan.Result.ConfigurationFingerprint, StringComparison.Ordinal))
throw new InvalidOperationException("OpenList 配置或目录内容已变化,请重新预检。");
if (await _db.Queryable<OpenListDirectoryRepairTask>().Where(x =>
x.Status == OpenListDirectoryRepairStatus.Queued
|| x.Status == OpenListDirectoryRepairStatus.Scanning
|| x.Status == OpenListDirectoryRepairStatus.AwaitingConfirmation
|| x.Status == OpenListDirectoryRepairStatus.Cleaning
|| x.Status == OpenListDirectoryRepairStatus.Paused).AnyAsync())
throw new InvalidOperationException("已有未结束的 OpenList 目录修复任务。");
var now = DateTime.Now;
var task = new OpenListDirectoryRepairTask
{
Id = Guid.NewGuid().ToString("N"),
Status = OpenListDirectoryRepairStatus.Queued,
ConfigurationFingerprint = plan.Result.ConfigurationFingerprint,
RequestedPath = plan.Result.RequestedPath,
CanonicalPath = plan.Result.CanonicalPath,
ActualParentPath = plan.ActualParentPath,
RequestedName = plan.RequestedName,
CandidatePattern = plan.Result.CandidatePattern,
TotalCount = plan.Candidates.Count,
PendingCount = plan.Candidates.Count,
CreatedAt = now,
UpdatedAt = now
};
var items = plan.Candidates.Select((candidate, index) => new OpenListDirectoryRepairItem
{
Id = Guid.NewGuid().ToString("N"),
TaskId = task.Id,
DirectoryName = candidate.Name,
ActualPath = candidate.Path,
Status = OpenListDirectoryRepairItemStatus.Pending,
CreatedAt = now.AddTicks(index),
UpdatedAt = now
}).ToList();
var transaction = await _db.Ado.UseTranAsync(async () =>
{
await _db.Insertable(task).ExecuteCommandAsync();
if (items.Count > 0) await _db.Insertable(items).ExecuteCommandAsync();
});
if (!transaction.IsSuccess)
throw transaction.ErrorException ?? new InvalidOperationException("创建目录修复任务失败。");
return await GetAsync(task.Id);
}
public async Task<OpenListDirectoryRepairTaskDetail> GetAsync(string id)
{
var task = await RequireTaskAsync(id);
var token = task.Status == OpenListDirectoryRepairStatus.AwaitingConfirmation
? await BuildConfirmationTokenAsync(task) : null;
return new OpenListDirectoryRepairTaskDetail
{
Task = task,
StatusText = StatusText(task.Status),
ConfirmationToken = token,
CanConfirmCleanup = task.Status == OpenListDirectoryRepairStatus.AwaitingConfirmation && task.EmptyCount > 0,
CanCancel = task.Status is OpenListDirectoryRepairStatus.Queued or OpenListDirectoryRepairStatus.Scanning
or OpenListDirectoryRepairStatus.AwaitingConfirmation or OpenListDirectoryRepairStatus.Cleaning,
CanResume = task.Status is OpenListDirectoryRepairStatus.Cancelled or OpenListDirectoryRepairStatus.Paused,
CanRetryFailed = task.FailedCount > 0 && task.Status is (OpenListDirectoryRepairStatus.PartiallyFailed
or OpenListDirectoryRepairStatus.Cancelled or OpenListDirectoryRepairStatus.Paused)
};
}
public async Task<OpenListDirectoryRepairTaskDetail> GetLatestAsync()
{
var task = await _db.Queryable<OpenListDirectoryRepairTask>()
.OrderByDescending(x => x.CreatedAt).FirstAsync();
return task == null ? null : await GetAsync(task.Id);
}
public async Task<OpenListDirectoryRepairItemPage> GetItemsAsync(
string id,
OpenListDirectoryRepairItemPageRequest request)
{
_ = await RequireTaskAsync(id);
request ??= new OpenListDirectoryRepairItemPageRequest();
var page = Math.Max(1, request.PageIndex);
var size = Math.Clamp(request.PageSize, 1, 100);
var query = _db.Queryable<OpenListDirectoryRepairItem>().Where(x => x.TaskId == id)
.WhereIF(request.Status.HasValue, x => x.Status == request.Status.Value)
.WhereIF(!string.IsNullOrWhiteSpace(request.Keyword), x =>
x.DirectoryName.Contains(request.Keyword) || x.ErrorMessage.Contains(request.Keyword));
return new OpenListDirectoryRepairItemPage
{
TotalCount = await query.CountAsync(),
Items = await query.OrderBy(x => x.CreatedAt).Skip((page - 1) * size).Take(size).ToListAsync()
};
}
public async Task ConfirmCleanupAsync(
string id,
ConfirmOpenListDirectoryRepairRequest request)
{
var task = await RequireTaskAsync(id);
if (task.Status != OpenListDirectoryRepairStatus.AwaitingConfirmation)
throw new InvalidOperationException("只有扫描完成并等待确认的任务可以开始清理。");
await EnsureFingerprintAsync(task);
var token = await BuildConfirmationTokenAsync(task);
if (string.IsNullOrWhiteSpace(request?.ConfirmationToken)
|| !CryptographicOperations.FixedTimeEquals(
Encoding.UTF8.GetBytes(token), Encoding.UTF8.GetBytes(request.ConfirmationToken)))
throw new InvalidOperationException("目录状态已变化,请刷新任务详情后重新确认。");
if (task.EmptyCount <= 0) throw new InvalidOperationException("没有已确认为空的目录可以清理。");
task.Status = OpenListDirectoryRepairStatus.Cleaning;
task.ErrorMessage = null;
task.CompletedAt = null;
task.UpdatedAt = DateTime.Now;
await _db.Updateable(task).ExecuteCommandAsync();
}
public async Task CancelAsync(string id)
{
var task = await RequireTaskAsync(id);
if (task.Status is not (OpenListDirectoryRepairStatus.Queued or OpenListDirectoryRepairStatus.Scanning
or OpenListDirectoryRepairStatus.AwaitingConfirmation or OpenListDirectoryRepairStatus.Cleaning))
throw new InvalidOperationException("该任务当前不能取消。");
task.Status = OpenListDirectoryRepairStatus.Cancelled;
task.CurrentDirectory = null;
task.CompletedAt = DateTime.Now;
task.UpdatedAt = DateTime.Now;
await _db.Updateable(task).ExecuteCommandAsync();
}
public async Task ResumeAsync(string id)
{
var task = await RequireTaskAsync(id);
if (task.Status is not (OpenListDirectoryRepairStatus.Cancelled or OpenListDirectoryRepairStatus.Paused))
throw new InvalidOperationException("只有已取消或因配置变化暂停的任务可以恢复。");
await EnsureFingerprintAsync(task);
await _db.Updateable<OpenListDirectoryRepairItem>()
.SetColumns(x => new OpenListDirectoryRepairItem
{
Status = OpenListDirectoryRepairItemStatus.Pending,
ErrorMessage = null,
CompletedAt = null,
UpdatedAt = DateTime.Now
})
.Where(x => x.TaskId == id && (x.Status == OpenListDirectoryRepairItemStatus.Inspecting
|| x.Status == OpenListDirectoryRepairItemStatus.Deleting)).ExecuteCommandAsync();
await RefreshCountsAsync(id);
task = await RequireTaskAsync(id);
task.Status = task.EmptyCount > 0 && task.PendingCount == 0
? OpenListDirectoryRepairStatus.AwaitingConfirmation
: OpenListDirectoryRepairStatus.Queued;
task.CurrentDirectory = null;
task.CompletedAt = null;
task.ErrorMessage = null;
task.UpdatedAt = DateTime.Now;
await _db.Updateable(task).ExecuteCommandAsync();
}
public async Task RetryFailedAsync(string id)
{
var task = await RequireTaskAsync(id);
if (task.Status is not (OpenListDirectoryRepairStatus.PartiallyFailed
or OpenListDirectoryRepairStatus.Cancelled or OpenListDirectoryRepairStatus.Paused))
throw new InvalidOperationException("该任务当前不能重试失败项。");
await EnsureFingerprintAsync(task);
var changed = await _db.Updateable<OpenListDirectoryRepairItem>()
.SetColumns(x => new OpenListDirectoryRepairItem
{
Status = OpenListDirectoryRepairItemStatus.Pending,
ErrorMessage = null,
CompletedAt = null,
UpdatedAt = DateTime.Now
})
.Where(x => x.TaskId == id && x.Status == OpenListDirectoryRepairItemStatus.Failed)
.ExecuteCommandAsync();
if (changed == 0) throw new InvalidOperationException("没有可重试的失败目录。");
task.Status = OpenListDirectoryRepairStatus.Queued;
task.CompletedAt = null;
task.ErrorMessage = null;
task.UpdatedAt = DateTime.Now;
await _db.Updateable(task).ExecuteCommandAsync();
await RefreshCountsAsync(id);
}
public async Task RefreshCountsAsync(string id)
{
var items = await _db.Queryable<OpenListDirectoryRepairItem>().Where(x => x.TaskId == id).ToListAsync();
var pending = items.Count(x => x.Status is OpenListDirectoryRepairItemStatus.Pending
or OpenListDirectoryRepairItemStatus.Inspecting or OpenListDirectoryRepairItemStatus.Deleting);
await _db.Updateable<OpenListDirectoryRepairTask>()
.SetColumns(x => new OpenListDirectoryRepairTask
{
TotalCount = items.Count,
PendingCount = pending,
EmptyCount = items.Count(x => x.Status == OpenListDirectoryRepairItemStatus.EmptyConfirmed),
DeletedCount = items.Count(x => x.Status == OpenListDirectoryRepairItemStatus.Deleted),
SkippedCount = items.Count(x => x.Status == OpenListDirectoryRepairItemStatus.SkippedNonEmpty),
FailedCount = items.Count(x => x.Status == OpenListDirectoryRepairItemStatus.Failed),
MissingCount = items.Count(x => x.Status == OpenListDirectoryRepairItemStatus.Missing),
UpdatedAt = DateTime.Now
}).Where(x => x.Id == id).ExecuteCommandAsync();
}
public async Task EnsureFingerprintAsync(OpenListDirectoryRepairTask task)
{
var settings = await _settingsService.GetAsync();
var current = StorageConfigurationFingerprint.Create(settings);
if (!string.Equals(current, task.ConfigurationFingerprint, StringComparison.Ordinal))
throw new InvalidOperationException("OpenList 配置已变化,请恢复原配置或重新创建目录修复任务。");
}
public async Task<(OpenListSettings Settings, string Password)> GetConnectionAsync()
{
var settings = await _settingsService.GetAsync();
var password = await _settingsService.GetPasswordAsync(settings);
if (string.IsNullOrWhiteSpace(password)) throw new InvalidOperationException("OpenList 密码无效,请重新保存配置。");
return (settings, password);
}
private async Task<RepairPlan> BuildPlanAsync(string logicalPath, CancellationToken cancellationToken)
{
var result = new OpenListDirectoryRepairPreflightResult();
if ((_common.GetConfig()?.StorageType ?? MediaStorageType.Local) != MediaStorageType.OpenList)
{
result.Errors.Add("当前存储不是 OpenList,不能执行目录修复。");
return new RepairPlan(result);
}
logicalPath = StoragePath.NormalizeRemote(string.IsNullOrWhiteSpace(logicalPath) ? "/collect/Kk" : logicalPath);
if (logicalPath == "/" || logicalPath.Count(x => x == '/') < 2)
{
result.Errors.Add("目录修复路径必须位于 OpenList 基础目录的子目录中。");
return new RepairPlan(result);
}
var (settings, password) = await GetConnectionAsync();
result.ConfigurationFingerprint = StorageConfigurationFingerprint.Create(settings);
result.RequestedPath = logicalPath;
var requestedActual = OpenListTransferService.ToActualPath(settings, logicalPath);
var canonicalActual = await _client.ResolveCanonicalObjectPathAsync(settings, password,
requestedActual, false, cancellationToken);
var info = await _client.TryGetObjectAsync(settings, password, canonicalActual, cancellationToken);
if (info is not { IsDirectory: true })
{
result.Errors.Add($"没有找到可复用的规范目录:{logicalPath}。");
return new RepairPlan(result);
}
var requestedName = Path.GetFileName(logicalPath);
var canonicalName = info.Name;
if (string.Equals(requestedName, canonicalName, StringComparison.Ordinal)
|| !string.Equals(requestedName, canonicalName, StringComparison.OrdinalIgnoreCase))
{
result.Errors.Add($"路径不存在纯大小写冲突:请求 {requestedName},实际 {canonicalName}。");
return new RepairPlan(result);
}
var actualParent = StoragePath.DirectoryName(canonicalActual);
var escaped = Regex.Escape(requestedName);
var pattern = $"^{escaped}_\\d{{8}}_\\d{{6}}$";
var regex = new Regex(pattern, RegexOptions.CultureInvariant | RegexOptions.Compiled);
var listing = await _client.ListDirectoriesAsync(settings, password, actualParent, true, cancellationToken);
var candidates = listing.Directories.Where(x => regex.IsMatch(x.Name))
.OrderBy(x => x.Name, StringComparer.Ordinal).ToList();
result.CanonicalPath = OpenListTransferService.ToLogicalPath(settings, canonicalActual);
result.CandidatePattern = pattern;
result.CandidateCount = candidates.Count;
result.CanStart = candidates.Count > 0;
if (candidates.Count == 0) result.Errors.Add("没有找到匹配的时间后缀异常目录。");
else result.Warnings.Add($"找到 {candidates.Count} 个候选目录;后台任务会逐个确认为空,非空目录绝不会删除。");
return new RepairPlan(result)
{
ActualParentPath = actualParent,
RequestedName = requestedName,
Candidates = candidates
};
}
private async Task<string> BuildConfirmationTokenAsync(OpenListDirectoryRepairTask task)
{
var items = await _db.Queryable<OpenListDirectoryRepairItem>().Where(x => x.TaskId == task.Id
&& x.Status == OpenListDirectoryRepairItemStatus.EmptyConfirmed)
.OrderBy(x => x.Id).ToListAsync();
var source = task.Id + "\n" + task.ConfigurationFingerprint + "\n"
+ string.Join("\n", items.Select(x => $"{x.Id}|{x.ActualPath}|{x.UpdatedAt.Ticks}|{x.EntryCount}"));
return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(source))).ToLowerInvariant();
}
private async Task<OpenListDirectoryRepairTask> RequireTaskAsync(string id) =>
await _db.Queryable<OpenListDirectoryRepairTask>().InSingleAsync(id)
?? throw new KeyNotFoundException("目录修复任务不存在。");
private static string StatusText(OpenListDirectoryRepairStatus status) => status switch
{
OpenListDirectoryRepairStatus.Queued => "等待扫描",
OpenListDirectoryRepairStatus.Scanning => "扫描中",
OpenListDirectoryRepairStatus.AwaitingConfirmation => "等待确认清理",
OpenListDirectoryRepairStatus.Cleaning => "清理中",
OpenListDirectoryRepairStatus.Completed => "已完成",
OpenListDirectoryRepairStatus.PartiallyFailed => "部分失败",
OpenListDirectoryRepairStatus.Cancelled => "已取消",
OpenListDirectoryRepairStatus.Paused => "配置变化,已暂停",
_ => status.ToString()
};
private sealed class RepairPlan
{
public RepairPlan(OpenListDirectoryRepairPreflightResult result) => Result = result;
public OpenListDirectoryRepairPreflightResult Result { get; }
public string ActualParentPath { get; set; }
public string RequestedName { get; set; }
public List<OpenListDirectoryItemDto> Candidates { get; set; } = new();
}
}
}
+369
View File
@@ -0,0 +1,369 @@
using dy.net.model.dto;
using dy.net.model.entity;
using dy.net.storage;
using SqlSugar;
using MediaStorageType = dy.net.model.dto.StorageType;
namespace dy.net.service
{
public sealed class OpenListDirectoryRepairWorker : BackgroundService
{
private const int InspectionBatchSize = 4;
private readonly IServiceScopeFactory _scopeFactory;
private readonly ILogger<OpenListDirectoryRepairWorker> _logger;
public OpenListDirectoryRepairWorker(
IServiceScopeFactory scopeFactory,
ILogger<OpenListDirectoryRepairWorker> logger)
{
_scopeFactory = scopeFactory;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
await RecoverInterruptedAsync();
while (!stoppingToken.IsCancellationRequested)
{
try
{
if (!await RunOneAsync(stoppingToken))
await Task.Delay(TimeSpan.FromSeconds(2), stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { }
catch (Exception ex)
{
_logger.LogError(ex, "OpenList 目录修复后台服务异常");
await Task.Delay(TimeSpan.FromSeconds(3), stoppingToken);
}
}
}
private async Task RecoverInterruptedAsync()
{
using var scope = _scopeFactory.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<ISqlSugarClient>();
var now = DateTime.Now;
await db.Updateable<OpenListDirectoryRepairItem>()
.SetColumns(x => new OpenListDirectoryRepairItem
{
Status = OpenListDirectoryRepairItemStatus.Pending,
ErrorMessage = "应用重启,目录将在恢复后重新检查。",
UpdatedAt = now
})
.Where(x => x.Status == OpenListDirectoryRepairItemStatus.Inspecting).ExecuteCommandAsync();
await db.Updateable<OpenListDirectoryRepairItem>()
.SetColumns(x => new OpenListDirectoryRepairItem
{
Status = OpenListDirectoryRepairItemStatus.EmptyConfirmed,
ErrorMessage = "应用在删除确认后重启,需再次确认清理。",
UpdatedAt = now
})
.Where(x => x.Status == OpenListDirectoryRepairItemStatus.Deleting).ExecuteCommandAsync();
var interrupted = await db.Queryable<OpenListDirectoryRepairTask>().Where(x =>
x.Status == OpenListDirectoryRepairStatus.Scanning
|| x.Status == OpenListDirectoryRepairStatus.Cleaning).ToListAsync();
foreach (var task in interrupted)
{
task.Status = task.Status == OpenListDirectoryRepairStatus.Cleaning
? OpenListDirectoryRepairStatus.AwaitingConfirmation
: OpenListDirectoryRepairStatus.Queued;
task.CurrentDirectory = null;
task.ErrorMessage = task.Status == OpenListDirectoryRepairStatus.AwaitingConfirmation
? "应用重启后已停止删除,请重新确认。"
: "应用重启后已恢复扫描。";
task.UpdatedAt = now;
await db.Updateable(task).ExecuteCommandAsync();
await scope.ServiceProvider.GetRequiredService<OpenListDirectoryRepairService>()
.RefreshCountsAsync(task.Id);
}
}
private async Task<bool> RunOneAsync(CancellationToken cancellationToken)
{
using var scope = _scopeFactory.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<ISqlSugarClient>();
var service = scope.ServiceProvider.GetRequiredService<OpenListDirectoryRepairService>();
var task = await db.Queryable<OpenListDirectoryRepairTask>().Where(x =>
x.Status == OpenListDirectoryRepairStatus.Queued
|| x.Status == OpenListDirectoryRepairStatus.Scanning
|| x.Status == OpenListDirectoryRepairStatus.Cleaning)
.OrderBy(x => x.CreatedAt).FirstAsync();
if (task == null) return false;
try
{
await service.EnsureFingerprintAsync(task);
}
catch (InvalidOperationException ex)
{
task.Status = OpenListDirectoryRepairStatus.Paused;
task.ErrorMessage = ex.Message;
task.CurrentDirectory = null;
task.UpdatedAt = DateTime.Now;
await db.Updateable(task).ExecuteCommandAsync();
return true;
}
if (task.Status == OpenListDirectoryRepairStatus.Queued)
{
task.Status = OpenListDirectoryRepairStatus.Scanning;
task.StartedAt ??= DateTime.Now;
task.CompletedAt = null;
task.UpdatedAt = DateTime.Now;
await db.Updateable(task).ExecuteCommandAsync();
}
if (task.Status == OpenListDirectoryRepairStatus.Scanning)
return await InspectOneAsync(scope.ServiceProvider, task, cancellationToken);
return await DeleteOneAsync(scope.ServiceProvider, task, cancellationToken);
}
private static async Task<bool> InspectOneAsync(
IServiceProvider services,
OpenListDirectoryRepairTask task,
CancellationToken cancellationToken)
{
var db = services.GetRequiredService<ISqlSugarClient>();
var service = services.GetRequiredService<OpenListDirectoryRepairService>();
var client = services.GetRequiredService<OpenListClient>();
var items = await db.Queryable<OpenListDirectoryRepairItem>().Where(x =>
x.TaskId == task.Id && x.Status == OpenListDirectoryRepairItemStatus.Pending)
.OrderBy(x => x.CreatedAt).Take(InspectionBatchSize).ToListAsync();
if (items.Count == 0)
{
await CompleteScanAsync(services, task.Id);
return true;
}
var startedAt = DateTime.Now;
foreach (var item in items)
{
item.Status = OpenListDirectoryRepairItemStatus.Inspecting;
item.Attempts++;
item.ErrorMessage = null;
item.UpdatedAt = startedAt;
}
task.CurrentDirectory = items[0].ActualPath;
task.UpdatedAt = startedAt;
await db.Updateable(items).ExecuteCommandAsync();
await db.Updateable(task).ExecuteCommandAsync();
var (settings, password) = await service.GetConnectionAsync();
await Task.WhenAll(items.Select(async item =>
{
try
{
var inspection = await client.InspectKnownDirectoryAsync(
settings, password, item.ActualPath, true, cancellationToken);
item.EntryCount = inspection.EntryCount;
item.Status = !inspection.Exists
? OpenListDirectoryRepairItemStatus.Missing
: inspection.EntryCount == 0
? OpenListDirectoryRepairItemStatus.EmptyConfirmed
: OpenListDirectoryRepairItemStatus.SkippedNonEmpty;
item.ErrorMessage = inspection.Exists && inspection.EntryCount > 0
? $"目录包含 {inspection.EntryCount} 个对象,已安全跳过。" : null;
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
catch (Exception ex)
{
item.Status = OpenListDirectoryRepairItemStatus.Failed;
item.ErrorMessage = ex.GetBaseException().Message;
}
item.CompletedAt = DateTime.Now;
item.UpdatedAt = item.CompletedAt.Value;
}));
await db.Updateable(items).ExecuteCommandAsync();
await service.RefreshCountsAsync(task.Id);
return true;
}
private static async Task CompleteScanAsync(IServiceProvider services, string taskId)
{
var db = services.GetRequiredService<ISqlSugarClient>();
var service = services.GetRequiredService<OpenListDirectoryRepairService>();
await service.RefreshCountsAsync(taskId);
var task = await db.Queryable<OpenListDirectoryRepairTask>().InSingleAsync(taskId);
if (task?.Status != OpenListDirectoryRepairStatus.Scanning) return;
task.Status = task.EmptyCount > 0
? OpenListDirectoryRepairStatus.AwaitingConfirmation
: task.FailedCount > 0
? OpenListDirectoryRepairStatus.PartiallyFailed
: OpenListDirectoryRepairStatus.Completed;
task.CurrentDirectory = null;
task.ScanCompletedAt = DateTime.Now;
task.CompletedAt = task.Status is OpenListDirectoryRepairStatus.Completed
or OpenListDirectoryRepairStatus.PartiallyFailed ? DateTime.Now : null;
task.ErrorMessage = task.FailedCount > 0 ? $"{task.FailedCount} 个目录检查失败,未执行删除。" : null;
task.UpdatedAt = DateTime.Now;
await db.Updateable(task).ExecuteCommandAsync();
}
private static async Task<bool> DeleteOneAsync(
IServiceProvider services,
OpenListDirectoryRepairTask task,
CancellationToken cancellationToken)
{
var db = services.GetRequiredService<ISqlSugarClient>();
var service = services.GetRequiredService<OpenListDirectoryRepairService>();
var client = services.GetRequiredService<OpenListClient>();
var item = await db.Queryable<OpenListDirectoryRepairItem>().Where(x =>
x.TaskId == task.Id && x.Status == OpenListDirectoryRepairItemStatus.EmptyConfirmed)
.OrderBy(x => x.CreatedAt).FirstAsync();
if (item == null)
{
await CompleteCleanupAsync(services, task.Id, cancellationToken);
return true;
}
item.Status = OpenListDirectoryRepairItemStatus.Deleting;
item.Attempts++;
item.ErrorMessage = null;
item.UpdatedAt = DateTime.Now;
task.CurrentDirectory = item.ActualPath;
task.UpdatedAt = item.UpdatedAt;
await db.Updateable(item).ExecuteCommandAsync();
await db.Updateable(task).ExecuteCommandAsync();
try
{
var (settings, password) = await service.GetConnectionAsync();
var before = await client.InspectKnownDirectoryAsync(settings, password, item.ActualPath, true, cancellationToken);
if (!before.Exists)
{
item.Status = OpenListDirectoryRepairItemStatus.Missing;
}
else if (before.EntryCount > 0)
{
item.EntryCount = before.EntryCount;
item.Status = OpenListDirectoryRepairItemStatus.SkippedNonEmpty;
item.ErrorMessage = $"删除前发现目录已有 {before.EntryCount} 个对象,已安全跳过。";
}
else
{
await client.DeleteKnownObjectAsync(settings, password, item.ActualPath, cancellationToken);
var after = await client.InspectKnownDirectoryAsync(settings, password, item.ActualPath, true, cancellationToken);
if (after.Exists) throw new IOException("OpenList 返回删除成功,但目录仍然存在。");
item.Status = OpenListDirectoryRepairItemStatus.Deleted;
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
catch (Exception ex)
{
item.Status = OpenListDirectoryRepairItemStatus.Failed;
item.ErrorMessage = ex.GetBaseException().Message;
}
item.CompletedAt = DateTime.Now;
item.UpdatedAt = item.CompletedAt.Value;
await db.Updateable(item).ExecuteCommandAsync();
await service.RefreshCountsAsync(task.Id);
return true;
}
private static async Task CompleteCleanupAsync(
IServiceProvider services,
string taskId,
CancellationToken cancellationToken)
{
var db = services.GetRequiredService<ISqlSugarClient>();
var service = services.GetRequiredService<OpenListDirectoryRepairService>();
await service.RefreshCountsAsync(taskId);
var task = await db.Queryable<OpenListDirectoryRepairTask>().InSingleAsync(taskId);
if (task?.Status != OpenListDirectoryRepairStatus.Cleaning) return;
task.Status = task.FailedCount > 0
? OpenListDirectoryRepairStatus.PartiallyFailed
: OpenListDirectoryRepairStatus.Completed;
task.CurrentDirectory = null;
task.CompletedAt = DateTime.Now;
task.ErrorMessage = task.FailedCount > 0 ? $"{task.FailedCount} 个目录清理失败,可重试。" : null;
task.UpdatedAt = DateTime.Now;
await db.Updateable(task).ExecuteCommandAsync();
await RecoverAffectedTasksAsync(services, task, cancellationToken);
}
private static async Task RecoverAffectedTasksAsync(
IServiceProvider services,
OpenListDirectoryRepairTask repair,
CancellationToken cancellationToken)
{
var db = services.GetRequiredService<ISqlSugarClient>();
var videos = await db.Queryable<DouyinVideo>().Where(x => x.StorageType == MediaStorageType.OpenList
&& x.VideoSavePath.StartsWith(repair.RequestedPath + "/")).ToListAsync();
var storage = services.GetRequiredService<OpenListMediaStorage>();
foreach (var video in videos)
{
cancellationToken.ThrowIfCancellationRequested();
var canonicalMain = ReplacePrefix(video.VideoSavePath, repair.RequestedPath, repair.CanonicalPath);
var length = await storage.GetLengthAsync(canonicalMain, cancellationToken);
if (!length.HasValue || length <= 0 || video.FileSize > 0 && length.Value != video.FileSize) continue;
video.VideoSavePath = canonicalMain;
video.VideoCoverSavePath = ReplacePrefix(video.VideoCoverSavePath, repair.RequestedPath, repair.CanonicalPath);
video.AuthorAvatar = ReplacePrefix(video.AuthorAvatar, repair.RequestedPath, repair.CanonicalPath);
if (!string.IsNullOrWhiteSpace(video.DynamicVideos))
video.DynamicVideos = video.DynamicVideos.Replace(repair.RequestedPath + "/",
repair.CanonicalPath + "/", StringComparison.Ordinal);
await db.Updateable(video).ExecuteCommandAsync();
}
var failedItems = await db.Queryable<VideoDownloadTaskItem>().Where(x =>
(x.Stage == VideoTaskItemStage.Failed || x.Stage == VideoTaskItemStage.WaitingForStorage)
&& x.RetrySnapshotJson != null
&& (x.ErrorType == VideoTaskErrorType.StorageUnavailable
|| x.ErrorType == VideoTaskErrorType.IntegrityCheckFailed)).ToListAsync();
var taskIds = new HashSet<string>(StringComparer.Ordinal);
foreach (var item in failedItems)
{
if (!(item.TargetPath?.StartsWith(repair.RequestedPath + "/", StringComparison.Ordinal) == true
|| item.ErrorMessage?.Contains("Operation canceled", StringComparison.OrdinalIgnoreCase) == true
|| item.ErrorMessage?.Contains("超时", StringComparison.Ordinal) == true
|| item.ErrorType == VideoTaskErrorType.IntegrityCheckFailed)) continue;
item.TargetPath = ReplacePrefix(item.TargetPath, repair.RequestedPath, repair.CanonicalPath);
item.RetrySnapshotJson = item.RetrySnapshotJson.Replace(repair.RequestedPath + "/",
repair.CanonicalPath + "/", StringComparison.Ordinal);
item.Stage = VideoTaskItemStage.Pending;
item.ErrorType = VideoTaskErrorType.None;
item.ErrorMessage = null;
item.CompletedAt = null;
item.WorkerManaged = true;
item.UpdatedAt = DateTime.Now;
taskIds.Add(item.TaskId);
}
if (failedItems.Count > 0) await db.Updateable(failedItems).ExecuteCommandAsync();
foreach (var taskId in taskIds)
{
await db.Updateable<VideoDownloadTask>()
.SetColumns(x => new VideoDownloadTask
{
Status = VideoTaskStatus.Queued,
CompletedAt = null,
ErrorMessage = null,
UpdatedAt = DateTime.Now
}).Where(x => x.Id == taskId).ExecuteCommandAsync();
await services.GetRequiredService<VideoTaskService>().RefreshCountsAsync(taskId);
}
try { _ = await services.GetRequiredService<VideoTaskService>().ProbeStorageAsync(cancellationToken); }
catch (Exception ex) { Serilog.Log.Warning(ex, "目录修复后 OpenList 健康检测失败"); }
try
{
var migrations = await db.Queryable<StorageMigrationTask>().Where(x =>
x.TargetStorageType == MediaStorageType.OpenList
&& x.Status == StorageMigrationTaskStatus.PartiallyFailed).ToListAsync();
foreach (var migration in migrations)
await services.GetRequiredService<StorageMigrationService>().RetryFailedAsync(migration.Id);
}
catch (Exception ex) { Serilog.Log.Warning(ex, "目录修复后迁移失败项自动重试未能提交"); }
try
{
_ = await services.GetRequiredService<DouyinQuartzJobService>()
.TriggerVideoJobsNowAsync(VideoTypeEnum.dy_follows);
}
catch (Exception ex) { Serilog.Log.Warning(ex, "目录修复后关注同步未能自动提交"); }
}
private static string ReplacePrefix(string value, string oldPrefix, string newPrefix)
{
if (string.IsNullOrWhiteSpace(value)) return value;
return value.StartsWith(oldPrefix + "/", StringComparison.Ordinal)
? newPrefix + value[oldPrefix.Length..]
: value;
}
}
}
+168
View File
@@ -0,0 +1,168 @@
using dy.net.model.dto;
using dy.net.model.entity;
using dy.net.extension;
using Microsoft.AspNetCore.DataProtection;
using SqlSugar;
using dy.net.utils;
using MediaStorageType = dy.net.model.dto.StorageType;
namespace dy.net.service
{
public class OpenListSettingsService
{
private const string SettingsId = "default";
private readonly ISqlSugarClient _db;
private readonly IDataProtector _protector;
private readonly WebDavSettingsService _legacySettings;
public OpenListSettingsService(
ISqlSugarClient db,
IDataProtectionProvider provider,
WebDavSettingsService legacySettings)
{
_db = db;
_protector = provider.CreateProtector("dysync.openlist.password.v1");
_legacySettings = legacySettings;
}
public async Task<OpenListSettings> GetAsync()
{
var existing = await _db.Queryable<OpenListSettings>().InSingleAsync(SettingsId);
if (existing != null)
{
Normalize(existing);
return existing;
}
var legacy = await _legacySettings.GetAsync();
var password = await _legacySettings.GetPasswordAsync(legacy);
var suggested = new OpenListSettings
{
Id = SettingsId,
Endpoint = NormalizeLegacyEndpoint(legacy.Endpoint),
BasePath = string.IsNullOrWhiteSpace(legacy.BasePath) ? "/dysync" : legacy.BasePath,
LocalStagingPath = GetDefaultLocalStagingPath(),
SourcePath = "/dysync-staging",
UserName = legacy.UserName,
ProtectedPassword = string.IsNullOrWhiteSpace(password) ? null : _protector.Protect(password)
};
Normalize(suggested);
return suggested;
}
public async Task<string> GetPasswordAsync(OpenListSettings settings = null)
{
settings ??= await GetAsync();
if (string.IsNullOrWhiteSpace(settings.ProtectedPassword)) return string.Empty;
try
{
return _protector.Unprotect(settings.ProtectedPassword);
}
catch (Exception ex)
{
Serilog.Log.Error(ex, "OpenList 密码解密失败,请重新输入密码");
return string.Empty;
}
}
public async Task<OpenListSettings> BuildCandidateAsync(OpenListTestRequest request)
{
request ??= new OpenListTestRequest();
var existing = await GetAsync();
var password = string.IsNullOrWhiteSpace(request.Password)
? await GetPasswordAsync(existing)
: request.Password;
var candidate = new OpenListSettings
{
Id = SettingsId,
Endpoint = string.IsNullOrWhiteSpace(request.Endpoint) ? existing.Endpoint : request.Endpoint.Trim(),
BasePath = string.IsNullOrWhiteSpace(request.BasePath) ? existing.BasePath : request.BasePath.Trim(),
LocalStagingPath = string.IsNullOrWhiteSpace(request.LocalStagingPath)
? existing.LocalStagingPath : request.LocalStagingPath.Trim(),
SourcePath = string.IsNullOrWhiteSpace(request.SourcePath) ? existing.SourcePath : request.SourcePath.Trim(),
UserName = string.IsNullOrWhiteSpace(request.UserName) ? existing.UserName : request.UserName.Trim(),
ProtectedPassword = string.IsNullOrWhiteSpace(password) ? null : _protector.Protect(password)
};
Normalize(candidate);
return candidate;
}
public async Task SaveAsync(OpenListSettings settings, bool tested, string testMessage)
{
settings.Id = SettingsId;
Normalize(settings);
settings.LastTestedAt = tested ? DateTime.Now : null;
settings.LastTestMessage = testMessage;
await _db.Storageable(settings).ExecuteCommandAsync();
}
public async Task<OpenListSettingsDto> ToDtoAsync(MediaStorageType storageType)
{
var settings = await GetAsync();
return new OpenListSettingsDto
{
StorageType = storageType,
Endpoint = settings.Endpoint,
BasePath = settings.BasePath,
LocalStagingPath = settings.LocalStagingPath,
SourcePath = settings.SourcePath,
UserName = settings.UserName,
HasPassword = !string.IsNullOrWhiteSpace(settings.ProtectedPassword),
LastTestedAt = settings.LastTestedAt,
LastTestMessage = settings.LastTestMessage
};
}
public async Task<bool> HasSavedSettingsAsync() =>
await _db.Queryable<OpenListSettings>().Where(x => x.Id == SettingsId).AnyAsync();
public static string GetDefaultLocalStagingPath()
{
var dataRoot = !string.IsNullOrWhiteSpace(ServiceExtension.FnDataFolder)
? Path.GetDirectoryName(Path.GetFullPath(ServiceExtension.FnDataFolder))
: null;
return Path.Combine(dataRoot ?? Environment.CurrentDirectory, "openlist-staging");
}
private static string NormalizeLegacyEndpoint(string endpoint)
{
if (string.IsNullOrWhiteSpace(endpoint)) return endpoint;
var value = endpoint.Trim().TrimEnd('/');
var dav = value.LastIndexOf("/dav", StringComparison.OrdinalIgnoreCase);
return dav >= 0 && dav + 4 == value.Length ? value[..dav] : value;
}
private static void Normalize(OpenListSettings settings)
{
settings.Endpoint = NormalizeLegacyEndpoint(settings.Endpoint);
if (string.IsNullOrWhiteSpace(settings.BasePath))
settings.BasePath = string.IsNullOrWhiteSpace(settings.DestinationPath) ? "/dysync" : settings.DestinationPath;
if (string.IsNullOrWhiteSpace(settings.LocalStagingPath))
settings.LocalStagingPath = GetDefaultLocalStagingPath();
if (string.IsNullOrWhiteSpace(settings.SourcePath)) settings.SourcePath = "/dysync-staging";
settings.DestinationPath = settings.BasePath;
}
public async Task InitializeCookiePathsAsync()
{
var cookies = await _db.Queryable<DouyinCookie>().ToListAsync();
foreach (var cookie in cookies)
{
ApplyDefaultCookiePaths(cookie);
}
if (cookies.Count > 0) await _db.Updateable(cookies).ExecuteCommandAsync();
}
public void ApplyDefaultCookiePaths(DouyinCookie cookie)
{
if (cookie == null) return;
var root = DouyinFileNameHelper.SanitizeLinuxFileName(cookie.UserName, cookie.Id, true);
if (string.IsNullOrWhiteSpace(cookie.WebDavCollectPath)) cookie.WebDavCollectPath = $"/{root}/collect";
if (string.IsNullOrWhiteSpace(cookie.WebDavFavoritePath)) cookie.WebDavFavoritePath = $"/{root}/favorite";
if (string.IsNullOrWhiteSpace(cookie.WebDavFollowPath)) cookie.WebDavFollowPath = $"/{root}/follow";
if (string.IsNullOrWhiteSpace(cookie.WebDavMixPath)) cookie.WebDavMixPath = $"/{root}/mix";
if (string.IsNullOrWhiteSpace(cookie.WebDavSeriesPath)) cookie.WebDavSeriesPath = $"/{root}/series";
}
}
}
+547
View File
@@ -0,0 +1,547 @@
using dy.net.model.dto;
using dy.net.model.entity;
using dy.net.storage;
using SqlSugar;
namespace dy.net.service
{
/// <summary>
/// 将本地中转文件交给 OpenList 服务端复制。状态与外部任务 ID 持久化,应用重启后可继续。
/// </summary>
public sealed class OpenListTransferService
{
private const int MaxAttempts = 6;
private static readonly TimeSpan LeaseDuration = TimeSpan.FromSeconds(30);
private static readonly TimeSpan StallTimeout = TimeSpan.FromMinutes(60);
private static readonly TimeSpan ExternalTimeout = TimeSpan.FromHours(24);
private static readonly TimeSpan[] RetryDelays =
{
TimeSpan.FromMinutes(1),
TimeSpan.FromMinutes(5),
TimeSpan.FromMinutes(15),
TimeSpan.FromHours(1),
TimeSpan.FromHours(6)
};
private readonly ISqlSugarClient _db;
private readonly OpenListSettingsService _settingsService;
private readonly OpenListClient _client;
public OpenListTransferService(
ISqlSugarClient db,
OpenListSettingsService settingsService,
OpenListClient client)
{
_db = db;
_settingsService = settingsService;
_client = client;
}
public async Task<long> TransferAsync(
string logicalTargetPath,
Stream source,
long? declaredLength,
CancellationToken cancellationToken)
{
var settings = await _settingsService.GetAsync();
ValidateSettings(settings);
logicalTargetPath = StoragePath.NormalizeRemote(logicalTargetPath);
if (logicalTargetPath == "/") throw new ArgumentException("OpenList 文件路径不能为空。", nameof(logicalTargetPath));
var password = await _settingsService.GetPasswordAsync(settings);
if (string.IsNullOrWhiteSpace(password)) throw new InvalidOperationException("OpenList 密码无效,请重新保存配置。");
var fingerprint = StorageConfigurationFingerprint.Create(settings);
var requestedLogicalTarget = logicalTargetPath;
var actualTarget = await _client.ResolveCanonicalObjectPathAsync(settings, password,
ToActualPath(settings, logicalTargetPath), false, cancellationToken);
logicalTargetPath = ToLogicalPath(settings, actualTarget);
if (declaredLength is > 0)
{
var target = await _client.TryGetObjectAsync(settings, password, actualTarget, cancellationToken);
if (target is { IsDirectory: false } && target.Size == declaredLength.Value) return target.Size;
}
var reusable = await _db.Queryable<OpenListTransferJob>().Where(x =>
x.ConfigurationFingerprint == fingerprint
&& (x.LogicalTargetPath == logicalTargetPath || x.LogicalTargetPath == requestedLogicalTarget)
&& x.Status != OpenListTransferStatus.Succeeded
&& x.Status != OpenListTransferStatus.Failed
&& x.Status != OpenListTransferStatus.Cancelled)
.OrderByDescending(x => x.CreatedAt).FirstAsync();
var job = reusable ?? CreateJob(settings, fingerprint, logicalTargetPath, actualTarget);
if (reusable != null && (job.LogicalTargetPath != logicalTargetPath || job.ActualTargetPath != actualTarget))
{
job.LogicalTargetPath = logicalTargetPath;
job.ActualTargetPath = actualTarget;
job.UpdatedAt = DateTime.Now;
}
if (!File.Exists(job.LocalSourcePath)
|| new FileInfo(job.LocalSourcePath).Length <= 0
|| declaredLength is > 0 && new FileInfo(job.LocalSourcePath).Length != declaredLength.Value)
{
await StageSourceAsync(job, source, declaredLength, cancellationToken);
}
else if (job.ExpectedLength <= 0)
{
job.ExpectedLength = new FileInfo(job.LocalSourcePath).Length;
}
if (reusable == null) await _db.Insertable(job).ExecuteCommandAsync();
else
{
job.Status = OpenListTransferStatus.Queued;
job.NextAttemptAt = null;
job.ErrorMessage = null;
job.UpdatedAt = DateTime.Now;
await _db.Updateable(job).ExecuteCommandAsync();
}
await RunUntilSettledAsync(job.Id, cancellationToken);
var completed = await _db.Queryable<OpenListTransferJob>().InSingleAsync(job.Id);
if (completed?.Status != OpenListTransferStatus.Succeeded)
throw new IOException(completed?.ErrorMessage ?? "OpenList 服务端复制未完成。");
return completed.ExpectedLength;
}
public async Task<bool> RecoverOneAsync(CancellationToken cancellationToken)
{
var now = DateTime.Now;
var job = await _db.Queryable<OpenListTransferJob>().Where(x =>
(x.Status == OpenListTransferStatus.Queued
|| x.Status == OpenListTransferStatus.WaitingForSource
|| x.Status == OpenListTransferStatus.Copying
|| x.Status == OpenListTransferStatus.Verifying
|| x.Status == OpenListTransferStatus.Promoting
|| x.Status == OpenListTransferStatus.WaitingRetry && x.NextAttemptAt <= now)
&& (x.LeaseUntil == null || x.LeaseUntil < now))
.OrderBy(x => x.CreatedAt).FirstAsync();
if (job == null) return false;
await RunUntilSettledAsync(job.Id, cancellationToken, throwOnScheduledRetry: false);
return true;
}
public static string ToActualPath(OpenListSettings settings, string logicalPath) =>
StoragePath.CombineRemote(settings.BasePath, StoragePath.NormalizeRemote(logicalPath));
public static string ToLogicalPath(OpenListSettings settings, string actualPath)
{
var actualSegments = StoragePath.NormalizeRemote(actualPath)
.Split('/', StringSplitOptions.RemoveEmptyEntries);
var baseSegments = StoragePath.NormalizeRemote(settings.BasePath)
.Split('/', StringSplitOptions.RemoveEmptyEntries);
if (actualSegments.Length < baseSegments.Length)
throw new InvalidOperationException($"OpenList 实际路径 '{actualPath}' 不在基础目录 '{settings.BasePath}' 下。");
for (var i = 0; i < baseSegments.Length; i++)
if (!string.Equals(actualSegments[i], baseSegments[i], StringComparison.OrdinalIgnoreCase))
throw new InvalidOperationException($"OpenList 实际路径 '{actualPath}' 不在基础目录 '{settings.BasePath}' 下。");
return StoragePath.NormalizeRemote(string.Join('/', actualSegments.Skip(baseSegments.Length)));
}
public static void ValidateSettings(OpenListSettings settings)
{
if (settings == null) throw new InvalidOperationException("尚未配置 OpenList。");
_ = OpenListClient.NormalizeBaseUrl(settings.Endpoint);
if (string.IsNullOrWhiteSpace(settings.UserName)) throw new InvalidOperationException("OpenList 用户名不能为空。");
if (string.IsNullOrWhiteSpace(settings.LocalStagingPath) || !Path.IsPathRooted(settings.LocalStagingPath))
throw new InvalidOperationException("本地中转目录必须是绝对路径。");
if (string.IsNullOrWhiteSpace(settings.SourcePath)) throw new InvalidOperationException("OpenList 源挂载目录不能为空。");
_ = StoragePath.NormalizeRemote(settings.SourcePath);
_ = StoragePath.NormalizeRemote(settings.BasePath);
}
private OpenListTransferJob CreateJob(
OpenListSettings settings,
string fingerprint,
string logicalTarget,
string actualTarget)
{
var id = Guid.NewGuid().ToString("N");
var fileName = GetFileName(logicalTarget);
var localDirectory = Path.Combine(Path.GetFullPath(settings.LocalStagingPath), id);
var now = DateTime.Now;
return new OpenListTransferJob
{
Id = id,
Status = OpenListTransferStatus.Queued,
ConfigurationFingerprint = fingerprint,
LogicalTargetPath = logicalTarget,
ActualTargetPath = actualTarget,
LocalSourcePath = Path.Combine(localDirectory, fileName),
OpenListSourcePath = StoragePath.CombineRemote(settings.SourcePath, id, fileName),
StagedTargetPath = StoragePath.CombineRemote(settings.BasePath, $".dysync-staging-{id}", fileName),
CreatedAt = now,
UpdatedAt = now
};
}
private async Task StageSourceAsync(
OpenListTransferJob job,
Stream source,
long? declaredLength,
CancellationToken cancellationToken)
{
var directory = Path.GetDirectoryName(job.LocalSourcePath)
?? throw new InvalidOperationException("OpenList 本地中转目录无效。");
Directory.CreateDirectory(directory);
var temporary = job.LocalSourcePath + ".writing-" + Guid.NewGuid().ToString("N");
try
{
await using (var output = new FileStream(temporary, FileMode.CreateNew, FileAccess.Write,
FileShare.None, 81920, FileOptions.Asynchronous | FileOptions.SequentialScan))
await source.CopyToAsync(output, 81920, cancellationToken);
var length = new FileInfo(temporary).Length;
if (length <= 0) throw new MediaIntegrityException("媒体来源写入 OpenList 中转文件后为空。");
if (declaredLength is > 0 && length != declaredLength.Value)
throw new MediaIntegrityException($"下载长度不一致:期望 {declaredLength},实际 {length}。");
File.Move(temporary, job.LocalSourcePath, true);
job.ExpectedLength = length;
}
finally
{
if (File.Exists(temporary)) File.Delete(temporary);
}
}
private async Task RunUntilSettledAsync(
string jobId,
CancellationToken cancellationToken,
bool throwOnScheduledRetry = true)
{
var leaseOwner = Guid.NewGuid().ToString("N");
if (!await TryAcquireLeaseAsync(jobId, leaseOwner))
{
await WaitForOtherProcessorAsync(jobId, cancellationToken);
return;
}
try
{
while (!cancellationToken.IsCancellationRequested)
{
var job = await _db.Queryable<OpenListTransferJob>().InSingleAsync(jobId)
?? throw new InvalidOperationException("OpenList 传输任务不存在。");
if (job.Status == OpenListTransferStatus.Succeeded) return;
if (job.Status is OpenListTransferStatus.Failed or OpenListTransferStatus.Cancelled)
throw new IOException(job.ErrorMessage ?? "OpenList 传输任务失败。");
if (job.Status == OpenListTransferStatus.WaitingRetry && job.NextAttemptAt > DateTime.Now)
{
if (throwOnScheduledRetry) throw new IOException(job.ErrorMessage ?? "OpenList 传输等待重试。");
return;
}
await RenewLeaseAsync(job, leaseOwner);
try
{
var settled = await ProcessStepAsync(job, cancellationToken);
if (settled) return;
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
catch (Exception ex)
{
await ScheduleRetryOrFailAsync(job, ex.GetBaseException().Message);
if (throwOnScheduledRetry) throw;
return;
}
await Task.Delay(TimeSpan.FromSeconds(2), cancellationToken);
}
}
finally
{
await ReleaseLeaseAsync(jobId, leaseOwner);
}
}
private async Task<bool> ProcessStepAsync(OpenListTransferJob job, CancellationToken cancellationToken)
{
var settings = await _settingsService.GetAsync();
if (!string.Equals(StorageConfigurationFingerprint.Create(settings), job.ConfigurationFingerprint, StringComparison.Ordinal))
throw new InvalidOperationException("OpenList 配置已变化,原传输任务不会写入新目标。");
var password = await _settingsService.GetPasswordAsync(settings);
var canonicalActual = await _client.ResolveCanonicalObjectPathAsync(settings, password,
job.ActualTargetPath, false, cancellationToken);
var canonicalLogical = ToLogicalPath(settings, canonicalActual);
if (!string.Equals(job.ActualTargetPath, canonicalActual, StringComparison.Ordinal)
|| !string.Equals(job.LogicalTargetPath, canonicalLogical, StringComparison.Ordinal))
{
job.ActualTargetPath = canonicalActual;
job.LogicalTargetPath = canonicalLogical;
job.UpdatedAt = DateTime.Now;
await _db.Updateable(job).ExecuteCommandAsync();
}
var target = await _client.TryGetObjectAsync(settings, password, job.ActualTargetPath, cancellationToken);
if (target is { IsDirectory: false } && target.Size == job.ExpectedLength)
{
await MarkSucceededAsync(job, settings, password, cancellationToken);
return true;
}
if (job.Status is OpenListTransferStatus.Queued or OpenListTransferStatus.WaitingForSource
or OpenListTransferStatus.WaitingRetry)
{
if (!File.Exists(job.LocalSourcePath) || new FileInfo(job.LocalSourcePath).Length != job.ExpectedLength)
throw new FileNotFoundException("OpenList 本地中转文件不存在或长度已变化。", job.LocalSourcePath);
var source = await _client.TryGetObjectAsync(settings, password, job.OpenListSourcePath, cancellationToken);
if (source is not { IsDirectory: false } || source.Size != job.ExpectedLength)
{
job.Status = OpenListTransferStatus.WaitingForSource;
job.ErrorMessage = "OpenList 尚未看到本地中转文件,请检查源挂载目录映射。";
job.UpdatedAt = DateTime.Now;
await _db.Updateable(job).ExecuteCommandAsync();
throw new IOException(job.ErrorMessage);
}
await _client.EnsureDirectoryAsync(settings, password,
StoragePath.DirectoryName(job.ActualTargetPath), cancellationToken);
await _client.EnsureDirectoryAsync(settings, password,
StoragePath.DirectoryName(job.StagedTargetPath), cancellationToken);
var staged = await _client.TryGetObjectAsync(settings, password, job.StagedTargetPath, cancellationToken);
if (staged is { IsDirectory: false } && staged.Size == job.ExpectedLength)
{
job.Status = OpenListTransferStatus.Verifying;
job.ErrorMessage = null;
job.UpdatedAt = DateTime.Now;
await _db.Updateable(job).ExecuteCommandAsync();
return false;
}
if (staged != null) await _client.DeleteObjectAsync(settings, password, job.StagedTargetPath, cancellationToken);
var copy = await _client.CopyFileAsync(settings, password,
job.OpenListSourcePath, job.StagedTargetPath, cancellationToken);
job.Attempts++;
job.ExternalTaskId = copy.TaskIds.FirstOrDefault();
job.ExternalTaskStartedAt = DateTime.Now;
job.LastProgressAt = DateTime.Now;
job.ExternalProgress = 0;
job.Status = string.IsNullOrWhiteSpace(job.ExternalTaskId)
? OpenListTransferStatus.Verifying
: OpenListTransferStatus.Copying;
job.ErrorMessage = null;
job.NextAttemptAt = null;
job.UpdatedAt = DateTime.Now;
await _db.Updateable(job).ExecuteCommandAsync();
return false;
}
if (job.Status == OpenListTransferStatus.Copying)
{
if (job.ExternalTaskStartedAt.HasValue && DateTime.Now - job.ExternalTaskStartedAt.Value > ExternalTimeout)
{
_ = await _client.TryCancelCopyTaskAsync(settings, password, job.ExternalTaskId, cancellationToken);
throw new TimeoutException("OpenList 复制任务运行超过 24 小时。");
}
if (job.LastProgressAt.HasValue && DateTime.Now - job.LastProgressAt.Value > StallTimeout)
{
var cancelled = await _client.TryCancelCopyTaskAsync(settings, password, job.ExternalTaskId, cancellationToken);
throw new TimeoutException(cancelled
? "OpenList 复制任务连续 60 分钟没有进度,已取消。"
: "OpenList 复制任务连续 60 分钟没有进度且无法取消。");
}
var task = await _client.TryGetCopyTaskAsync(settings, password, job.ExternalTaskId, cancellationToken);
if (task == null)
{
var staged = await _client.TryGetObjectAsync(settings, password, job.StagedTargetPath, cancellationToken);
if (staged is not { IsDirectory: false } || staged.Size != job.ExpectedLength)
throw new IOException($"OpenList 复制任务不存在:{job.ExternalTaskId}");
job.Status = OpenListTransferStatus.Verifying;
}
else if (task.State == 2) job.Status = OpenListTransferStatus.Verifying;
else if (task.State is 4 or 7)
throw new IOException(string.IsNullOrWhiteSpace(task.Error)
? $"OpenList 复制任务以状态 {task.State} 结束。" : task.Error);
else if (Math.Abs(task.Progress - job.ExternalProgress) > 0.001)
{
job.ExternalProgress = task.Progress;
job.LastProgressAt = DateTime.Now;
}
job.UpdatedAt = DateTime.Now;
await _db.Updateable(job).ExecuteCommandAsync();
return false;
}
if (job.Status == OpenListTransferStatus.Verifying)
{
var staged = await _client.TryGetObjectAsync(settings, password, job.StagedTargetPath, cancellationToken);
if (staged is not { IsDirectory: false } || staged.Size != job.ExpectedLength)
throw new IOException($"OpenList 暂存文件长度校验失败:期望 {job.ExpectedLength},实际 {staged?.Size.ToString() ?? ""}。");
job.Status = OpenListTransferStatus.Promoting;
job.UpdatedAt = DateTime.Now;
await _db.Updateable(job).ExecuteCommandAsync();
return false;
}
if (job.Status == OpenListTransferStatus.Promoting)
{
await PromoteAsync(job, settings, password, cancellationToken);
await MarkSucceededAsync(job, settings, password, cancellationToken);
return true;
}
return false;
}
private async Task PromoteAsync(
OpenListTransferJob job,
OpenListSettings settings,
string password,
CancellationToken cancellationToken)
{
var target = await _client.TryGetObjectAsync(settings, password, job.ActualTargetPath, cancellationToken);
if (target is { IsDirectory: false } && target.Size == job.ExpectedLength) return;
if (target != null && string.IsNullOrWhiteSpace(job.BackupTargetPath))
{
var backupName = $".dysync-old-{job.Id[..8]}-{GetFileName(job.ActualTargetPath)}";
await _client.RenameAsync(settings, password, job.ActualTargetPath, backupName, cancellationToken);
job.BackupTargetPath = StoragePath.CombineRemote(
StoragePath.DirectoryName(job.ActualTargetPath), backupName);
job.UpdatedAt = DateTime.Now;
await _db.Updateable(job).ExecuteCommandAsync();
}
try
{
var final = await _client.TryGetObjectAsync(settings, password, job.ActualTargetPath, cancellationToken);
if (final == null)
await _client.MoveAsync(settings, password, job.StagedTargetPath,
StoragePath.DirectoryName(job.ActualTargetPath), false, cancellationToken);
final = await _client.TryGetObjectAsync(settings, password, job.ActualTargetPath, cancellationToken);
if (final is not { IsDirectory: false } || final.Size != job.ExpectedLength)
throw new IOException("OpenList 暂存文件提升后最终目标校验失败。");
if (!string.IsNullOrWhiteSpace(job.BackupTargetPath))
{
await _client.DeleteObjectAsync(settings, password, job.BackupTargetPath, cancellationToken);
job.BackupTargetPath = null;
}
}
catch
{
var final = await _client.TryGetObjectAsync(settings, password, job.ActualTargetPath, cancellationToken);
var backup = string.IsNullOrWhiteSpace(job.BackupTargetPath) ? null
: await _client.TryGetObjectAsync(settings, password, job.BackupTargetPath, cancellationToken);
if (final == null && backup != null)
{
await _client.RenameAsync(settings, password, job.BackupTargetPath,
GetFileName(job.ActualTargetPath), cancellationToken);
job.BackupTargetPath = null;
job.UpdatedAt = DateTime.Now;
await _db.Updateable(job).ExecuteCommandAsync();
}
throw;
}
}
private async Task MarkSucceededAsync(
OpenListTransferJob job,
OpenListSettings settings,
string password,
CancellationToken cancellationToken)
{
var final = await _client.TryGetObjectAsync(settings, password, job.ActualTargetPath, cancellationToken);
if (final is not { IsDirectory: false } || final.Size != job.ExpectedLength)
throw new IOException("OpenList 最终目标不存在或长度不一致。");
job.Status = OpenListTransferStatus.Succeeded;
job.ErrorMessage = null;
job.NextAttemptAt = null;
job.ExternalTaskId = null;
job.CompletedAt = DateTime.Now;
job.UpdatedAt = DateTime.Now;
await _db.Updateable(job).ExecuteCommandAsync();
CleanupLocal(job.LocalSourcePath);
try
{
var stagedDirectory = StoragePath.DirectoryName(job.StagedTargetPath);
await _client.DeleteObjectAsync(settings, password, stagedDirectory, cancellationToken);
}
catch (Exception ex) { Serilog.Log.Debug(ex, "清理 OpenList 暂存目录失败:{Path}", job.StagedTargetPath); }
}
private async Task ScheduleRetryOrFailAsync(OpenListTransferJob job, string error)
{
job.ExternalTaskId = null;
job.ExternalProgress = 0;
job.ExternalTaskStartedAt = null;
job.LastProgressAt = null;
job.ErrorMessage = error;
job.UpdatedAt = DateTime.Now;
if (job.Attempts >= MaxAttempts)
{
job.Status = OpenListTransferStatus.Failed;
job.CompletedAt = DateTime.Now;
job.NextAttemptAt = null;
}
else
{
var index = Math.Clamp(Math.Max(1, job.Attempts) - 1, 0, RetryDelays.Length - 1);
job.Status = OpenListTransferStatus.WaitingRetry;
job.NextAttemptAt = DateTime.Now.Add(RetryDelays[index]);
}
await _db.Updateable(job).ExecuteCommandAsync();
}
private async Task<bool> TryAcquireLeaseAsync(string jobId, string owner)
{
var now = DateTime.Now;
var changed = await _db.Updateable<OpenListTransferJob>()
.SetColumns(x => new OpenListTransferJob
{
LeaseOwner = owner,
LeaseUntil = now.Add(LeaseDuration),
UpdatedAt = now
})
.Where(x => x.Id == jobId && (x.LeaseUntil == null || x.LeaseUntil < now))
.ExecuteCommandAsync();
return changed == 1;
}
private async Task RenewLeaseAsync(OpenListTransferJob job, string owner)
{
job.LeaseOwner = owner;
job.LeaseUntil = DateTime.Now.Add(LeaseDuration);
job.UpdatedAt = DateTime.Now;
await _db.Updateable(job).ExecuteCommandAsync();
}
private async Task ReleaseLeaseAsync(string jobId, string owner)
{
await _db.Updateable<OpenListTransferJob>()
.SetColumns(x => new OpenListTransferJob { LeaseOwner = null, LeaseUntil = null, UpdatedAt = DateTime.Now })
.Where(x => x.Id == jobId && x.LeaseOwner == owner).ExecuteCommandAsync();
}
private async Task WaitForOtherProcessorAsync(string jobId, CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
var job = await _db.Queryable<OpenListTransferJob>().InSingleAsync(jobId)
?? throw new InvalidOperationException("OpenList 传输任务不存在。");
if (job.Status == OpenListTransferStatus.Succeeded) return;
if (job.Status is OpenListTransferStatus.Failed or OpenListTransferStatus.Cancelled
or OpenListTransferStatus.WaitingRetry)
throw new IOException(job.ErrorMessage ?? "OpenList 传输未完成。");
if (job.LeaseUntil == null || job.LeaseUntil < DateTime.Now)
{
await RunUntilSettledAsync(jobId, cancellationToken);
return;
}
await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
}
}
private static void CleanupLocal(string path)
{
try
{
if (File.Exists(path)) File.Delete(path);
var directory = Path.GetDirectoryName(path);
if (!string.IsNullOrWhiteSpace(directory) && Directory.Exists(directory)
&& !Directory.EnumerateFileSystemEntries(directory).Any()) Directory.Delete(directory);
}
catch (Exception ex) { Serilog.Log.Warning(ex, "清理 OpenList 本地中转文件失败:{Path}", path); }
}
private static string GetFileName(string path)
{
var normalized = StoragePath.NormalizeRemote(path);
return normalized[(normalized.LastIndexOf('/') + 1)..];
}
}
}
+29
View File
@@ -0,0 +1,29 @@
namespace dy.net.service
{
public sealed class OpenListTransferWorker : BackgroundService
{
private readonly IServiceScopeFactory _scopeFactory;
public OpenListTransferWorker(IServiceScopeFactory scopeFactory) => _scopeFactory = scopeFactory;
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
try
{
using var scope = _scopeFactory.CreateScope();
var worked = await scope.ServiceProvider.GetRequiredService<OpenListTransferService>()
.RecoverOneAsync(stoppingToken);
if (!worked) await Task.Delay(TimeSpan.FromSeconds(3), stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { }
catch (Exception ex)
{
Serilog.Log.Error(ex, "OpenList 传输恢复服务异常");
await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken);
}
}
}
}
}
+438
View File
@@ -0,0 +1,438 @@
using System.Text;
using dy.net.model.dto;
using dy.net.model.entity;
using dy.net.storage;
using dy.net.utils;
using Newtonsoft.Json;
using SqlSugar;
using MediaStorageType = dy.net.model.dto.StorageType;
namespace dy.net.service
{
public class StorageMigrationItemProcessor
{
private readonly ISqlSugarClient _db;
private readonly OpenListSettingsService _settings;
private readonly OpenListMediaStorage _openList;
private readonly DouyinHttpClientService _http;
private readonly DouyinMigrationSourceResolver _sourceResolver;
public StorageMigrationItemProcessor(
ISqlSugarClient db,
OpenListSettingsService settings,
OpenListMediaStorage openList,
DouyinHttpClientService http,
DouyinMigrationSourceResolver sourceResolver)
{
_db = db;
_settings = settings;
_openList = openList;
_http = http;
_sourceResolver = sourceResolver;
}
public async Task ProcessAsync(string itemId, CancellationToken cancellationToken)
{
var item = await _db.Queryable<StorageMigrationItem>().InSingleAsync(itemId);
if (item == null || item.Stage != StorageMigrationItemStage.Pending) return;
var task = await _db.Queryable<StorageMigrationTask>().InSingleAsync(item.TaskId);
if (task?.Status != StorageMigrationTaskStatus.Running) return;
if (task.TargetStorageType != MediaStorageType.OpenList)
{
await _db.Updateable<StorageMigrationTask>()
.SetColumns(x => new StorageMigrationTask
{
Status = StorageMigrationTaskStatus.Cancelled,
ErrorMessage = "旧 WebDAV 迁移任务仅保留审计信息,不能继续写入。",
CompletedAt = DateTime.Now,
UpdatedAt = DateTime.Now
}).Where(x => x.Id == task.Id).ExecuteCommandAsync();
return;
}
var targetSettings = await _settings.GetAsync();
var fingerprint = StorageConfigurationFingerprint.Create(targetSettings);
if (!string.Equals(fingerprint, task.ConfigurationFingerprint, StringComparison.Ordinal))
{
await _db.Updateable<StorageMigrationTask>()
.SetColumns(x => new StorageMigrationTask
{
Status = StorageMigrationTaskStatus.Paused,
ErrorMessage = "OpenList 配置已变化,请恢复原配置并重新预检。",
UpdatedAt = DateTime.Now
}).Where(x => x.Id == task.Id).ExecuteCommandAsync();
return;
}
var targetStorage = _openList.Bind(targetSettings);
var snapshot = JsonConvert.DeserializeObject<DouyinVideo>(item.OldSnapshotJson)
?? throw new InvalidOperationException("迁移条目缺少旧视频快照");
var cookie = await _db.Queryable<DouyinCookie>().InSingleAsync(snapshot.CookieId);
var roots = StorageMigrationPathPolicy.GetLocalRoots(cookie);
var attachments = JsonConvert.DeserializeObject<List<StorageMigrationService.MigrationAttachmentPlan>>(item.AttachmentPlanJson) ?? new();
var uploaded = DeserializeSet(item.UploadedPathsJson);
var owned = DeserializeSet(item.OwnedRemotePathsJson);
var warnings = new List<string>();
try
{
if (targetStorage is ICanonicalMediaStorage canonicalStorage)
{
item.TargetVideoPath = await canonicalStorage.CanonicalizePathAsync(
item.TargetVideoPath, true, cancellationToken);
foreach (var attachment in attachments.Where(x => !string.IsNullOrWhiteSpace(x.TargetPath)))
attachment.TargetPath = await canonicalStorage.CanonicalizePathAsync(
attachment.TargetPath, true, cancellationToken);
item.AttachmentPlanJson = JsonConvert.SerializeObject(attachments);
item.UpdatedAt = DateTime.Now;
await _db.Updateable(item).ExecuteCommandAsync();
}
if (item.AdoptExistingTarget && snapshot.StorageType == MediaStorageType.WebDav)
{
await AdoptExistingAsync(item, snapshot, targetStorage, cancellationToken);
return;
}
item.Stage = StorageMigrationItemStage.Uploading;
item.Attempts++;
item.ErrorMessage = null;
item.UpdatedAt = DateTime.Now;
await _db.Updateable(item).ExecuteCommandAsync();
var mainLength = await UploadMainAsync(snapshot, cookie, roots, item, targetStorage, uploaded, owned, cancellationToken);
item.ExpectedLength = mainLength;
var newDynamic = new List<DouyinMergeVideoDto>();
foreach (var attachment in attachments)
{
if (attachment.TargetPath == item.TargetVideoPath) continue;
try
{
var length = await UploadAttachmentAsync(attachment, cookie, roots, targetStorage, uploaded, owned, cancellationToken);
if (length.HasValue && attachment.Kind == "dynamic")
newDynamic.Add(new DouyinMergeVideoDto { Path = attachment.TargetPath, Height = attachment.Height, Width = attachment.Width });
}
catch (Exception ex)
{
warnings.Add($"{AttachmentName(attachment.Kind)}{ex.Message}");
}
}
item.Stage = StorageMigrationItemStage.Verifying;
item.UploadedPathsJson = JsonConvert.SerializeObject(uploaded);
item.OwnedRemotePathsJson = JsonConvert.SerializeObject(owned);
item.WarningMessage = warnings.Count == 0 ? null : string.Join("", warnings);
item.UpdatedAt = DateTime.Now;
await _db.Updateable(item).ExecuteCommandAsync();
var remoteLength = await targetStorage.GetLengthAsync(item.TargetVideoPath, cancellationToken);
if (!remoteLength.HasValue || remoteLength.Value != mainLength)
throw new IOException($"远端主媒体长度校验失败,期望 {mainLength},实际 {remoteLength?.ToString() ?? ""}");
var migrated = JsonConvert.DeserializeObject<DouyinVideo>(item.OldSnapshotJson);
migrated.StorageType = MediaStorageType.OpenList;
migrated.VideoSavePath = item.TargetVideoPath;
migrated.VideoCoverSavePath = attachments.FirstOrDefault(x => x.Kind == "cover" && uploaded.Contains(x.TargetPath))?.TargetPath;
migrated.AuthorAvatar = attachments.FirstOrDefault(x => x.Kind == "avatar" && uploaded.Contains(x.TargetPath))?.TargetPath;
migrated.DynamicVideos = newDynamic.Count == 0 ? null : JsonConvert.SerializeObject(newDynamic);
migrated.FileSize = mainLength;
var category = string.IsNullOrWhiteSpace(snapshot.CateId) ? null : await _db.Queryable<DouyinCollectCate>().InSingleAsync(snapshot.CateId);
foreach (var nfo in NfoContentBuilder.Build(migrated, category?.Name))
{
var bytes = Encoding.UTF8.GetBytes(nfo.Value);
var existed = await targetStorage.ExistsAsync(nfo.Key, cancellationToken);
await using var content = new MemoryStream(bytes, false);
await targetStorage.WriteAsync(nfo.Key, content, bytes.Length, "application/xml", cancellationToken);
uploaded.Add(nfo.Key);
if (!existed) owned.Add(nfo.Key);
}
item.Stage = StorageMigrationItemStage.Committing;
item.UploadedPathsJson = JsonConvert.SerializeObject(uploaded);
item.OwnedRemotePathsJson = JsonConvert.SerializeObject(owned);
item.UpdatedAt = DateTime.Now;
await _db.Updateable(item).ExecuteCommandAsync();
var transaction = await _db.Ado.UseTranAsync(async () =>
{
var current = await _db.Queryable<DouyinVideo>().InSingleAsync(item.VideoId)
?? throw new InvalidOperationException("原视频记录已不存在");
if (current.StorageType == MediaStorageType.OpenList && current.VideoSavePath == item.TargetVideoPath)
{
// A previous process committed the video and died before updating the item.
}
else if (await TryReconcileCurrentOpenListAsync(
current, item, snapshot, targetStorage, cancellationToken))
{
// A normal sync already converged this record while migration was running.
// The verified OpenList record is authoritative and must not be overwritten.
}
else
{
if (current.StorageType != snapshot.StorageType || current.VideoSavePath != snapshot.VideoSavePath)
throw new InvalidOperationException("原视频记录在迁移期间已变化,拒绝覆盖");
current.StorageType = MediaStorageType.OpenList;
current.VideoSavePath = migrated.VideoSavePath;
current.VideoCoverSavePath = migrated.VideoCoverSavePath;
current.AuthorAvatar = migrated.AuthorAvatar;
current.DynamicVideos = migrated.DynamicVideos;
current.FileSize = migrated.FileSize;
var changed = await _db.Updateable(current).ExecuteCommandAsync();
if (changed != 1) throw new InvalidOperationException("提交原视频记录失败");
}
item.Stage = warnings.Count == 0 ? StorageMigrationItemStage.Succeeded : StorageMigrationItemStage.SucceededWithWarnings;
item.CompletedAt = DateTime.Now;
item.UpdatedAt = DateTime.Now;
item.WarningMessage = warnings.Count == 0 ? null : string.Join("", warnings);
var itemChanged = await _db.Updateable(item).ExecuteCommandAsync();
if (itemChanged != 1) throw new InvalidOperationException("提交迁移条目失败");
});
if (!transaction.IsSuccess) throw transaction.ErrorException ?? new InvalidOperationException("数据库事务提交失败");
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
await FailAsync(item, "应用正在停止,条目将在下次启动后重试。", resetToPending: true);
}
catch (Exception ex)
{
Serilog.Log.Warning(ex, "存储迁移条目失败:{ItemId}/{VideoId}", item.Id, item.VideoId);
item.UploadedPathsJson = JsonConvert.SerializeObject(uploaded);
item.OwnedRemotePathsJson = JsonConvert.SerializeObject(owned);
await FailAsync(item, ex.Message, resetToPending: false);
}
}
private async Task AdoptExistingAsync(
StorageMigrationItem item,
DouyinVideo snapshot,
IMediaStorage targetStorage,
CancellationToken cancellationToken)
{
item.Stage = StorageMigrationItemStage.Verifying;
item.Attempts++;
item.ErrorMessage = null;
item.UpdatedAt = DateTime.Now;
await _db.Updateable(item).ExecuteCommandAsync();
var length = await targetStorage.GetLengthAsync(item.TargetVideoPath, cancellationToken);
if (!length.HasValue || length <= 0
|| snapshot.FileSize > 0 && length.Value != snapshot.FileSize)
throw new IOException($"OpenList 接管校验失败:期望 {snapshot.FileSize},实际 {length?.ToString() ?? ""}");
var warnings = new List<string>();
foreach (var attachment in new[]
{
(Name: "封面", Path: snapshot.VideoCoverSavePath),
(Name: "头像", Path: snapshot.AuthorAvatar)
})
{
if (string.IsNullOrWhiteSpace(attachment.Path)) continue;
try
{
if (!await targetStorage.ExistsAsync(attachment.Path, cancellationToken))
warnings.Add($"{attachment.Name}在 OpenList 中不存在");
}
catch (Exception ex) { warnings.Add($"{attachment.Name}检查失败:{ex.GetBaseException().Message}"); }
}
item.Stage = StorageMigrationItemStage.Committing;
item.ExpectedLength = length.Value;
item.WarningMessage = warnings.Count == 0 ? null : string.Join("", warnings);
item.UpdatedAt = DateTime.Now;
await _db.Updateable(item).ExecuteCommandAsync();
var transaction = await _db.Ado.UseTranAsync(async () =>
{
var current = await _db.Queryable<DouyinVideo>().InSingleAsync(item.VideoId)
?? throw new InvalidOperationException("原视频记录已不存在");
if (current.StorageType == MediaStorageType.OpenList
&& current.VideoSavePath == item.TargetVideoPath)
{
// 上次进程可能已提交视频,但尚未更新迁移条目。
}
else if (await TryReconcileCurrentOpenListAsync(
current, item, snapshot, targetStorage, cancellationToken))
{
// 普通同步已先完成接管;验证当前远端文件后直接收敛迁移条目。
}
else
{
if (current.StorageType != snapshot.StorageType
|| current.VideoSavePath != snapshot.VideoSavePath)
throw new InvalidOperationException("原视频记录在接管期间已变化,拒绝覆盖");
current.StorageType = MediaStorageType.OpenList;
current.VideoSavePath = item.TargetVideoPath;
current.FileSize = length.Value;
if (await _db.Updateable(current).ExecuteCommandAsync() != 1)
throw new InvalidOperationException("提交 OpenList 接管记录失败");
}
item.Stage = warnings.Count == 0
? StorageMigrationItemStage.Succeeded
: StorageMigrationItemStage.SucceededWithWarnings;
item.CompletedAt = DateTime.Now;
item.UpdatedAt = DateTime.Now;
if (await _db.Updateable(item).ExecuteCommandAsync() != 1)
throw new InvalidOperationException("提交 OpenList 接管条目失败");
});
if (!transaction.IsSuccess)
throw transaction.ErrorException ?? new InvalidOperationException("OpenList 接管事务提交失败");
}
private static async Task<bool> TryReconcileCurrentOpenListAsync(
DouyinVideo current,
StorageMigrationItem item,
DouyinVideo snapshot,
IMediaStorage targetStorage,
CancellationToken cancellationToken)
{
if (current?.StorageType != MediaStorageType.OpenList
|| string.IsNullOrWhiteSpace(current.VideoSavePath)) return false;
var canonicalPath = targetStorage is ICanonicalMediaStorage canonicalStorage
? await canonicalStorage.CanonicalizePathAsync(current.VideoSavePath, false, cancellationToken)
: current.VideoSavePath;
var remoteLength = await targetStorage.GetLengthAsync(canonicalPath, cancellationToken);
if (!remoteLength.HasValue || remoteLength.Value <= 0)
throw new IOException($"并发接管记录已指向 OpenList,但远端主媒体不存在:{canonicalPath}");
var expectedLength = current.FileSize > 0
? current.FileSize
: item.ExpectedLength > 0 ? item.ExpectedLength : snapshot?.FileSize ?? 0;
if (expectedLength > 0 && remoteLength.Value != expectedLength)
throw new MediaIntegrityException(
$"并发接管后的远端主媒体长度不一致,期望 {expectedLength},实际 {remoteLength.Value}");
item.TargetVideoPath = canonicalPath;
item.ExpectedLength = remoteLength.Value;
return true;
}
private async Task<long> UploadMainAsync(
DouyinVideo snapshot,
DouyinCookie cookie,
IReadOnlyList<string> roots,
StorageMigrationItem item,
IMediaStorage targetStorage,
HashSet<string> uploaded,
HashSet<string> owned,
CancellationToken cancellationToken)
{
if (SafeLocalMigrationFile.TryResolve(snapshot.VideoSavePath, roots, out var local, out _)
&& new FileInfo(local).Length > 0)
{
var expected = new FileInfo(local).Length;
await UploadLocalAsync(local, item.TargetVideoPath, expected, targetStorage, uploaded, owned, cancellationToken);
return expected;
}
var knownLength = item.ExpectedLength > 0 ? item.ExpectedLength : snapshot.FileSize;
var existingLength = await targetStorage.GetLengthAsync(item.TargetVideoPath, cancellationToken);
if (knownLength > 0 && existingLength == knownLength)
{
uploaded.Add(item.TargetVideoPath);
return knownLength;
}
var existed = existingLength.HasValue;
if (!string.IsNullOrWhiteSpace(snapshot.VideoUrl))
{
var direct = await _http.DownloadToStorageAsync(MediaStorageType.OpenList, snapshot.VideoUrl, item.TargetVideoPath,
cookie?.Cookies, cancellationToken: cancellationToken, maxRetryCount: 1, storageOverride: targetStorage);
if (direct.Success)
{
var directLength = await targetStorage.GetLengthAsync(item.TargetVideoPath, cancellationToken);
if (directLength.HasValue && directLength.Value > 0)
{
uploaded.Add(item.TargetVideoPath);
if (!existed) owned.Add(item.TargetVideoPath);
return directLength.Value;
}
}
}
var urls = (await _sourceResolver.ResolveAsync(snapshot, cookie, cancellationToken)).ToList();
if (urls.Count == 0) throw new FileNotFoundException("本地主媒体缺失,记录 URL 和抖音全量列表均未找到可用回源地址");
var result = await _http.DownloadToStorageAsync(MediaStorageType.OpenList, urls[0], item.TargetVideoPath,
cookie?.Cookies, urls.Skip(1).ToList(), cancellationToken, Math.Min(6, urls.Count), targetStorage);
if (!result.Success) throw result.ToException();
var length = await targetStorage.GetLengthAsync(item.TargetVideoPath, cancellationToken);
if (!length.HasValue || length.Value <= 0) throw new IOException("回源上传后远端主媒体为空");
uploaded.Add(item.TargetVideoPath);
if (!existed) owned.Add(item.TargetVideoPath);
return length.Value;
}
private async Task<long?> UploadAttachmentAsync(
StorageMigrationService.MigrationAttachmentPlan attachment,
DouyinCookie cookie,
IReadOnlyList<string> roots,
IMediaStorage targetStorage,
HashSet<string> uploaded,
HashSet<string> owned,
CancellationToken cancellationToken)
{
if (SafeLocalMigrationFile.TryResolve(attachment.SourcePath, roots, out var local, out _))
{
var length = new FileInfo(local).Length;
if (length <= 0) throw new IOException("本地附件为空");
await UploadLocalAsync(local, attachment.TargetPath, length, targetStorage, uploaded, owned, cancellationToken);
return length;
}
if (string.IsNullOrWhiteSpace(attachment.SourceUrl)) throw new FileNotFoundException("本地附件缺失且无回源地址");
var before = await targetStorage.GetLengthAsync(attachment.TargetPath, cancellationToken);
var result = await _http.DownloadToStorageAsync(MediaStorageType.OpenList, attachment.SourceUrl, attachment.TargetPath,
cookie?.Cookies, cancellationToken: cancellationToken, storageOverride: targetStorage);
if (!result.Success) throw result.ToException();
var after = await targetStorage.GetLengthAsync(attachment.TargetPath, cancellationToken);
if (!after.HasValue || after.Value <= 0) throw new IOException("附件上传后为空");
uploaded.Add(attachment.TargetPath);
if (!before.HasValue) owned.Add(attachment.TargetPath);
return after;
}
private async Task UploadLocalAsync(
string local,
string target,
long expected,
IMediaStorage targetStorage,
HashSet<string> uploaded,
HashSet<string> owned,
CancellationToken cancellationToken)
{
var before = await targetStorage.GetLengthAsync(target, cancellationToken);
if (before == expected)
{
uploaded.Add(target);
return;
}
await using var source = new FileStream(local, FileMode.Open, FileAccess.Read, FileShare.Read, 81920, true);
await targetStorage.WriteAsync(target, source, expected, null, cancellationToken);
var after = await targetStorage.GetLengthAsync(target, cancellationToken);
if (after != expected) throw new IOException($"附件长度校验失败,期望 {expected},实际 {after?.ToString() ?? ""}");
uploaded.Add(target);
if (!before.HasValue) owned.Add(target);
}
private async Task FailAsync(StorageMigrationItem item, string error, bool resetToPending)
{
item.Stage = resetToPending ? StorageMigrationItemStage.Pending : StorageMigrationItemStage.Failed;
item.ErrorMessage = error;
item.UpdatedAt = DateTime.Now;
await _db.Updateable(item).ExecuteCommandAsync();
}
private static HashSet<string> DeserializeSet(string json)
{
try { return new HashSet<string>(JsonConvert.DeserializeObject<List<string>>(json) ?? new(), StringComparer.Ordinal); }
catch { return new HashSet<string>(StringComparer.Ordinal); }
}
private static string AttachmentName(string kind) => kind switch
{
"cover" => "封面",
"avatar" => "头像",
"dynamic" => "图文/动态附件",
_ => "附件"
};
}
}
+958
View File
@@ -0,0 +1,958 @@
using System.Security.Cryptography;
using System.Text;
using dy.net.model.dto;
using dy.net.model.entity;
using dy.net.storage;
using dy.net.utils;
using Newtonsoft.Json;
using SqlSugar;
using MediaStorageType = dy.net.model.dto.StorageType;
namespace dy.net.service
{
public class StorageMigrationService
{
private readonly ISqlSugarClient _db;
private readonly DouyinCommonService _commonService;
private readonly OpenListSettingsService _settingsService;
private readonly OpenListMediaStorage _openList;
public StorageMigrationService(
ISqlSugarClient db,
DouyinCommonService commonService,
OpenListSettingsService settingsService,
OpenListMediaStorage openList)
{
_db = db;
_commonService = commonService;
_settingsService = settingsService;
_openList = openList;
}
public async Task<StorageMigrationPreflightResult> PreflightAsync(bool verifyCapabilities, CancellationToken cancellationToken)
{
var result = new StorageMigrationPreflightResult();
if (_commonService.GetConfig()?.StorageType != MediaStorageType.OpenList)
{
result.Errors.Add("请先完成测试并启用 OpenList 原生存储;迁移不会自动切换存储模式。");
return result;
}
var settings = await _settingsService.GetAsync();
try
{
OpenListTransferService.ValidateSettings(settings);
if (string.IsNullOrWhiteSpace(await _settingsService.GetPasswordAsync(settings)))
result.Errors.Add("OpenList 密码无效,请重新输入并保存配置。");
else
result.ConfigurationFingerprint = StorageConfigurationFingerprint.Create(settings);
}
catch (Exception ex)
{
result.Errors.Add(ex.GetBaseException().Message);
}
if (!settings.LastTestedAt.HasValue)
result.Errors.Add("当前 OpenList 配置尚未通过服务端复制能力测试。");
if (verifyCapabilities)
{
var probe = await _openList.ProbeAsync(settings, cancellationToken);
if (!probe.Success) result.Errors.Add("OpenList 能力测试失败:" + probe.Message);
}
if (result.Errors.Count > 0) return result;
List<PlanningEntry> planning;
try { planning = await BuildPlanningAsync(); }
catch (Exception ex)
{
result.Errors.Add("读取 OpenList 目标失败:" + ex.GetBaseException().Message);
return result;
}
result.RecordCount = planning.Count;
result.LocalRecordCount = planning.Count(x => x.Video.StorageType == MediaStorageType.Local);
result.LegacyWebDavRecordCount = planning.Count(x => x.Video.StorageType == MediaStorageType.WebDav);
result.AdoptableCount = planning.Count(x => x.AdoptExistingTarget);
result.TransferRequiredCount = planning.Count(x => !x.AdoptExistingTarget);
result.ReadableFileCount = planning.Count(x => x.LocalReadable);
result.MissingFileCount = planning.Count(x => !x.AdoptExistingTarget && !x.LocalReadable);
result.TotalBytes = planning.Where(x => x.LocalReadable).Sum(x => x.LocalLength);
result.InvalidCookieCount = planning.Where(x => x.Cookie == null || x.Cookie.StatusCode != 0 || string.IsNullOrWhiteSpace(x.Cookie.Cookies))
.Select(x => x.Video.CookieId).Distinct().Count();
result.MissingTargetPathCount = planning.Count(x => x.Plan == null);
result.ConflictCount = planning.Where(x => x.Plan != null)
.GroupBy(x => x.Plan.VideoPath, StringComparer.Ordinal)
.Where(x => x.Count() > 1).Sum(x => x.Count());
foreach (var error in planning.Select(x => x.PlanError).Where(x => !string.IsNullOrWhiteSpace(x)).Distinct().Take(20))
result.Errors.Add(error);
if (result.ConflictCount > 0)
result.Errors.Add($"发现 {result.ConflictCount} 条记录映射到重复目标路径,必须先调整目录或命名配置。");
if (result.MissingFileCount > 0)
result.Warnings.Add($"{result.MissingFileCount} 条记录的本地主媒体缺失或不可安全读取,执行时会先尝试记录 URL,再扫描对应抖音全量列表回源。");
if (result.InvalidCookieCount > 0)
result.Warnings.Add($"{result.InvalidCookieCount} 个账号 Cookie 无效;这些账号下缺失本地文件的作品可能无法回源。");
result.Warnings.Add($"OpenList 无法统一报告所有后端剩余容量;容量未知,预计需读取本地数据 {FormatBytes(result.TotalBytes)}。");
result.CanStart = result.RecordCount > 0 && result.Errors.Count == 0;
if (result.RecordCount == 0) result.Errors.Add("没有需要接管到 OpenList 的旧存储视频记录。");
return result;
}
public async Task<StorageMigrationTask> CreateAsync(CreateStorageMigrationRequest request, CancellationToken cancellationToken)
{
request ??= new CreateStorageMigrationRequest();
if (request.Concurrency is < 1 or > 3) throw new InvalidOperationException("迁移并发只能选择 13。");
var preflight = await PreflightAsync(false, cancellationToken);
if (!preflight.CanStart) throw new InvalidOperationException(string.Join("", preflight.Errors));
if (string.IsNullOrWhiteSpace(request.ConfigurationFingerprint)
|| !string.Equals(request.ConfigurationFingerprint, preflight.ConfigurationFingerprint, StringComparison.Ordinal))
throw new InvalidOperationException("OpenList 配置已变化,请重新执行迁移预检。");
var active = await _db.Queryable<StorageMigrationTask>().Where(x =>
x.Status == StorageMigrationTaskStatus.Queued || x.Status == StorageMigrationTaskStatus.Running
|| x.Status == StorageMigrationTaskStatus.Paused || x.Status == StorageMigrationTaskStatus.Cleaning).AnyAsync();
if (active) throw new InvalidOperationException("已有未结束的存储迁移任务,请先处理当前任务。");
var planning = await BuildPlanningAsync();
if (planning.Any(x => x.Plan == null || !string.IsNullOrWhiteSpace(x.PlanError)))
throw new InvalidOperationException("迁移计划在创建前发生变化,请重新执行预检。");
var now = DateTime.Now;
var task = new StorageMigrationTask
{
Id = Guid.NewGuid().ToString("N"),
Status = StorageMigrationTaskStatus.Queued,
TargetStorageType = MediaStorageType.OpenList,
Concurrency = request.Concurrency,
TotalCount = planning.Count,
PendingCount = planning.Count,
TotalBytes = planning.Where(x => x.LocalReadable).Sum(x => x.LocalLength),
ConfigurationFingerprint = preflight.ConfigurationFingerprint,
CreatedAt = now,
UpdatedAt = now
};
var items = planning.Select(entry => new StorageMigrationItem
{
Id = Guid.NewGuid().ToString("N"),
TaskId = task.Id,
VideoId = entry.Video.Id,
SourceStorageType = entry.Video.StorageType,
AdoptExistingTarget = entry.AdoptExistingTarget,
Stage = StorageMigrationItemStage.Pending,
ExpectedLength = entry.LocalLength > 0 ? entry.LocalLength : entry.Video.FileSize,
OldVideoPath = entry.Video.VideoSavePath,
TargetVideoPath = entry.Plan.VideoPath,
TargetCoverPath = entry.Plan.CoverPath,
OldSnapshotJson = JsonConvert.SerializeObject(entry.Video),
AttachmentPlanJson = JsonConvert.SerializeObject(BuildAttachments(entry)),
UploadedPathsJson = "[]",
OwnedRemotePathsJson = "[]",
CreatedAt = now,
UpdatedAt = now
}).ToList();
var transaction = await _db.Ado.UseTranAsync(async () =>
{
await _db.Insertable(task).ExecuteCommandAsync();
await _db.Insertable(items).ExecuteCommandAsync();
});
if (!transaction.IsSuccess) throw new InvalidOperationException("创建迁移任务失败:" + transaction.ErrorMessage);
return task;
}
public async Task<StorageMigrationTaskDetail> GetAsync(string id)
{
var task = await _db.Queryable<StorageMigrationTask>().InSingleAsync(id)
?? throw new KeyNotFoundException("迁移任务不存在");
return ToDetail(task);
}
public async Task<StorageMigrationTaskDetail> GetLatestAsync()
{
var task = await _db.Queryable<StorageMigrationTask>().Where(x => !x.IsArchived)
.OrderByDescending(x => x.CreatedAt).FirstAsync();
return task == null ? null : ToDetail(task);
}
public async Task ArchiveAsync(string id)
{
var task = await RequireTaskAsync(id);
if (!CanArchive(task))
throw new InvalidOperationException("该迁移仍有成功项等待清理或回滚,请先处理旧文件后再归档。");
task.IsArchived = true;
task.UpdatedAt = DateTime.Now;
if (await _db.Updateable(task).ExecuteCommandAsync() != 1)
throw new InvalidOperationException("归档迁移历史失败。");
}
public async Task<StorageMigrationArchiveResult> ArchiveAllSettledAsync()
{
var tasks = await _db.Queryable<StorageMigrationTask>().Where(x => !x.IsArchived).ToListAsync();
var archivable = tasks.Where(CanArchive).ToList();
var requiresCleanup = tasks.Count(x => IsTerminal(x.Status) && !CanArchive(x));
if (archivable.Count > 0)
{
var now = DateTime.Now;
foreach (var task in archivable)
{
task.IsArchived = true;
task.UpdatedAt = now;
}
var changed = await _db.Updateable(archivable).ExecuteCommandAsync();
if (changed != archivable.Count) throw new InvalidOperationException("部分迁移历史归档失败。");
}
return new StorageMigrationArchiveResult
{
ArchivedCount = archivable.Count,
RequiresCleanupCount = requiresCleanup,
Message = requiresCleanup > 0
? $"已隐藏 {archivable.Count} 条迁移历史;另有 {requiresCleanup} 条包含已迁移视频,请先清理旧文件或回滚。"
: $"已隐藏 {archivable.Count} 条已结束的迁移历史。"
};
}
public async Task<StorageMigrationItemPage> GetItemsAsync(string taskId, StorageMigrationItemPageRequest request)
{
request ??= new StorageMigrationItemPageRequest();
request.PageIndex = Math.Max(1, request.PageIndex);
request.PageSize = Math.Clamp(request.PageSize, 1, 100);
var query = _db.Queryable<StorageMigrationItem>().Where(x => x.TaskId == taskId)
.WhereIF(request.Stage.HasValue, x => x.Stage == request.Stage.Value);
return new StorageMigrationItemPage
{
TotalCount = await query.CountAsync(),
Items = await query.OrderBy(x => x.CreatedAt).Skip((request.PageIndex - 1) * request.PageSize).Take(request.PageSize).ToListAsync()
};
}
public async Task<FailedMigrationRecordPreview> PreviewFailedRecordRemovalAsync()
{
var plan = await BuildFailedRecordRemovalPlanAsync();
var preview = new FailedMigrationRecordPreview
{
ActiveMigrationCount = plan.ActiveMigrationCount,
FailedItemCount = plan.Groups.Sum(x => x.Items.Count),
DistinctVideoCount = plan.Groups.Count,
EligibleRecordCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.Eligible),
AlreadyMissingCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.AlreadyMissing),
ChangedRecordCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.Changed),
PermanentlyExcludedCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.PermanentlyExcluded),
InvalidSnapshotCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.InvalidSnapshot)
};
preview.CanExecute = preview.ActiveMigrationCount == 0
&& preview.EligibleRecordCount + preview.AlreadyMissingCount > 0;
preview.ConfirmationToken = preview.CanExecute ? plan.ConfirmationToken : null;
if (preview.ActiveMigrationCount > 0)
preview.Errors.Add("存在排队、运行、暂停或清理中的存储迁移任务,请先结束当前迁移再处理历史失败记录。");
else if (preview.FailedItemCount == 0)
preview.Errors.Add("当前没有可处理的历史迁移失败项。");
else if (!preview.CanExecute)
preview.Errors.Add("没有可安全删除的视频记录;已变化、已迁移、永久排除或快照损坏的记录不会被修改。");
preview.Warnings.Add("此操作只删除数据库视频记录,不删除旧本地文件或 OpenList 文件,也不会自动启动同步。");
if (preview.ChangedRecordCount > 0)
preview.Warnings.Add($"{preview.ChangedRecordCount} 条视频记录已变化、已迁移或已重新创建,将安全跳过。");
if (preview.PermanentlyExcludedCount > 0)
preview.Warnings.Add($"{preview.PermanentlyExcludedCount} 条视频已被永久排除,请先在任务中心取消排除后再处理。");
if (preview.InvalidSnapshotCount > 0)
preview.Warnings.Add($"{preview.InvalidSnapshotCount} 条迁移快照无效,将保留原记录和失败状态。");
return preview;
}
public async Task<RemoveFailedMigrationRecordsResult> RemoveFailedRecordsAsync(RemoveFailedMigrationRecordsRequest request)
{
if (string.IsNullOrWhiteSpace(request?.ConfirmationToken))
throw new InvalidOperationException("缺少删除确认信息,请重新打开预览后确认。");
RemoveFailedMigrationRecordsResult result = null;
var transaction = await _db.Ado.UseTranAsync(async () =>
{
var plan = await BuildFailedRecordRemovalPlanAsync();
if (plan.ActiveMigrationCount > 0)
throw new InvalidOperationException("存在未结束的存储迁移任务,请先结束当前迁移。");
if (!string.Equals(plan.ConfirmationToken, request.ConfirmationToken, StringComparison.Ordinal))
throw new InvalidOperationException("迁移失败记录已发生变化,请重新预览并确认。");
var actionable = plan.Groups.Where(x => x.Disposition is FailedRecordDisposition.Eligible
or FailedRecordDisposition.AlreadyMissing).ToList();
if (actionable.Count == 0)
throw new InvalidOperationException("没有可安全删除的视频记录。");
var now = DateTime.Now;
var changedItems = new List<StorageMigrationItem>();
var deletedRecords = 0;
foreach (var group in actionable)
{
if (group.Disposition == FailedRecordDisposition.Eligible)
{
var current = group.Current;
var expectedStorageType = current.StorageType;
var deleted = await _db.Deleteable<DouyinVideo>().Where(x => x.Id == current.Id
&& x.AwemeId == current.AwemeId && x.StorageType == expectedStorageType
&& x.VideoSavePath == current.VideoSavePath).ExecuteCommandAsync();
if (deleted != 1)
throw new InvalidOperationException($"视频 {current.AwemeId ?? current.Id} 已发生变化,请重新预览。");
deletedRecords++;
}
foreach (var item in group.Items)
{
item.Stage = StorageMigrationItemStage.RecordRemoved;
item.ErrorMessage = null;
item.WarningMessage = group.Disposition == FailedRecordDisposition.Eligible
? "视频记录已删除;旧文件和 OpenList 文件均已保留,等待后续正常同步重新发现。"
: "原视频记录已不存在;迁移失败项已归档,等待后续正常同步重新发现。";
item.UpdatedAt = now;
item.CompletedAt = now;
changedItems.Add(item);
}
}
if (changedItems.Count > 0)
{
var changed = await _db.Updateable(changedItems).ExecuteCommandAsync();
if (changed != changedItems.Count)
throw new InvalidOperationException("更新迁移失败项状态不完整,操作已回滚。");
}
var affectedTaskIds = changedItems.Select(x => x.TaskId).Distinct().ToList();
foreach (var taskId in affectedTaskIds) await RefreshCountsAsync(taskId);
result = new RemoveFailedMigrationRecordsResult
{
DeletedRecordCount = deletedRecords,
AlreadyMissingCount = actionable.Count(x => x.Disposition == FailedRecordDisposition.AlreadyMissing),
RemovedItemCount = changedItems.Count,
AffectedTaskCount = affectedTaskIds.Count,
SkippedChangedCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.Changed),
SkippedExcludedCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.PermanentlyExcluded),
InvalidSnapshotCount = plan.Groups.Count(x => x.Disposition == FailedRecordDisposition.InvalidSnapshot),
Message = $"已删除 {deletedRecords} 条视频记录,更新 {changedItems.Count} 条迁移失败项;旧文件已保留,将随系统后续正常同步重新发现。"
};
});
if (!transaction.IsSuccess)
throw transaction.ErrorException ?? new InvalidOperationException("删除迁移失败记录失败,数据库操作已回滚。");
return result;
}
public Task PauseAsync(string id) => ChangeStatusAsync(id,
new[] { StorageMigrationTaskStatus.Queued, StorageMigrationTaskStatus.Running }, StorageMigrationTaskStatus.Paused, null);
public async Task ResumeAsync(string id)
{
var task = await RequireTaskAsync(id);
if (task.Status != StorageMigrationTaskStatus.Paused) throw new InvalidOperationException("只有暂停中的任务可以恢复。");
var fingerprint = StorageConfigurationFingerprint.Create(await _settingsService.GetAsync());
if (!string.Equals(fingerprint, task.ConfigurationFingerprint, StringComparison.Ordinal))
throw new InvalidOperationException("OpenList 地址、源挂载、基础目录或账号已变化,请重新预检并创建新任务。");
await ChangeStatusAsync(id, new[] { StorageMigrationTaskStatus.Paused }, StorageMigrationTaskStatus.Queued, null);
}
public Task CancelAsync(string id) => ChangeStatusAsync(id,
new[] { StorageMigrationTaskStatus.Queued, StorageMigrationTaskStatus.Running, StorageMigrationTaskStatus.Paused },
StorageMigrationTaskStatus.Cancelled, null);
public async Task RetryFailedAsync(string id)
{
var task = await RequireTaskAsync(id);
if (task.Status is not (StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled or StorageMigrationTaskStatus.Paused))
throw new InvalidOperationException("当前任务没有可重试的失败项。");
var fingerprint = StorageConfigurationFingerprint.Create(await _settingsService.GetAsync());
if (fingerprint != task.ConfigurationFingerprint) throw new InvalidOperationException("OpenList 配置已变化,请重新预检。");
await _db.Updateable<StorageMigrationItem>()
.SetColumns(x => new StorageMigrationItem { Stage = StorageMigrationItemStage.Pending, ErrorMessage = null, UpdatedAt = DateTime.Now })
.Where(x => x.TaskId == id && x.Stage == StorageMigrationItemStage.Failed).ExecuteCommandAsync();
await _db.Updateable<StorageMigrationTask>()
.SetColumns(x => new StorageMigrationTask { Status = StorageMigrationTaskStatus.Queued, CompletedAt = null, ErrorMessage = null, UpdatedAt = DateTime.Now })
.Where(x => x.Id == id).ExecuteCommandAsync();
await RefreshCountsAsync(id);
}
public async Task CleanupAsync(string id, CancellationToken cancellationToken)
{
var task = await RequireTaskAsync(id);
if (task.Status is not (StorageMigrationTaskStatus.Completed or StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled))
throw new InvalidOperationException("只有已结束的任务可以清理旧文件。");
var targetStorage = await BindTaskTargetAsync(task);
await _db.Updateable<StorageMigrationTask>()
.SetColumns(x => new StorageMigrationTask { Status = StorageMigrationTaskStatus.Cleaning, ErrorMessage = null, UpdatedAt = DateTime.Now })
.Where(x => x.Id == id).ExecuteCommandAsync();
var items = await _db.Queryable<StorageMigrationItem>().Where(x => x.TaskId == id
&& (x.Stage == StorageMigrationItemStage.Succeeded || x.Stage == StorageMigrationItemStage.SucceededWithWarnings)).ToListAsync();
var errors = new List<string>();
foreach (var item in items)
{
try { await CleanupItemAsync(item, targetStorage, cancellationToken); }
catch (Exception ex)
{
errors.Add($"{item.VideoId}: {ex.Message}");
item.ErrorMessage = "清理失败:" + ex.Message;
item.UpdatedAt = DateTime.Now;
await _db.Updateable(item).ExecuteCommandAsync();
}
}
var remaining = await _db.Queryable<StorageMigrationItem>().Where(x => x.TaskId == id
&& (x.Stage == StorageMigrationItemStage.Succeeded || x.Stage == StorageMigrationItemStage.SucceededWithWarnings)).CountAsync();
await _db.Updateable<StorageMigrationTask>()
.SetColumns(x => new StorageMigrationTask
{
Status = remaining == 0 ? StorageMigrationTaskStatus.Cleaned : StorageMigrationTaskStatus.PartiallyFailed,
CleanedAt = remaining == 0 ? DateTime.Now : null,
ErrorMessage = errors.Count == 0 ? null : string.Join("", errors.Take(20)),
UpdatedAt = DateTime.Now
}).Where(x => x.Id == id).ExecuteCommandAsync();
await RefreshCountsAsync(id);
}
public async Task RollbackAsync(string id, CancellationToken cancellationToken)
{
var task = await RequireTaskAsync(id);
if (task.Status is not (StorageMigrationTaskStatus.Completed or StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled))
throw new InvalidOperationException("当前任务状态不允许回滚。");
if (task.CleanedAt.HasValue || await _db.Queryable<StorageMigrationItem>().Where(x => x.TaskId == id && x.Stage == StorageMigrationItemStage.Cleaned).AnyAsync())
throw new InvalidOperationException("旧文件清理已开始,不能再执行回滚。");
var targetStorage = await BindTaskTargetAsync(task);
var items = await _db.Queryable<StorageMigrationItem>().Where(x => x.TaskId == id
&& (x.Stage == StorageMigrationItemStage.Succeeded || x.Stage == StorageMigrationItemStage.SucceededWithWarnings)).ToListAsync();
var snapshots = items.ToDictionary(x => x.Id, x => JsonConvert.DeserializeObject<DouyinVideo>(x.OldSnapshotJson));
foreach (var pair in snapshots)
{
var snapshot = pair.Value ?? throw new InvalidOperationException("旧视频快照损坏,无法回滚。");
var item = items.First(x => x.Id == pair.Key);
var sourceType = item.SourceStorageType ?? snapshot.StorageType;
if (sourceType == MediaStorageType.Local)
{
var cookie = await _db.Queryable<DouyinCookie>().InSingleAsync(snapshot.CookieId);
if (!SafeLocalMigrationFile.TryResolve(snapshot.VideoSavePath,
StorageMigrationPathPolicy.GetLocalRoots(cookie), out var local, out var error)
|| !File.Exists(local) || new FileInfo(local).Length <= 0)
throw new InvalidOperationException($"视频 {snapshot.AwemeId} 的旧主媒体不可用:{error ?? ""}");
}
else if (sourceType == MediaStorageType.WebDav)
{
var length = await targetStorage.GetLengthAsync(snapshot.VideoSavePath, cancellationToken);
if (!length.HasValue || length <= 0
|| snapshot.FileSize > 0 && length.Value != snapshot.FileSize)
throw new InvalidOperationException($"视频 {snapshot.AwemeId} 的原 WebDAV 文件已不可用,无法恢复旧记录。");
}
else
{
throw new InvalidOperationException($"视频 {snapshot.AwemeId} 的旧存储类型不支持回滚。");
}
}
var transaction = await _db.Ado.UseTranAsync(async () =>
{
foreach (var item in items)
{
var snapshot = snapshots[item.Id];
var current = await _db.Queryable<DouyinVideo>().InSingleAsync(item.VideoId)
?? throw new InvalidOperationException($"视频记录 {item.VideoId} 已不存在");
if (current.StorageType != MediaStorageType.OpenList || current.VideoSavePath != item.TargetVideoPath)
throw new InvalidOperationException($"视频 {snapshot.AwemeId} 已被其他操作修改,无法回滚");
if (await _db.Updateable(snapshot).ExecuteCommandAsync() != 1)
throw new InvalidOperationException($"恢复视频 {snapshot.AwemeId} 失败");
item.Stage = StorageMigrationItemStage.RolledBack;
item.UpdatedAt = DateTime.Now;
item.CompletedAt = DateTime.Now;
await _db.Updateable(item).ExecuteCommandAsync();
}
});
if (!transaction.IsSuccess) throw transaction.ErrorException ?? new InvalidOperationException("回滚数据库事务失败");
var warnings = new List<string>();
foreach (var item in items)
{
var snapshot = snapshots[item.Id];
if ((item.SourceStorageType ?? snapshot.StorageType) == MediaStorageType.WebDav)
continue;
foreach (var remotePath in DeserializePaths(item.OwnedRemotePathsJson))
{
try
{
if (!await IsRemotePathReferencedAsync(remotePath, cancellationToken)) await targetStorage.DeleteAsync(remotePath, cancellationToken);
}
catch (Exception ex) { warnings.Add($"{remotePath}: {ex.Message}"); }
}
}
await _db.Updateable<StorageMigrationTask>()
.SetColumns(x => new StorageMigrationTask
{
Status = StorageMigrationTaskStatus.RolledBack,
RolledBackAt = DateTime.Now,
ErrorMessage = warnings.Count == 0 ? null : "远端独占文件清理警告:" + string.Join("", warnings.Take(20)),
UpdatedAt = DateTime.Now
}).Where(x => x.Id == id).ExecuteCommandAsync();
await RefreshCountsAsync(id);
}
private async Task CleanupItemAsync(StorageMigrationItem item, IMediaStorage targetStorage, CancellationToken cancellationToken)
{
var current = await _db.Queryable<DouyinVideo>().InSingleAsync(item.VideoId)
?? throw new InvalidOperationException("视频记录不存在");
if (current.StorageType != MediaStorageType.OpenList || current.VideoSavePath != item.TargetVideoPath)
throw new InvalidOperationException("数据库已不再指向本任务的 OpenList 目标");
var remoteLength = await targetStorage.GetLengthAsync(current.VideoSavePath, cancellationToken);
if (!remoteLength.HasValue || remoteLength.Value != item.ExpectedLength)
throw new InvalidOperationException("远端主媒体不存在或长度已变化");
var snapshot = JsonConvert.DeserializeObject<DouyinVideo>(item.OldSnapshotJson)
?? throw new InvalidOperationException("旧视频快照损坏");
if ((item.SourceStorageType ?? snapshot.StorageType) == MediaStorageType.WebDav)
{
item.Stage = StorageMigrationItemStage.Cleaned;
item.ErrorMessage = null;
item.WarningMessage = string.IsNullOrWhiteSpace(item.WarningMessage)
? "原 WebDAV 记录与 OpenList 指向同一文件,无需删除媒体。"
: item.WarningMessage + ";原 WebDAV 记录与 OpenList 指向同一文件,无需删除媒体。";
item.UpdatedAt = DateTime.Now;
item.CompletedAt = DateTime.Now;
await _db.Updateable(item).ExecuteCommandAsync();
return;
}
var cookie = await _db.Queryable<DouyinCookie>().InSingleAsync(snapshot.CookieId);
var roots = StorageMigrationPathPolicy.GetLocalRoots(cookie);
var candidates = BuildLocalCleanupCandidates(snapshot);
var localVideos = await _db.Queryable<DouyinVideo>().Where(x => x.StorageType == MediaStorageType.Local).ToListAsync();
foreach (var candidate in candidates)
{
if (string.IsNullOrWhiteSpace(candidate.Path) || !File.Exists(candidate.Path)) continue;
if (!SafeLocalMigrationFile.TryResolve(candidate.Path, roots, out var safePath, out var error))
throw new InvalidOperationException($"拒绝清理不安全路径 {candidate.Path}{error}");
if (IsLocalPathReferenced(candidate.Path, localVideos, snapshot.Id, candidate.Shared)) continue;
File.Delete(safePath);
}
DeleteEmptyParents(snapshot.VideoSavePath, roots);
item.Stage = StorageMigrationItemStage.Cleaned;
item.ErrorMessage = null;
item.UpdatedAt = DateTime.Now;
item.CompletedAt = DateTime.Now;
await _db.Updateable(item).ExecuteCommandAsync();
}
private static List<CleanupCandidate> BuildLocalCleanupCandidates(DouyinVideo snapshot)
{
var result = new List<CleanupCandidate>
{
new(snapshot.VideoSavePath, false),
new(snapshot.VideoCoverSavePath, snapshot.ViedoType is VideoTypeEnum.dy_mix or VideoTypeEnum.dy_series),
new(snapshot.AuthorAvatar, true)
};
if (!string.IsNullOrWhiteSpace(snapshot.VideoSavePath))
{
var directory = Path.GetDirectoryName(snapshot.VideoSavePath);
var basename = Path.GetFileNameWithoutExtension(snapshot.VideoSavePath);
result.Add(new CleanupCandidate(Path.Combine(directory ?? string.Empty, basename + ".nfo"), false));
if (snapshot.ViedoType is VideoTypeEnum.dy_mix or VideoTypeEnum.dy_series)
result.Add(new CleanupCandidate(Path.Combine(directory ?? string.Empty, "tvshow.nfo"), true));
}
if (!string.IsNullOrWhiteSpace(snapshot.DynamicVideos))
{
try
{
foreach (var attachment in JsonConvert.DeserializeObject<List<DouyinMergeVideoDto>>(snapshot.DynamicVideos) ?? new())
result.Add(new CleanupCandidate(attachment.Path, false));
}
catch (JsonException) { }
}
return result.Where(x => !string.IsNullOrWhiteSpace(x.Path)).DistinctBy(x => x.Path).ToList();
}
private static bool IsLocalPathReferenced(string path, IEnumerable<DouyinVideo> localVideos, string excludedVideoId, bool shared)
{
foreach (var video in localVideos.Where(x => x.Id != excludedVideoId))
{
if (path == video.VideoSavePath || path == video.VideoCoverSavePath || path == video.AuthorAvatar) return true;
if (shared && !string.IsNullOrWhiteSpace(video.VideoSavePath)
&& string.Equals(Path.GetDirectoryName(video.VideoSavePath), Path.GetDirectoryName(path), StringComparison.Ordinal)) return true;
if (!string.IsNullOrWhiteSpace(video.DynamicVideos) && video.DynamicVideos.Contains(path, StringComparison.Ordinal)) return true;
}
return false;
}
private async Task<bool> IsRemotePathReferencedAsync(string path, CancellationToken cancellationToken)
{
var videos = await _db.Queryable<DouyinVideo>().Where(x => x.StorageType.IsRemote()).ToListAsync();
foreach (var video in videos)
{
if (video.VideoSavePath == path || video.VideoCoverSavePath == path || video.AuthorAvatar == path) return true;
if (!string.IsNullOrWhiteSpace(video.DynamicVideos) && video.DynamicVideos.Contains(path, StringComparison.Ordinal)) return true;
if (NfoContentBuilder.Build(video).Keys.Contains(path)) return true;
if (Path.GetFileName(path).Equals("tvshow.nfo", StringComparison.OrdinalIgnoreCase)
&& StoragePath.DirectoryName(video.VideoSavePath) == StoragePath.DirectoryName(path)) return true;
}
return false;
}
private static void DeleteEmptyParents(string filePath, IReadOnlyList<string> roots)
{
if (string.IsNullOrWhiteSpace(filePath)) return;
var directory = Path.GetDirectoryName(Path.GetFullPath(filePath));
var root = roots.Where(x => SafeLocalMigrationFile.IsWithin(filePath, x)).OrderByDescending(x => x.Length).FirstOrDefault();
while (!string.IsNullOrWhiteSpace(directory) && !string.Equals(directory, root, StringComparison.Ordinal)
&& Directory.Exists(directory) && !Directory.EnumerateFileSystemEntries(directory).Any())
{
Directory.Delete(directory, false);
directory = Path.GetDirectoryName(directory);
}
}
private static IReadOnlyList<string> DeserializePaths(string json)
{
try { return JsonConvert.DeserializeObject<List<string>>(json) ?? new(); }
catch { return Array.Empty<string>(); }
}
public async Task RefreshCountsAsync(string taskId)
{
var items = await _db.Queryable<StorageMigrationItem>().Where(x => x.TaskId == taskId).ToListAsync();
var pending = items.Count(i => i.Stage == StorageMigrationItemStage.Pending || i.Stage == StorageMigrationItemStage.Uploading
|| i.Stage == StorageMigrationItemStage.Verifying || i.Stage == StorageMigrationItemStage.Committing);
var success = items.Count(i => i.Stage == StorageMigrationItemStage.Succeeded || i.Stage == StorageMigrationItemStage.SucceededWithWarnings
|| i.Stage == StorageMigrationItemStage.Cleaned);
var warning = items.Count(i => i.Stage == StorageMigrationItemStage.SucceededWithWarnings);
var failed = items.Count(i => i.Stage == StorageMigrationItemStage.Failed);
var removed = items.Count(i => i.Stage == StorageMigrationItemStage.RecordRemoved);
var now = DateTime.Now;
await _db.Updateable<StorageMigrationTask>()
.SetColumns(x => new StorageMigrationTask
{
PendingCount = pending,
SuccessCount = success,
WarningCount = warning,
FailedCount = failed,
RemovedCount = removed,
UpdatedAt = now
}).Where(x => x.Id == taskId).ExecuteCommandAsync();
}
private async Task<FailedRecordRemovalPlan> BuildFailedRecordRemovalPlanAsync()
{
var activeMigrationCount = await _db.Queryable<StorageMigrationTask>().Where(x =>
x.Status == StorageMigrationTaskStatus.Queued || x.Status == StorageMigrationTaskStatus.Running
|| x.Status == StorageMigrationTaskStatus.Paused || x.Status == StorageMigrationTaskStatus.Cleaning).CountAsync();
var terminalTaskIds = await _db.Queryable<StorageMigrationTask>().Where(x =>
x.Status == StorageMigrationTaskStatus.Completed || x.Status == StorageMigrationTaskStatus.PartiallyFailed
|| x.Status == StorageMigrationTaskStatus.Cancelled || x.Status == StorageMigrationTaskStatus.Cleaned
|| x.Status == StorageMigrationTaskStatus.RolledBack).Select(x => x.Id).ToListAsync();
var failedItems = new List<StorageMigrationItem>();
foreach (var taskIdBatch in terminalTaskIds.Chunk(500))
failedItems.AddRange(await _db.Queryable<StorageMigrationItem>().Where(x => taskIdBatch.Contains(x.TaskId)
&& x.Stage == StorageMigrationItemStage.Failed).ToListAsync());
var groups = failedItems.GroupBy(x => x.VideoId).OrderBy(x => x.Key, StringComparer.Ordinal)
.Select(x => new FailedRecordGroup { VideoId = x.Key, Items = x.OrderBy(i => i.Id, StringComparer.Ordinal).ToList() })
.ToList();
foreach (var group in groups)
{
foreach (var item in group.Items)
{
try
{
var snapshot = JsonConvert.DeserializeObject<DouyinVideo>(item.OldSnapshotJson);
if (snapshot != null) group.Snapshots.Add(snapshot);
}
catch (JsonException) { }
}
}
var videoIds = groups.Select(x => x.VideoId).Where(x => !string.IsNullOrWhiteSpace(x)).Distinct().ToList();
var currentVideosById = new List<DouyinVideo>();
foreach (var videoIdBatch in videoIds.Chunk(500))
currentVideosById.AddRange(await _db.Queryable<DouyinVideo>().Where(x => videoIdBatch.Contains(x.Id)).ToListAsync());
var currentById = currentVideosById.ToDictionary(x => x.Id, StringComparer.Ordinal);
var snapshotAwemeIds = groups.SelectMany(x => x.Snapshots).Select(x => x.AwemeId)
.Where(x => !string.IsNullOrWhiteSpace(x)).Distinct().ToList();
var currentVideosByAweme = new List<DouyinVideo>();
var excludedAwemeIds = new HashSet<string>(StringComparer.Ordinal);
foreach (var awemeIdBatch in snapshotAwemeIds.Chunk(500))
{
currentVideosByAweme.AddRange(await _db.Queryable<DouyinVideo>()
.Where(x => awemeIdBatch.Contains(x.AwemeId)).ToListAsync());
excludedAwemeIds.UnionWith(await _db.Queryable<DouyinVideoDelete>()
.Where(x => awemeIdBatch.Contains(x.ViedoId)).Select(x => x.ViedoId).ToListAsync());
}
var currentByAweme = currentVideosByAweme.GroupBy(x => x.AwemeId)
.ToDictionary(x => x.Key, x => x.ToList(), StringComparer.Ordinal);
foreach (var group in groups)
{
var awemeIds = group.Snapshots.Select(x => x.AwemeId).Where(x => !string.IsNullOrWhiteSpace(x))
.Distinct(StringComparer.Ordinal).ToList();
if (group.Snapshots.Count == 0 || awemeIds.Count != 1
|| group.Snapshots.All(x => !string.Equals(x.Id, group.VideoId, StringComparison.Ordinal)))
{
group.Disposition = FailedRecordDisposition.InvalidSnapshot;
continue;
}
group.AwemeId = awemeIds[0];
if (excludedAwemeIds.Contains(group.AwemeId))
{
group.Disposition = FailedRecordDisposition.PermanentlyExcluded;
continue;
}
if (!currentById.TryGetValue(group.VideoId, out var current))
{
group.Disposition = currentByAweme.TryGetValue(group.AwemeId, out var replacements)
&& replacements.Any(x => x.Id != group.VideoId)
? FailedRecordDisposition.Changed
: FailedRecordDisposition.AlreadyMissing;
continue;
}
group.Current = current;
var matchesSnapshot = group.Snapshots.Any(snapshot => current.StorageType == snapshot.StorageType
&& string.Equals(current.AwemeId, group.AwemeId, StringComparison.Ordinal)
&& string.Equals(snapshot.Id, current.Id, StringComparison.Ordinal)
&& string.Equals(snapshot.AwemeId, current.AwemeId, StringComparison.Ordinal)
&& string.Equals(snapshot.VideoSavePath, current.VideoSavePath, StringComparison.Ordinal));
group.Disposition = matchesSnapshot ? FailedRecordDisposition.Eligible : FailedRecordDisposition.Changed;
}
var tokenSource = string.Join("\n", groups.SelectMany(group => group.Items.Select(item =>
$"{item.Id}|{item.TaskId}|{item.VideoId}|{item.UpdatedAt.Ticks}|{(int)group.Disposition}|{group.AwemeId}|{group.Current?.Id}|{(int?)(group.Current?.StorageType)}|{group.Current?.VideoSavePath}")));
var token = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(tokenSource))).ToLowerInvariant();
return new FailedRecordRemovalPlan
{
ActiveMigrationCount = activeMigrationCount,
Groups = groups,
ConfirmationToken = token
};
}
private async Task ChangeStatusAsync(string id, StorageMigrationTaskStatus[] allowed, StorageMigrationTaskStatus status, string error)
{
var task = await RequireTaskAsync(id);
if (!allowed.Contains(task.Status)) throw new InvalidOperationException($"任务状态 {task.Status} 不允许此操作。");
await _db.Updateable<StorageMigrationTask>()
.SetColumns(x => new StorageMigrationTask { Status = status, ErrorMessage = error, UpdatedAt = DateTime.Now })
.Where(x => x.Id == id).ExecuteCommandAsync();
}
private async Task<StorageMigrationTask> RequireTaskAsync(string id) =>
await _db.Queryable<StorageMigrationTask>().InSingleAsync(id) ?? throw new KeyNotFoundException("迁移任务不存在");
private async Task<IMediaStorage> BindTaskTargetAsync(StorageMigrationTask task)
{
if ((task.TargetStorageType ?? MediaStorageType.WebDav) != MediaStorageType.OpenList)
throw new InvalidOperationException("旧 WebDAV 迁移任务只保留审计信息,不能再执行写入、清理或回滚。");
var settings = await _settingsService.GetAsync();
if (!string.Equals(StorageConfigurationFingerprint.Create(settings), task.ConfigurationFingerprint, StringComparison.Ordinal))
throw new InvalidOperationException("OpenList 地址、源挂载、基础目录或账号已变化,请恢复任务原目标后再操作。");
return _openList.Bind(settings);
}
private async Task<List<PlanningEntry>> BuildPlanningAsync()
{
var videos = await _db.Queryable<DouyinVideo>()
.Where(x => x.StorageType != MediaStorageType.OpenList)
.OrderBy(x => x.CreateTime).ToListAsync();
var cookies = (await _db.Queryable<DouyinCookie>().ToListAsync()).ToDictionary(x => x.Id ?? string.Empty, StringComparer.Ordinal);
var categories = (await _db.Queryable<DouyinCollectCate>().ToListAsync()).ToDictionary(x => x.Id ?? string.Empty, StringComparer.Ordinal);
var followed = await _db.Queryable<DouyinFollowed>().ToListAsync();
var config = _commonService.GetConfig() ?? new AppConfig();
var episodes = videos.Where(x => x.ViedoType is VideoTypeEnum.dy_mix or VideoTypeEnum.dy_series)
.GroupBy(x => x.CateId ?? x.CateXId ?? string.Empty)
.SelectMany(group => group.OrderBy(x => x.CreateTime).ThenBy(x => x.Id).Select((video, index) => new { video.Id, Episode = index + 1 }))
.ToDictionary(x => x.Id, x => x.Episode);
var result = new List<PlanningEntry>(videos.Count);
var openListSettings = await _settingsService.GetAsync();
var targetStorage = _openList.Bind(openListSettings);
foreach (var video in videos)
{
cookies.TryGetValue(video.CookieId ?? string.Empty, out var cookie);
categories.TryGetValue(video.CateId ?? string.Empty, out var category);
var follow = followed.FirstOrDefault(x => x.UperId == video.AuthorId);
StorageMigrationPathPlan plan = null;
string planError = null;
try
{
plan = video.StorageType == MediaStorageType.WebDav
? new StorageMigrationPathPlan
{
VideoPath = StoragePath.NormalizeRemote(video.VideoSavePath),
CoverPath = string.IsNullOrWhiteSpace(video.VideoCoverSavePath)
? string.Empty : StoragePath.NormalizeRemote(video.VideoCoverSavePath),
AvatarPath = string.IsNullOrWhiteSpace(video.AuthorAvatar)
? string.Empty : StoragePath.NormalizeRemote(video.AuthorAvatar)
}
: StorageMigrationPathPolicy.Build(video, cookie, category, follow, config,
episodes.GetValueOrDefault(video.Id, 1));
}
catch (Exception ex) { planError = $"视频 {video.AwemeId ?? video.Id}{ex.Message}"; }
var roots = StorageMigrationPathPolicy.GetLocalRoots(cookie);
var readable = SafeLocalMigrationFile.TryResolve(video.VideoSavePath, roots, out var fullPath, out _)
&& new FileInfo(fullPath).Length > 0;
long? existingTargetLength = null;
if (plan != null && video.StorageType == MediaStorageType.WebDav)
{
try { existingTargetLength = await targetStorage.GetLengthAsync(plan.VideoPath); }
catch (Exception ex)
{
planError = $"视频 {video.AwemeId ?? video.Id} OpenList 接管检测失败:{ex.GetBaseException().Message}";
}
}
result.Add(new PlanningEntry
{
Video = video,
Cookie = cookie,
Category = category,
Followed = follow,
Plan = plan,
PlanError = planError,
LocalRoots = roots,
LocalReadable = readable,
LocalLength = readable ? new FileInfo(fullPath).Length : 0,
AdoptExistingTarget = existingTargetLength is > 0
&& (video.FileSize <= 0 || existingTargetLength.Value == video.FileSize)
});
}
return result;
}
private static List<MigrationAttachmentPlan> BuildAttachments(PlanningEntry entry)
{
var attachments = new List<MigrationAttachmentPlan>();
Add(attachments, "cover", entry.Video.VideoCoverSavePath, entry.Video.VideoCoverUrl, entry.Plan.CoverPath, false);
Add(attachments, "avatar", entry.Video.AuthorAvatar, entry.Video.AuthorAvatarUrl, entry.Plan.AvatarPath, true);
if (!string.IsNullOrWhiteSpace(entry.Video.DynamicVideos))
{
try
{
var dynamicItems = JsonConvert.DeserializeObject<List<DouyinMergeVideoDto>>(entry.Video.DynamicVideos) ?? new();
var index = 0;
foreach (var item in dynamicItems)
{
index++;
var oldName = Path.GetFileName(item.Path);
var name = string.IsNullOrWhiteSpace(oldName) ? $"attachment-{index:D3}.bin" : oldName;
Add(attachments, "dynamic", item.Path, Uri.IsWellFormedUriString(item.Path, UriKind.Absolute) ? item.Path : null,
StoragePath.CombineRemote(entry.Plan.DirectoryPath, name), false, item.Height, item.Width);
}
}
catch (JsonException ex)
{
Serilog.Log.Warning(ex, "解析迁移附件失败:{VideoId}", entry.Video.Id);
}
}
return attachments;
}
private static void Add(List<MigrationAttachmentPlan> list, string kind, string source, string sourceUrl, string target, bool shared, int height = 0, int width = 0)
{
if (string.IsNullOrWhiteSpace(target)) return;
list.Add(new MigrationAttachmentPlan { Kind = kind, SourcePath = source, SourceUrl = sourceUrl, TargetPath = target, Shared = shared, Height = height, Width = width });
}
private static StorageMigrationTaskDetail ToDetail(StorageMigrationTask task)
{
var isOpenListTask = task.TargetStorageType == MediaStorageType.OpenList;
return new StorageMigrationTaskDetail
{
Task = task,
StatusText = task.Status switch
{
StorageMigrationTaskStatus.Queued => "排队",
StorageMigrationTaskStatus.Running => "运行中",
StorageMigrationTaskStatus.Paused => "已暂停",
StorageMigrationTaskStatus.Completed => "已完成",
StorageMigrationTaskStatus.PartiallyFailed => "部分失败",
StorageMigrationTaskStatus.Cancelled => "已取消",
StorageMigrationTaskStatus.Cleaning => "清理中",
StorageMigrationTaskStatus.Cleaned => "已清理",
StorageMigrationTaskStatus.RolledBack => "已回滚",
_ => task.Status.ToString()
},
CurrentFile = task.CurrentFile,
CanPause = isOpenListTask && task.Status is (StorageMigrationTaskStatus.Queued or StorageMigrationTaskStatus.Running),
CanResume = isOpenListTask && task.Status == StorageMigrationTaskStatus.Paused,
CanCancel = isOpenListTask && task.Status is (StorageMigrationTaskStatus.Queued or StorageMigrationTaskStatus.Running or StorageMigrationTaskStatus.Paused),
CanRetryFailed = isOpenListTask && task.FailedCount > 0
&& task.Status is (StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled or StorageMigrationTaskStatus.Paused),
CanCleanup = isOpenListTask && task.SuccessCount > 0
&& task.Status is (StorageMigrationTaskStatus.Completed or StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled),
CanRollback = isOpenListTask && task.SuccessCount > 0 && !task.CleanedAt.HasValue
&& task.Status is (StorageMigrationTaskStatus.Completed or StorageMigrationTaskStatus.PartiallyFailed or StorageMigrationTaskStatus.Cancelled),
CanArchive = CanArchive(task)
};
}
private static bool CanArchive(StorageMigrationTask task) => task != null && !task.IsArchived
&& IsTerminal(task.Status)
&& (task.TargetStorageType != MediaStorageType.OpenList
|| task.Status is StorageMigrationTaskStatus.Cleaned or StorageMigrationTaskStatus.RolledBack
|| task.SuccessCount == 0);
private static bool IsTerminal(StorageMigrationTaskStatus status) => status is
StorageMigrationTaskStatus.Completed or StorageMigrationTaskStatus.PartiallyFailed
or StorageMigrationTaskStatus.Cancelled or StorageMigrationTaskStatus.Cleaned
or StorageMigrationTaskStatus.RolledBack;
private static string FormatBytes(long value)
{
string[] units = { "B", "KB", "MB", "GB", "TB" };
var size = (double)value;
var unit = 0;
while (size >= 1024 && unit < units.Length - 1) { size /= 1024; unit++; }
return $"{size:0.##} {units[unit]}";
}
private sealed class PlanningEntry
{
public DouyinVideo Video { get; set; }
public DouyinCookie Cookie { get; set; }
public DouyinCollectCate Category { get; set; }
public DouyinFollowed Followed { get; set; }
public StorageMigrationPathPlan Plan { get; set; }
public string PlanError { get; set; }
public IReadOnlyList<string> LocalRoots { get; set; }
public bool LocalReadable { get; set; }
public long LocalLength { get; set; }
public bool AdoptExistingTarget { get; set; }
}
internal sealed class MigrationAttachmentPlan
{
public string Kind { get; set; }
public string SourcePath { get; set; }
public string SourceUrl { get; set; }
public string TargetPath { get; set; }
public bool Shared { get; set; }
public int Height { get; set; }
public int Width { get; set; }
}
private enum FailedRecordDisposition
{
Eligible,
AlreadyMissing,
Changed,
PermanentlyExcluded,
InvalidSnapshot
}
private sealed class FailedRecordRemovalPlan
{
public int ActiveMigrationCount { get; set; }
public List<FailedRecordGroup> Groups { get; set; } = new();
public string ConfirmationToken { get; set; }
}
private sealed class FailedRecordGroup
{
public string VideoId { get; set; }
public string AwemeId { get; set; }
public DouyinVideo Current { get; set; }
public List<DouyinVideo> Snapshots { get; set; } = new();
public List<StorageMigrationItem> Items { get; set; } = new();
public FailedRecordDisposition Disposition { get; set; }
}
private sealed record CleanupCandidate(string Path, bool Shared);
}
}
+155
View File
@@ -0,0 +1,155 @@
using dy.net.model.dto;
using dy.net.model.entity;
using Microsoft.Extensions.DependencyInjection;
using SqlSugar;
using MediaStorageType = dy.net.model.dto.StorageType;
namespace dy.net.service
{
public class StorageMigrationWorker : BackgroundService
{
private readonly IServiceScopeFactory _scopeFactory;
private readonly ILogger<StorageMigrationWorker> _logger;
public StorageMigrationWorker(IServiceScopeFactory scopeFactory, ILogger<StorageMigrationWorker> logger)
{
_scopeFactory = scopeFactory;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
await RecoverInterruptedAsync();
while (!stoppingToken.IsCancellationRequested)
{
try
{
var worked = await RunOneBatchAsync(stoppingToken);
if (!worked) await Task.Delay(TimeSpan.FromSeconds(2), stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { }
catch (Exception ex)
{
_logger.LogError(ex, "存储迁移后台服务发生错误");
await Task.Delay(TimeSpan.FromSeconds(3), stoppingToken);
}
}
}
private async Task RecoverInterruptedAsync()
{
using var scope = _scopeFactory.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<ISqlSugarClient>();
await db.Updateable<StorageMigrationTask>()
.SetColumns(x => new StorageMigrationTask
{
Status = StorageMigrationTaskStatus.Cancelled,
ErrorMessage = "旧 WebDAV 迁移任务已停止,仅保留审计信息;请创建新的 OpenList 接管任务。",
CurrentFile = null,
CurrentVideoId = null,
CompletedAt = DateTime.Now,
UpdatedAt = DateTime.Now
})
.Where(x => x.TargetStorageType == null
&& (x.Status == StorageMigrationTaskStatus.Queued
|| x.Status == StorageMigrationTaskStatus.Running
|| x.Status == StorageMigrationTaskStatus.Paused
|| x.Status == StorageMigrationTaskStatus.Cleaning))
.ExecuteCommandAsync();
var running = await db.Queryable<StorageMigrationTask>().Where(x =>
x.TargetStorageType == MediaStorageType.OpenList
&& x.Status == StorageMigrationTaskStatus.Running).ToListAsync();
foreach (var task in running)
{
await db.Updateable<StorageMigrationItem>()
.SetColumns(x => new StorageMigrationItem { Stage = StorageMigrationItemStage.Pending, UpdatedAt = DateTime.Now })
.Where(x => x.TaskId == task.Id && (x.Stage == StorageMigrationItemStage.Uploading
|| x.Stage == StorageMigrationItemStage.Verifying || x.Stage == StorageMigrationItemStage.Committing))
.ExecuteCommandAsync();
await db.Updateable<StorageMigrationTask>()
.SetColumns(x => new StorageMigrationTask { Status = StorageMigrationTaskStatus.Queued, CurrentFile = null, CurrentVideoId = null, UpdatedAt = DateTime.Now })
.Where(x => x.Id == task.Id).ExecuteCommandAsync();
}
await db.Updateable<StorageMigrationTask>()
.SetColumns(x => new StorageMigrationTask
{
Status = StorageMigrationTaskStatus.PartiallyFailed,
ErrorMessage = "旧文件清理被应用重启中断,请确认后重试清理。",
UpdatedAt = DateTime.Now
})
.Where(x => x.Status == StorageMigrationTaskStatus.Cleaning)
.ExecuteCommandAsync();
}
private async Task<bool> RunOneBatchAsync(CancellationToken cancellationToken)
{
string taskId;
int concurrency;
List<string> itemIds;
using (var scope = _scopeFactory.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService<ISqlSugarClient>();
var task = await db.Queryable<StorageMigrationTask>()
.Where(x => x.TargetStorageType == MediaStorageType.OpenList
&& (x.Status == StorageMigrationTaskStatus.Queued || x.Status == StorageMigrationTaskStatus.Running))
.OrderBy(x => x.CreatedAt).FirstAsync();
if (task == null) return false;
taskId = task.Id;
concurrency = Math.Clamp(task.Concurrency, 1, 3);
if (task.Status == StorageMigrationTaskStatus.Queued)
{
await db.Updateable<StorageMigrationTask>()
.SetColumns(x => new StorageMigrationTask { Status = StorageMigrationTaskStatus.Running, StartedAt = DateTime.Now, UpdatedAt = DateTime.Now })
.Where(x => x.Id == task.Id && x.Status == StorageMigrationTaskStatus.Queued).ExecuteCommandAsync();
}
itemIds = await db.Queryable<StorageMigrationItem>().Where(x => x.TaskId == task.Id && x.Stage == StorageMigrationItemStage.Pending)
.OrderBy(x => x.CreatedAt).Take(concurrency).Select(x => x.Id).ToListAsync();
}
if (itemIds.Count == 0)
{
await CompleteTaskAsync(taskId);
return true;
}
await Task.WhenAll(itemIds.Select(itemId => ProcessInScopeAsync(taskId, itemId, cancellationToken)));
using (var scope = _scopeFactory.CreateScope())
await scope.ServiceProvider.GetRequiredService<StorageMigrationService>().RefreshCountsAsync(taskId);
return true;
}
private async Task ProcessInScopeAsync(string taskId, string itemId, CancellationToken cancellationToken)
{
using var scope = _scopeFactory.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<ISqlSugarClient>();
var item = await db.Queryable<StorageMigrationItem>().InSingleAsync(itemId);
var videoId = item == null ? null : item.VideoId;
var oldVideoPath = item == null ? null : item.OldVideoPath;
var now = DateTime.Now;
await db.Updateable<StorageMigrationTask>()
.SetColumns(x => new StorageMigrationTask { CurrentVideoId = videoId, CurrentFile = oldVideoPath, UpdatedAt = now })
.Where(x => x.Id == taskId && x.Status == StorageMigrationTaskStatus.Running).ExecuteCommandAsync();
await scope.ServiceProvider.GetRequiredService<StorageMigrationItemProcessor>().ProcessAsync(itemId, cancellationToken);
}
private async Task CompleteTaskAsync(string taskId)
{
using var scope = _scopeFactory.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<ISqlSugarClient>();
var task = await db.Queryable<StorageMigrationTask>().InSingleAsync(taskId);
if (task?.Status != StorageMigrationTaskStatus.Running) return;
var failed = await db.Queryable<StorageMigrationItem>().Where(x => x.TaskId == taskId && x.Stage == StorageMigrationItemStage.Failed).CountAsync();
await db.Updateable<StorageMigrationTask>()
.SetColumns(x => new StorageMigrationTask
{
Status = failed == 0 ? StorageMigrationTaskStatus.Completed : StorageMigrationTaskStatus.PartiallyFailed,
CurrentVideoId = null,
CurrentFile = null,
CompletedAt = DateTime.Now,
UpdatedAt = DateTime.Now
}).Where(x => x.Id == taskId && x.Status == StorageMigrationTaskStatus.Running).ExecuteCommandAsync();
await scope.ServiceProvider.GetRequiredService<StorageMigrationService>().RefreshCountsAsync(taskId);
}
}
}
+124
View File
@@ -0,0 +1,124 @@
using dy.net.model.dto;
using dy.net.model.entity;
using Newtonsoft.Json;
using SqlSugar;
using MediaStorageType = dy.net.model.dto.StorageType;
namespace dy.net.service
{
public class VideoExclusionService
{
private readonly ISqlSugarClient _db;
private readonly VideoTaskService _tasks;
public VideoExclusionService(ISqlSugarClient db, VideoTaskService tasks)
{
_db = db;
_tasks = tasks;
}
public async Task<VideoExclusionPage> GetPageAsync(VideoExclusionPageRequest request)
{
request ??= new VideoExclusionPageRequest();
var page = Math.Max(1, request.PageIndex);
var size = Math.Clamp(request.PageSize, 1, 100);
var query = _db.Queryable<DouyinVideoDelete>()
.WhereIF(!string.IsNullOrWhiteSpace(request.Keyword), x => x.VideoTitle.Contains(request.Keyword)
|| x.ViedoId.Contains(request.Keyword) || x.Author.Contains(request.Keyword));
return new VideoExclusionPage
{
TotalCount = await query.CountAsync(),
Items = await query.OrderByDescending(x => x.DeleteTime).Skip((page - 1) * size).Take(size).ToListAsync()
};
}
public async Task<UnexcludeVideosResult> UnexcludeAsync(UnexcludeVideosRequest request)
{
if (request?.Ids == null || request.Ids.Count == 0) throw new InvalidOperationException("请选择要取消排除的视频。");
var exclusions = await _db.Queryable<DouyinVideoDelete>().Where(x => request.Ids.Contains(x.Id)).ToListAsync();
if (exclusions.Count == 0) throw new KeyNotFoundException("未找到永久排除记录。");
var awemeIds = exclusions.Select(x => x.ViedoId).Where(x => !string.IsNullOrWhiteSpace(x)).Distinct().ToList();
var allDuplicates = awemeIds.Count == 0
? exclusions
: await _db.Queryable<DouyinVideoDelete>().Where(x => awemeIds.Contains(x.ViedoId)).ToListAsync();
var snapshots = request.CreateDownloadTask
? allDuplicates.Select(TryGetSnapshot).Where(x => x != null).GroupBy(x => x.AwemeId).Select(x => x.First()).ToList()
: new List<DouyinVideo>();
VideoDownloadTask task = null;
var result = new UnexcludeVideosResult();
var transaction = await _db.Ado.UseTranAsync(async () =>
{
if (request.CreateDownloadTask && snapshots.Count > 0)
{
task = await _tasks.CreateTaskAsync(VideoTaskType.ExclusionRestore, VideoTaskTrigger.UserAction,
$"取消排除恢复({snapshots.Count} 条)", storageType: snapshots[0].StorageType);
foreach (var video in snapshots)
{
var exclusion = allDuplicates.First(x => x.ViedoId == video.AwemeId);
await _tasks.AddItemAsync(task.Id, video.AwemeId, video.CookieId, null, video.ViedoType,
video.VideoTitle, video.Author, video.VideoSavePath,
new[] { video.VideoUrl }, video, video.FileSize, workerManaged: true);
}
}
await _db.Deleteable<DouyinVideoDelete>()
.Where(x => request.Ids.Contains(x.Id) || awemeIds.Contains(x.ViedoId)).ExecuteCommandAsync();
var now = DateTime.Now;
if (awemeIds.Count > 0)
{
var relatedTaskId = task?.Id;
await _db.Updateable<VideoDownloadTaskItem>()
.SetColumns(x => new VideoDownloadTaskItem
{
ExclusionReleasedAt = now,
RelatedTaskId = relatedTaskId,
UpdatedAt = now
}).Where(x => awemeIds.Contains(x.AwemeId) && x.SkipReason == VideoTaskSkipReason.PermanentlyExcluded)
.ExecuteCommandAsync();
}
});
if (!transaction.IsSuccess) throw new InvalidOperationException("取消排除失败:" + transaction.ErrorMessage);
result.ReleasedCount = awemeIds.Count;
result.QueuedCount = snapshots.Count;
result.TaskId = task?.Id;
result.CannotQueueIds = request.CreateDownloadTask
? awemeIds.Where(id => snapshots.All(x => x.AwemeId != id)).ToList()
: new List<string>();
result.Message = result.CannotQueueIds.Count > 0
? $"已取消排除 {result.ReleasedCount} 条,其中 {result.CannotQueueIds.Count} 条旧记录资料不足,将在下次正常同步时重新发现。"
: task == null
? $"已取消排除 {result.ReleasedCount} 条,将在下次正常同步时重新发现。"
: $"已取消排除并创建 {result.QueuedCount} 条恢复下载任务。";
return result;
}
private static DouyinVideo TryGetSnapshot(DouyinVideoDelete exclusion)
{
if (!string.IsNullOrWhiteSpace(exclusion.RestoreSnapshotJson))
{
try
{
var snapshot = JsonConvert.DeserializeObject<DouyinVideo>(exclusion.RestoreSnapshotJson);
if (snapshot != null && !string.IsNullOrWhiteSpace(snapshot.AwemeId) && !string.IsNullOrWhiteSpace(snapshot.CookieId)) return snapshot;
}
catch { }
}
if (string.IsNullOrWhiteSpace(exclusion.ViedoId) || string.IsNullOrWhiteSpace(exclusion.CookieId)
|| !exclusion.VideoType.HasValue || string.IsNullOrWhiteSpace(exclusion.VideoUrl)) return null;
return new DouyinVideo
{
Id = Guid.NewGuid().ToString("N"),
AwemeId = exclusion.ViedoId,
CookieId = exclusion.CookieId,
ViedoType = exclusion.VideoType.Value,
VideoTitle = exclusion.VideoTitle,
VideoSavePath = exclusion.VideoSavePath,
VideoUrl = exclusion.VideoUrl,
AuthorId = exclusion.AuthorId,
Author = exclusion.Author,
StorageType = MediaStorageType.Local,
SyncTime = DateTime.Now
};
}
}
}
+225
View File
@@ -0,0 +1,225 @@
using dy.net.model.dto;
using dy.net.model.entity;
using dy.net.storage;
using Microsoft.Extensions.DependencyInjection;
using SqlSugar;
using MediaStorageType = dy.net.model.dto.StorageType;
namespace dy.net.service
{
/// <summary>
/// Replaces only the main media after a complete new download. The old row and file remain valid on failure.
/// </summary>
public class VideoRedownloadWorker : BackgroundService
{
private readonly IServiceScopeFactory _scopeFactory;
public VideoRedownloadWorker(IServiceScopeFactory scopeFactory) => _scopeFactory = scopeFactory;
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
try
{
using var scope = _scopeFactory.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<ISqlSugarClient>();
var waitingTaskIds = await db.Queryable<VideoDownloadTask>()
.Where(x => x.Status == VideoTaskStatus.WaitingForStorage)
.Select(x => x.Id).ToListAsync();
var pendingJobs = await db.Queryable<DouyinReDownload>()
.Where(x => x.Status == 0).OrderBy(x => x.CreateTime).ToListAsync();
var job = pendingJobs.FirstOrDefault(x => string.IsNullOrWhiteSpace(x.TaskId)
|| !waitingTaskIds.Contains(x.TaskId));
if (job == null)
{
await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken);
continue;
}
await ProcessAsync(scope.ServiceProvider, job, stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { }
catch (Exception ex)
{
Serilog.Log.Error(ex, "安全重新下载后台任务异常");
await Task.Delay(TimeSpan.FromSeconds(3), stoppingToken);
}
}
}
private static async Task ProcessAsync(IServiceProvider services, DouyinReDownload job, CancellationToken cancellationToken)
{
var db = services.GetRequiredService<ISqlSugarClient>();
var tasks = services.GetRequiredService<VideoTaskService>();
if (!string.IsNullOrWhiteSpace(job.TaskId))
{
var parent = await db.Queryable<VideoDownloadTask>().InSingleAsync(job.TaskId);
if (parent?.Status is VideoTaskStatus.WaitingForStorage or VideoTaskStatus.WaitingForSource)
{
if (parent.Status == VideoTaskStatus.WaitingForStorage) return;
}
try
{
await tasks.EnsureTaskStorageAvailableAsync(job.TaskId);
}
catch (InvalidOperationException ex)
{
if (!string.IsNullOrWhiteSpace(job.TaskItemId))
{
var blockedItem = await db.Queryable<VideoDownloadTaskItem>().InSingleAsync(job.TaskItemId);
if (blockedItem != null)
{
blockedItem.Stage = VideoTaskItemStage.WaitingForStorage;
blockedItem.ErrorType = VideoTaskErrorType.StorageUnavailable;
blockedItem.ErrorMessage = ex.Message;
blockedItem.UpdatedAt = DateTime.Now;
await db.Updateable(blockedItem).ExecuteCommandAsync();
}
}
await tasks.BlockTaskForStorageAsync(job.TaskId, ex.Message);
await tasks.RefreshCountsAsync(job.TaskId);
return;
}
}
var http = services.GetRequiredService<DouyinHttpClientService>();
var resolver = services.GetRequiredService<DouyinMigrationSourceResolver>();
var video = !string.IsNullOrWhiteSpace(job.VideoRecordId)
? await db.Queryable<DouyinVideo>().InSingleAsync(job.VideoRecordId)
: await db.Queryable<DouyinVideo>().Where(x => x.AwemeId == job.ViedoId).FirstAsync();
if (video == null)
{
await FinishAsync(db, job, false, "原视频记录不存在");
if (!string.IsNullOrWhiteSpace(job.TaskItemId))
await tasks.MarkFailedAsync(job.TaskItemId, new InvalidOperationException("原视频记录不存在"), VideoTaskErrorType.SourceUnavailable);
await CompleteTaskWhenIdleAsync(db, tasks, job.TaskId);
return;
}
var cookie = await db.Queryable<DouyinCookie>().InSingleAsync(video.CookieId);
if (!string.IsNullOrWhiteSpace(job.TaskItemId))
{
var tracked = await db.Queryable<VideoDownloadTaskItem>().InSingleAsync(job.TaskItemId);
var access = tracked?.Stage == VideoTaskItemStage.WaitingForSource
? await tasks.TryAcquireSourceAsync(video.CookieId, tracked.Id)
: await tasks.GetSourceAccessAsync(video.CookieId);
if (!access.Allowed)
{
await tasks.WaitItemForSourceAsync(job.TaskItemId, video.CookieId, access.Message);
return;
}
if (tracked?.Stage == VideoTaskItemStage.WaitingForSource)
{
tracked.Stage = VideoTaskItemStage.Pending;
tracked.ErrorMessage = null;
tracked.ErrorType = VideoTaskErrorType.None;
tracked.UpdatedAt = DateTime.Now;
await db.Updateable(tracked).ExecuteCommandAsync();
}
}
if (!string.IsNullOrWhiteSpace(job.TaskId)) await tasks.StartTaskAsync(job.TaskId);
if (!string.IsNullOrWhiteSpace(job.TaskItemId))
await tasks.SetItemStageAsync(job.TaskItemId, VideoTaskItemStage.Downloading, job.SavePath);
var urls = new List<string>();
if (!string.IsNullOrWhiteSpace(video.VideoUrl)) urls.Add(video.VideoUrl);
foreach (var url in await resolver.ResolveAsync(video, cookie, cancellationToken))
if (!urls.Contains(url)) urls.Add(url);
if (urls.Count == 0)
{
await FinishAsync(db, job, false, "记录 URL 与抖音全量列表均无可用地址");
if (!string.IsNullOrWhiteSpace(job.TaskItemId))
await tasks.MarkFailedAsync(job.TaskItemId, new InvalidOperationException("记录 URL 与抖音全量列表均无可用地址"), VideoTaskErrorType.SourceUnavailable);
await CompleteTaskWhenIdleAsync(db, tasks, job.TaskId);
return;
}
job.Attempts++;
job.UpdateTime = DateTime.UtcNow;
await db.Updateable(job).ExecuteCommandAsync();
try
{
long length;
if (video.StorageType == MediaStorageType.Local)
{
var target = Path.GetFullPath(video.VideoSavePath);
var directory = Path.GetDirectoryName(target) ?? throw new InvalidOperationException("原视频目录无效");
Directory.CreateDirectory(directory);
var temporary = Path.Combine(directory, "." + Path.GetFileName(target) + ".redownload-" + Guid.NewGuid().ToString("N"));
try
{
var downloaded = await http.DownloadAsync(urls[0], temporary, cookie?.Cookies, urls.Skip(1).ToList(), cancellationToken, maxRetryCount: Math.Min(12, urls.Count));
if (!downloaded.Success)
{
var disposition = SourceFailureDisposition.Continue;
if (!string.IsNullOrWhiteSpace(job.TaskItemId))
disposition = await tasks.MarkDownloadFailedAsync(job.TaskItemId, video.CookieId, downloaded);
if (disposition == SourceFailureDisposition.Continue)
await FinishAsync(db, job, false, downloaded.Message);
await CompleteTaskWhenIdleAsync(db, tasks, job.TaskId);
return;
}
await tasks.RecordSourceSuccessAsync(video.CookieId);
if (!File.Exists(downloaded.ActualSavePath)) throw new MediaStorageException("重新下载完成后临时文件不存在。");
length = new FileInfo(downloaded.ActualSavePath).Length;
if (length <= 0) throw new IOException("重新下载文件为空");
File.Move(downloaded.ActualSavePath, target, true);
}
finally
{
if (File.Exists(temporary)) File.Delete(temporary);
}
}
else
{
var downloaded = await http.DownloadToStorageAsync(video.StorageType, urls[0], video.VideoSavePath,
cookie?.Cookies, urls.Skip(1).ToList(), cancellationToken, Math.Min(12, urls.Count));
if (!downloaded.Success)
{
var disposition = SourceFailureDisposition.Continue;
if (!string.IsNullOrWhiteSpace(job.TaskItemId))
disposition = await tasks.MarkDownloadFailedAsync(job.TaskItemId, video.CookieId, downloaded);
if (disposition == SourceFailureDisposition.Continue)
await FinishAsync(db, job, false, downloaded.Message);
await CompleteTaskWhenIdleAsync(db, tasks, job.TaskId);
return;
}
await tasks.RecordSourceSuccessAsync(video.CookieId);
length = await services.GetRequiredService<MediaStorageRouter>()
.Resolve(video.StorageType).GetLengthAsync(video.VideoSavePath, cancellationToken) ?? 0;
if (length <= 0) throw new IOException($"{video.StorageType} 替换后的文件为空");
}
if (!string.IsNullOrWhiteSpace(job.TaskItemId))
await tasks.SetItemStageAsync(job.TaskItemId, VideoTaskItemStage.Verifying, video.VideoSavePath);
if (length <= 0) throw new IOException("重新下载后的文件为空");
if (!string.IsNullOrWhiteSpace(job.TaskItemId))
await tasks.SetItemStageAsync(job.TaskItemId, VideoTaskItemStage.Committing, video.VideoSavePath);
video.FileSize = length;
video.SyncTime = DateTime.Now;
await db.Updateable(video).ExecuteCommandAsync();
await FinishAsync(db, job, true, null);
if (!string.IsNullOrWhiteSpace(job.TaskItemId)) await tasks.MarkSucceededAsync(job.TaskItemId, length, video.Id);
}
catch (Exception ex)
{
Serilog.Log.Warning(ex, "安全重新下载失败,旧记录和旧文件保持不变:{VideoId}", video.Id);
await FinishAsync(db, job, false, ex.Message);
if (!string.IsNullOrWhiteSpace(job.TaskItemId)) await tasks.MarkFailedAsync(job.TaskItemId, ex);
}
await CompleteTaskWhenIdleAsync(db, tasks, job.TaskId);
}
private static async Task CompleteTaskWhenIdleAsync(ISqlSugarClient db, VideoTaskService tasks, string taskId)
{
if (string.IsNullOrWhiteSpace(taskId)) return;
var pending = await db.Queryable<DouyinReDownload>().Where(x => x.TaskId == taskId && x.Status == 0).CountAsync();
if (pending == 0) await tasks.CompleteTaskAsync(taskId);
}
private static async Task FinishAsync(ISqlSugarClient db, DouyinReDownload job, bool success, string error)
{
job.Status = success ? 1 : 2;
job.ErrorMessage = error;
job.UpdateTime = DateTime.UtcNow;
if (success) job.DownTime = DateTime.UtcNow;
await db.Updateable(job).ExecuteCommandAsync();
}
}
}
File diff suppressed because it is too large Load Diff
+222
View File
@@ -0,0 +1,222 @@
using dy.net.model.dto;
using dy.net.model.entity;
using dy.net.storage;
using Microsoft.Extensions.DependencyInjection;
using Newtonsoft.Json;
using SqlSugar;
namespace dy.net.service
{
public class VideoTaskWorker : BackgroundService
{
private readonly IServiceScopeFactory _scopeFactory;
public VideoTaskWorker(IServiceScopeFactory scopeFactory) => _scopeFactory = scopeFactory;
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
using (var scope = _scopeFactory.CreateScope())
{
var tasks = scope.ServiceProvider.GetRequiredService<VideoTaskService>();
await tasks.RecoverInterruptedAsync();
await BackfillLegacyRedownloadsAsync(scope.ServiceProvider);
await tasks.CleanupHistoryAsync();
}
var lastCleanup = DateTime.UtcNow;
while (!stoppingToken.IsCancellationRequested)
{
try
{
var worked = await ProcessOneAsync(stoppingToken);
if (DateTime.UtcNow - lastCleanup > TimeSpan.FromDays(1))
{
using var scope = _scopeFactory.CreateScope();
await scope.ServiceProvider.GetRequiredService<VideoTaskService>().CleanupHistoryAsync();
lastCleanup = DateTime.UtcNow;
}
if (!worked) await Task.Delay(TimeSpan.FromSeconds(2), stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { }
catch (Exception ex)
{
Serilog.Log.Error(ex, "视频任务后台服务异常");
await Task.Delay(TimeSpan.FromSeconds(3), stoppingToken);
}
}
}
private async Task<bool> ProcessOneAsync(CancellationToken cancellationToken)
{
using var scope = _scopeFactory.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<ISqlSugarClient>();
var tasks = scope.ServiceProvider.GetRequiredService<VideoTaskService>();
var candidates = await db.Queryable<VideoDownloadTaskItem>()
.Where(x => x.WorkerManaged && (x.Stage == VideoTaskItemStage.Pending
|| x.Stage == VideoTaskItemStage.WaitingForSource))
.OrderBy(x => x.CreatedAt).Take(50).ToListAsync();
VideoDownloadTaskItem item = null;
var isSourceProbe = false;
foreach (var candidate in candidates)
{
var access = candidate.Stage == VideoTaskItemStage.WaitingForSource
? await tasks.TryAcquireSourceAsync(candidate.CookieId, candidate.Id)
: await tasks.GetSourceAccessAsync(candidate.CookieId);
if (access.Allowed)
{
item = candidate;
isSourceProbe = access.IsProbe;
if (item.Stage == VideoTaskItemStage.WaitingForSource)
{
item.Stage = VideoTaskItemStage.Pending;
item.ErrorMessage = null;
item.ErrorType = VideoTaskErrorType.None;
item.RetryAfter = null;
item.UpdatedAt = DateTime.Now;
await db.Updateable(item).ExecuteCommandAsync();
}
break;
}
if (candidate.Stage == VideoTaskItemStage.Pending)
await tasks.WaitItemForSourceAsync(candidate.Id, candidate.CookieId, access.Message);
}
if (item == null) return false;
var task = await db.Queryable<VideoDownloadTask>().InSingleAsync(item.TaskId);
if (task == null || task.Type == VideoTaskType.Redownload) return false;
try
{
await tasks.EnsureTaskStorageAvailableAsync(task.Id);
}
catch (InvalidOperationException ex)
{
item.Stage = VideoTaskItemStage.WaitingForStorage;
item.ErrorType = VideoTaskErrorType.StorageUnavailable;
item.ErrorMessage = ex.Message;
item.UpdatedAt = DateTime.Now;
await db.Updateable(item).ExecuteCommandAsync();
await tasks.BlockTaskForStorageAsync(task.Id, ex.Message);
await tasks.RefreshCountsAsync(task.Id);
return true;
}
await tasks.StartTaskAsync(task.Id);
try
{
if (string.IsNullOrWhiteSpace(item.RetrySnapshotJson))
throw new InvalidOperationException("该失败条目缺少可恢复快照,请重新执行对应的同步任务。");
var video = JsonConvert.DeserializeObject<DouyinVideo>(item.RetrySnapshotJson)
?? throw new InvalidOperationException("任务缺少视频恢复快照。");
var cookie = await db.Queryable<DouyinCookie>().InSingleAsync(video.CookieId)
?? throw new InvalidOperationException("任务所属抖音授权不存在。");
if (string.IsNullOrWhiteSpace(cookie.Cookies)) throw new InvalidOperationException("Cookie 无效,无法恢复下载。");
var storedUrls = DeserializeUrls(item.SourceUrlsJson);
if (!string.IsNullOrWhiteSpace(video.VideoUrl) && !storedUrls.Contains(video.VideoUrl)) storedUrls.Insert(0, video.VideoUrl);
var resolvedUrls = (await scope.ServiceProvider.GetRequiredService<DouyinMigrationSourceResolver>()
.ResolveAsync(video, cookie, cancellationToken)).ToList();
var urls = (isSourceProbe ? resolvedUrls.Concat(storedUrls) : storedUrls.Concat(resolvedUrls))
.Where(x => !string.IsNullOrWhiteSpace(x)).Distinct(StringComparer.Ordinal).Take(12).ToList();
if (urls.Count == 0) throw new InvalidOperationException("作品没有可用下载地址,可能已下架或设为私密。");
await tasks.SetItemStageAsync(item.Id, VideoTaskItemStage.Downloading, video.VideoSavePath);
var storage = scope.ServiceProvider.GetRequiredService<MediaStorageRouter>().Resolve(video.StorageType);
var existingLength = await storage.GetLengthAsync(video.VideoSavePath, cancellationToken);
if (!existingLength.HasValue || existingLength <= 0)
{
var downloaded = await scope.ServiceProvider.GetRequiredService<DouyinHttpClientService>()
.DownloadToStorageAsync(video.StorageType, urls[0], video.VideoSavePath, cookie.Cookies,
urls.Skip(1).ToList(), cancellationToken, Math.Min(12, urls.Count));
if (!downloaded.Success)
{
await tasks.MarkDownloadFailedAsync(item.Id, cookie.Id, downloaded);
await CompleteWhenIdleAsync(db, tasks, task.Id);
return true;
}
video.VideoSavePath = downloaded.ActualSavePath;
await tasks.RecordSourceSuccessAsync(cookie.Id);
}
await tasks.SetItemStageAsync(item.Id, VideoTaskItemStage.Verifying, video.VideoSavePath);
var length = await storage.GetLengthAsync(video.VideoSavePath, cancellationToken) ?? 0;
if (length <= 0) throw new IOException("下载后的主媒体不存在或为空。");
if (item.ExpectedLength > 0 && length != item.ExpectedLength)
throw new IOException($"下载长度不一致:期望 {item.ExpectedLength},实际 {length}。");
await tasks.SetItemStageAsync(item.Id, VideoTaskItemStage.Committing, video.VideoSavePath);
video.FileSize = length;
video.SyncTime = DateTime.Now;
if (string.IsNullOrWhiteSpace(video.Id)) video.Id = Guid.NewGuid().ToString("N");
await db.Storageable(video).ExecuteCommandAsync();
var warning = "恢复任务仅保证主媒体,封面、头像或 NFO 将在后续同步补齐。";
var cleanupPending = false;
string cleanupError = null;
if (video.SupersededStorageSnapshot != null)
{
try
{
await scope.ServiceProvider.GetRequiredService<DouyinVideoService>()
.CleanupSupersededLocalArtifactsAsync(video.SupersededStorageSnapshot, video.VideoSavePath);
}
catch (Exception cleanupException)
{
cleanupPending = true;
cleanupError = cleanupException.GetBaseException().Message;
warning += $" 旧本地文件清理失败,可在确认远端文件后稍后重试:{cleanupError}";
}
}
await tasks.MarkSucceededAsync(item.Id, length, video.Id, warning, cleanupPending, cleanupError);
await CompleteWhenIdleAsync(db, tasks, task.Id);
}
catch (Exception ex)
{
await tasks.MarkFailedAsync(item.Id, ex);
await CompleteWhenIdleAsync(db, tasks, task.Id);
}
return true;
}
private static async Task CompleteWhenIdleAsync(ISqlSugarClient db, VideoTaskService tasks, string taskId)
{
var remaining = await db.Queryable<VideoDownloadTaskItem>().Where(x => x.TaskId == taskId
&& (x.Stage == VideoTaskItemStage.Pending || x.Stage == VideoTaskItemStage.Downloading
|| x.Stage == VideoTaskItemStage.Verifying || x.Stage == VideoTaskItemStage.Committing
|| x.Stage == VideoTaskItemStage.WaitingForSource)).CountAsync();
if (remaining == 0) await tasks.CompleteTaskAsync(taskId);
}
private static List<string> DeserializeUrls(string json)
{
try { return JsonConvert.DeserializeObject<List<string>>(json) ?? new(); }
catch { return new(); }
}
private static async Task BackfillLegacyRedownloadsAsync(IServiceProvider services)
{
var db = services.GetRequiredService<ISqlSugarClient>();
var legacy = await db.Queryable<DouyinReDownload>().Where(x => string.IsNullOrEmpty(x.TaskId)).ToListAsync();
if (legacy.Count == 0) return;
var tasks = services.GetRequiredService<VideoTaskService>();
var task = await tasks.CreateTaskAsync(VideoTaskType.Redownload, VideoTaskTrigger.UpgradeRecovery,
$"升级前重新下载({legacy.Count} 条)");
foreach (var job in legacy)
{
var video = !string.IsNullOrWhiteSpace(job.VideoRecordId)
? await db.Queryable<DouyinVideo>().InSingleAsync(job.VideoRecordId)
: await db.Queryable<DouyinVideo>().Where(x => x.AwemeId == job.ViedoId).FirstAsync();
var item = await tasks.AddItemAsync(task.Id, job.ViedoId, job.CookieId, null, video?.ViedoType,
video?.VideoTitle, video?.Author, job.SavePath, new[] { video?.VideoUrl }, video, video?.FileSize ?? 0);
item.Stage = job.Status switch
{
1 => VideoTaskItemStage.Succeeded,
2 => VideoTaskItemStage.Failed,
_ => VideoTaskItemStage.Pending
};
item.ErrorMessage = job.ErrorMessage;
item.ErrorType = job.Status == 2 ? VideoTaskErrorType.Unknown : VideoTaskErrorType.None;
item.CompletedAt = job.Status == 0 ? null : job.UpdateTime;
item.UpdatedAt = job.UpdateTime == default ? DateTime.Now : job.UpdateTime;
await db.Updateable(item).ExecuteCommandAsync();
job.TaskId = task.Id;
job.TaskItemId = item.Id;
}
await db.Updateable(legacy).ExecuteCommandAsync();
await tasks.RefreshCountsAsync(task.Id);
var pending = legacy.Count(x => x.Status == 0);
if (pending == 0) await tasks.CompleteTaskAsync(task.Id);
}
}
}
+107
View File
@@ -0,0 +1,107 @@
using dy.net.model.dto;
using dy.net.model.entity;
using Microsoft.AspNetCore.DataProtection;
using SqlSugar;
using dy.net.utils;
using MediaStorageType = dy.net.model.dto.StorageType;
namespace dy.net.service
{
public class WebDavSettingsService
{
private const string SettingsId = "default";
private readonly ISqlSugarClient _db;
private readonly IDataProtector _protector;
public WebDavSettingsService(ISqlSugarClient db, IDataProtectionProvider provider)
{
_db = db;
_protector = provider.CreateProtector("dysync.webdav.password.v1");
}
public async Task<WebDavSettings> GetAsync()
{
return await _db.Queryable<WebDavSettings>().InSingleAsync(SettingsId)
?? new WebDavSettings { Id = SettingsId, BasePath = "/dysync" };
}
public async Task<string> GetPasswordAsync(WebDavSettings settings = null)
{
settings ??= await GetAsync();
if (string.IsNullOrWhiteSpace(settings.ProtectedPassword)) return string.Empty;
try
{
return _protector.Unprotect(settings.ProtectedPassword);
}
catch (Exception ex)
{
Serilog.Log.Error(ex, "WebDAV 密码解密失败,请重新输入密码");
return string.Empty;
}
}
public async Task<WebDavSettings> BuildCandidateAsync(WebDavTestRequest request)
{
request ??= new WebDavTestRequest();
var existing = await GetAsync();
var password = string.IsNullOrWhiteSpace(request.Password)
? await GetPasswordAsync(existing)
: request.Password;
return new WebDavSettings
{
Id = SettingsId,
Endpoint = string.IsNullOrWhiteSpace(request.Endpoint) ? existing.Endpoint : request.Endpoint.Trim(),
BasePath = string.IsNullOrWhiteSpace(request.BasePath) ? existing.BasePath : request.BasePath.Trim(),
UserName = string.IsNullOrWhiteSpace(request.UserName) ? existing.UserName : request.UserName.Trim(),
ProtectedPassword = string.IsNullOrWhiteSpace(password) ? null : _protector.Protect(password),
AllowInvalidCertificate = request.AllowInvalidCertificate
};
}
public async Task SaveAsync(WebDavSettings settings, bool tested, string testMessage)
{
settings.Id = SettingsId;
settings.LastTestedAt = tested ? DateTime.Now : null;
settings.LastTestMessage = testMessage;
await _db.Storageable(settings).ExecuteCommandAsync();
}
public async Task<WebDavSettingsDto> ToDtoAsync(MediaStorageType storageType)
{
var settings = await GetAsync();
return new WebDavSettingsDto
{
StorageType = storageType,
Endpoint = settings.Endpoint,
BasePath = settings.BasePath,
UserName = settings.UserName,
HasPassword = !string.IsNullOrWhiteSpace(settings.ProtectedPassword),
AllowInvalidCertificate = settings.AllowInvalidCertificate,
LastTestedAt = settings.LastTestedAt,
LastTestMessage = settings.LastTestMessage
};
}
public async Task InitializeCookiePathsAsync()
{
var cookies = await _db.Queryable<DouyinCookie>().ToListAsync();
foreach (var cookie in cookies)
{
ApplyDefaultCookiePaths(cookie);
}
if (cookies.Count > 0) await _db.Updateable(cookies).ExecuteCommandAsync();
}
public void ApplyDefaultCookiePaths(DouyinCookie cookie)
{
if (cookie == null) return;
var root = DouyinFileNameHelper.SanitizeLinuxFileName(cookie.UserName, cookie.Id, true);
if (string.IsNullOrWhiteSpace(cookie.WebDavCollectPath)) cookie.WebDavCollectPath = $"/{root}/collect";
if (string.IsNullOrWhiteSpace(cookie.WebDavFavoritePath)) cookie.WebDavFavoritePath = $"/{root}/favorite";
if (string.IsNullOrWhiteSpace(cookie.WebDavFollowPath)) cookie.WebDavFollowPath = $"/{root}/follow";
if (string.IsNullOrWhiteSpace(cookie.WebDavMixPath)) cookie.WebDavMixPath = $"/{root}/mix";
if (string.IsNullOrWhiteSpace(cookie.WebDavSeriesPath)) cookie.WebDavSeriesPath = $"/{root}/series";
}
}
}