feat: add fnOS packaging, storage workflows and release pipeline

This commit is contained in:
2026-08-11 18:05:49 +08:00
parent c5922f9b08
commit 95932f0199
181 changed files with 24024 additions and 1164 deletions
+305 -62
View File
@@ -5,6 +5,9 @@ using dy.net.repository;
using dy.net.utils;
using Serilog;
using SqlSugar;
using dy.net.storage;
using Newtonsoft.Json;
using MediaStorageType = dy.net.model.dto.StorageType;
namespace dy.net.service
{
@@ -15,12 +18,45 @@ namespace dy.net.service
private readonly DouyinVideoRepository _dyCollectVideoRepository;
private readonly DouyinCookieRepository douyinCookieRepository;
private readonly MediaStorageRouter _storageRouter;
private readonly VideoTaskService _videoTasks;
public DouyinVideoService(DouyinVideoRepository dyCollectVideoRepository, DouyinCookieRepository douyinCookieRepository, ISqlSugarClient sqlSugarClient)
public DouyinVideoService(DouyinVideoRepository dyCollectVideoRepository, DouyinCookieRepository douyinCookieRepository, ISqlSugarClient sqlSugarClient, MediaStorageRouter storageRouter, VideoTaskService videoTasks)
{
_dyCollectVideoRepository = dyCollectVideoRepository;
this.douyinCookieRepository = douyinCookieRepository;
this.sqlSugarClient = sqlSugarClient;
_storageRouter = storageRouter;
_videoTasks = videoTasks;
}
public async Task<StorageRecordInventory> GetStorageInventoryAsync(MediaStorageType currentStorageType)
{
var records = await sqlSugarClient.Queryable<DouyinVideo>()
.Select(x => new DouyinVideo { StorageType = x.StorageType, FileSize = x.FileSize })
.ToListAsync();
var local = records.Where(x => x.StorageType == MediaStorageType.Local).ToList();
var webDav = records.Where(x => x.StorageType == MediaStorageType.WebDav).ToList();
var openList = records.Where(x => x.StorageType == MediaStorageType.OpenList).ToList();
var currentCount = currentStorageType switch
{
MediaStorageType.WebDav => webDav.Count,
MediaStorageType.OpenList => openList.Count,
_ => local.Count
};
return new StorageRecordInventory
{
CurrentStorageType = currentStorageType,
TotalRecordCount = records.Count,
LocalRecordCount = local.Count,
WebDavRecordCount = webDav.Count,
OpenListRecordCount = openList.Count,
CurrentStorageRecordCount = currentCount,
OtherStorageRecordCount = records.Count - currentCount,
LocalDeclaredBytes = local.Sum(x => x.FileSize),
WebDavDeclaredBytes = webDav.Sum(x => x.FileSize),
OpenListDeclaredBytes = openList.Sum(x => x.FileSize)
};
}
@@ -76,9 +112,23 @@ namespace dy.net.service
{
if (updateMap.TryGetValue(existingVideo.AwemeId, out var updateData))
{
// 保留原记录主键,并把实际主键回写给任务条目使用。
updateData.Id = existingVideo.Id;
existingVideo.VideoSavePath = updateData.VideoSavePath;
existingVideo.VideoCoverSavePath = updateData.VideoCoverSavePath;
existingVideo.ViedoType = updateData.ViedoType;
existingVideo.StorageType = updateData.StorageType;
existingVideo.FileSize = updateData.FileSize;
existingVideo.FileHash = updateData.FileHash;
existingVideo.Resolution = updateData.Resolution;
existingVideo.DynamicVideos = updateData.DynamicVideos;
existingVideo.OnlyImgOrOnlyMp3 = updateData.OnlyImgOrOnlyMp3;
existingVideo.IsMergeVideo = updateData.IsMergeVideo;
existingVideo.VideoUrl = updateData.VideoUrl;
existingVideo.AuthorAvatar = updateData.AuthorAvatar;
existingVideo.CateId = updateData.CateId;
existingVideo.CateXId = updateData.CateXId;
existingVideo.SyncTime = updateData.SyncTime;
}
}
// 批量更新数据库
@@ -98,6 +148,167 @@ namespace dy.net.service
return transaction;
}
/// <summary>
/// 普通同步已经把同一作品提交到远端存储后,安全清理不再被本地记录引用的旧文件。
/// 数据库或远端主媒体未验证通过时不会删除任何本地文件。
/// </summary>
public async Task<int> CleanupSupersededLocalArtifactsAsync(DouyinVideo snapshot, string replacementPath)
{
if (snapshot == null || snapshot.StorageType != MediaStorageType.Local
|| string.IsNullOrWhiteSpace(snapshot.AwemeId) || string.IsNullOrWhiteSpace(replacementPath)) return 0;
var current = await sqlSugarClient.Queryable<DouyinVideo>()
.Where(x => x.AwemeId == snapshot.AwemeId).FirstAsync();
if (current == null || !current.StorageType.IsRemote()
|| !string.Equals(current.VideoSavePath, replacementPath, StringComparison.Ordinal))
throw new InvalidOperationException("新存储记录尚未提交,保留旧本地文件。");
var remoteLength = await _storageRouter.Resolve(current.StorageType).GetLengthAsync(replacementPath);
if (!remoteLength.HasValue || remoteLength.Value <= 0)
throw new InvalidOperationException("新存储主媒体不存在或为空,保留旧本地文件。");
var cookie = await sqlSugarClient.Queryable<DouyinCookie>().InSingleAsync(snapshot.CookieId);
var roots = StorageMigrationPathPolicy.GetLocalRoots(cookie);
if (roots.Count == 0) throw new InvalidOperationException("旧存储根目录不可用,保留旧本地文件。");
var localVideos = await sqlSugarClient.Queryable<DouyinVideo>()
.Where(x => x.StorageType == MediaStorageType.Local).ToListAsync();
var deleted = 0;
foreach (var candidate in BuildLocalCleanupCandidates(snapshot))
{
if (!File.Exists(candidate.Path)) continue;
if (!SafeLocalMigrationFile.TryResolve(candidate.Path, roots, out var safePath, out var error))
throw new InvalidOperationException($"拒绝清理不安全旧路径 {candidate.Path}{error}");
if (IsLocalPathReferenced(candidate.Path, localVideos, snapshot.Id, candidate.Shared)) continue;
File.Delete(safePath);
deleted++;
}
DeleteEmptyParents(snapshot.VideoSavePath, roots);
return deleted;
}
public async Task<int> RetryTaskItemLocalCleanupAsync(string taskId, string itemId)
{
var task = await sqlSugarClient.Queryable<VideoDownloadTask>().InSingleAsync(taskId)
?? throw new KeyNotFoundException("任务不存在。");
if (task.Type != VideoTaskType.Sync)
throw new InvalidOperationException("仅普通同步任务支持重试旧本地文件清理。");
var item = await sqlSugarClient.Queryable<VideoDownloadTaskItem>().InSingleAsync(itemId)
?? throw new KeyNotFoundException("任务条目不存在。");
if (item.TaskId != taskId || !VideoTaskService.CanRetryCleanup(item))
throw new InvalidOperationException("该条目当前没有可重试的旧本地文件清理。");
DouyinVideo replacement;
try
{
replacement = JsonConvert.DeserializeObject<DouyinVideo>(item.RetrySnapshotJson);
}
catch (JsonException ex)
{
throw new InvalidOperationException("旧本地记录快照损坏,无法安全重试清理。", ex);
}
var snapshot = replacement?.SupersededStorageSnapshot;
if (snapshot == null || snapshot.StorageType != MediaStorageType.Local)
throw new InvalidOperationException("任务没有可验证的旧本地记录快照,拒绝清理。");
var current = !string.IsNullOrWhiteSpace(item.VideoId)
? await sqlSugarClient.Queryable<DouyinVideo>().InSingleAsync(item.VideoId)
: null;
current ??= await sqlSugarClient.Queryable<DouyinVideo>()
.Where(x => x.AwemeId == item.AwemeId).FirstAsync();
if (current == null)
throw new InvalidOperationException("当前视频记录不存在,拒绝清理旧文件。");
try
{
var deleted = await CleanupSupersededLocalArtifactsAsync(snapshot, current.VideoSavePath);
item.CleanupPending = false;
item.CleanupError = null;
item.WarningMessage = RemoveCleanupWarning(item.WarningMessage);
item.Stage = string.IsNullOrWhiteSpace(item.WarningMessage)
? VideoTaskItemStage.Succeeded
: VideoTaskItemStage.SucceededWithWarnings;
item.UpdatedAt = DateTime.Now;
await sqlSugarClient.Updateable(item).ExecuteCommandAsync();
await _videoTasks.RefreshCountsAsync(taskId);
return deleted;
}
catch (Exception ex)
{
var reason = ex.GetBaseException().Message;
item.CleanupPending = true;
item.CleanupError = reason;
item.UpdatedAt = DateTime.Now;
await sqlSugarClient.Updateable(item).ExecuteCommandAsync();
await _videoTasks.RefreshCountsAsync(taskId);
throw new InvalidOperationException($"旧本地文件清理仍失败:{reason}", ex);
}
}
internal static string RemoveCleanupWarning(string warning)
{
if (string.IsNullOrWhiteSpace(warning)) return null;
var failureIndex = warning.IndexOf("旧本地文件清理失败", StringComparison.Ordinal);
if (failureIndex < 0) return warning.Trim();
var start = warning.LastIndexOf(";已切换到", failureIndex, StringComparison.Ordinal);
if (start >= 0) return warning[..start].Trim().TrimEnd('');
start = warning.LastIndexOf("已切换到", failureIndex, StringComparison.Ordinal);
if (start == 0) return null;
return warning[..failureIndex].Trim().TrimEnd('', '', ',');
}
private static List<LocalCleanupCandidate> BuildLocalCleanupCandidates(DouyinVideo snapshot)
{
var result = new List<LocalCleanupCandidate>
{
new(snapshot.VideoSavePath, false),
new(snapshot.VideoCoverSavePath, snapshot.ViedoType is VideoTypeEnum.dy_mix or VideoTypeEnum.dy_series),
new(snapshot.AuthorAvatar, true)
};
if (!string.IsNullOrWhiteSpace(snapshot.VideoSavePath))
{
var directory = Path.GetDirectoryName(snapshot.VideoSavePath);
var basename = Path.GetFileNameWithoutExtension(snapshot.VideoSavePath);
result.Add(new LocalCleanupCandidate(Path.Combine(directory ?? string.Empty, basename + ".nfo"), false));
if (snapshot.ViedoType is VideoTypeEnum.dy_mix or VideoTypeEnum.dy_series)
result.Add(new LocalCleanupCandidate(Path.Combine(directory ?? string.Empty, "tvshow.nfo"), true));
}
if (!string.IsNullOrWhiteSpace(snapshot.DynamicVideos))
{
try
{
foreach (var attachment in JsonConvert.DeserializeObject<List<DouyinMergeVideoDto>>(snapshot.DynamicVideos) ?? new())
result.Add(new LocalCleanupCandidate(attachment.Path, false));
}
catch (JsonException) { }
}
return result.Where(x => !string.IsNullOrWhiteSpace(x.Path)).DistinctBy(x => x.Path).ToList();
}
private static bool IsLocalPathReferenced(string path, IEnumerable<DouyinVideo> localVideos, string excludedVideoId, bool shared)
{
foreach (var video in localVideos.Where(x => x.Id != excludedVideoId))
{
if (path == video.VideoSavePath || path == video.VideoCoverSavePath || path == video.AuthorAvatar) return true;
if (shared && !string.IsNullOrWhiteSpace(video.VideoSavePath)
&& string.Equals(Path.GetDirectoryName(video.VideoSavePath), Path.GetDirectoryName(path), StringComparison.Ordinal)) return true;
if (!string.IsNullOrWhiteSpace(video.DynamicVideos) && video.DynamicVideos.Contains(path, StringComparison.Ordinal)) return true;
}
return false;
}
private static void DeleteEmptyParents(string filePath, IReadOnlyList<string> roots)
{
if (string.IsNullOrWhiteSpace(filePath)) return;
var directory = Path.GetDirectoryName(Path.GetFullPath(filePath));
var root = roots.Where(x => SafeLocalMigrationFile.IsWithin(filePath, x))
.OrderByDescending(x => x.Length).FirstOrDefault();
while (!string.IsNullOrWhiteSpace(directory) && !string.Equals(directory, root, StringComparison.Ordinal)
&& Directory.Exists(directory) && !Directory.EnumerateFileSystemEntries(directory).Any())
{
Directory.Delete(directory, false);
directory = Path.GetDirectoryName(directory);
}
}
private sealed record LocalCleanupCandidate(string Path, bool Shared);
public async Task<bool> UpdateOne(DouyinVideo video)
{
return await _dyCollectVideoRepository.UpdateAsync(video);
@@ -246,31 +457,57 @@ namespace dy.net.service
return false;
}
// Permanent deletion is an explicit, separate operation. A normal re-download never removes
// the database row or old media first; VideoRedownloadWorker replaces the main file atomically.
if (forever)
{
foreach (var video in videos)
{
try
{
var storage = _storageRouter.Resolve(video.StorageType);
if (video.StorageType.IsRemote())
await StorageArtifactCleaner.DeleteWebDavArtifactsAsync(storage, video);
else
{
if (!string.IsNullOrWhiteSpace(video.VideoSavePath)) await storage.DeleteAsync(video.VideoSavePath);
if (!string.IsNullOrWhiteSpace(video.VideoCoverSavePath)) await storage.DeleteAsync(video.VideoCoverSavePath);
}
if (!await _dyCollectVideoRepository.DeleteByIdAsync(video.Id)) return false;
}
catch (Exception ex)
{
Serilog.Log.Error(ex, "永久删除视频失败,数据库记录已保留:{VideoId}", video.Id);
return false;
}
}
return true;
}
// 3. 构建重新下载记录(提前准备数据,避免事务内耗时操作)
var reDownList = new List<DouyinReDownload>();
var filePathsToDelete = new List<(string path, bool onlyImgOrMp3)>(); // 收集待删除文件路径,统一处理
foreach (var video in videos)
{
// 跳过无保存路径的视频(避免无效文件操作
// 跳过无保存路径的视频(避免创建无效任务
if (string.IsNullOrWhiteSpace(video.VideoSavePath))
{
Serilog.Log.Debug("视频无保存路径,跳过文件删除VideoId={0}", video.Id);
Serilog.Log.Debug("视频无保存路径,跳过重新下载VideoId={0}", video.Id);
continue;
}
// 构建重新下载记录
reDownList.Add(new DouyinReDownload
{
Id = IdGener.GetLong().ToString(),
Id = Guid.NewGuid().ToString("N"),
CreateTime = DateTime.UtcNow, // 统一使用UTC时间,避免时区问题
Status = 0, // 0=待下载(建议用枚举替代魔法值)
SavePath = video.VideoSavePath,
ViedoId = video.AwemeId,
CookieId = video.CookieId
CookieId = video.CookieId,
VideoRecordId = video.Id,
UpdateTime = DateTime.UtcNow
});
filePathsToDelete.Add((video.VideoSavePath, video.OnlyImgOrOnlyMp3));
}
// 无有效重新下载记录时直接返回
@@ -282,52 +519,40 @@ namespace dy.net.service
try
{
// 4. 数据库操作(事务保证一致性:创建重新下载记录 + 删除原视频记录必须同时成功/失败)
var transactionResult = await _dyCollectVideoRepository.UseTranAsync(async () =>
var videosById = videos.ToDictionary(x => x.Id);
var groups = reDownList.GroupBy(x => videosById[x.VideoRecordId].StorageType).ToList();
var transaction = await sqlSugarClient.Ado.UseTranAsync(async () =>
{
// 4.1 批量插入重新下载记录(SqlSugar批量插入效率更高)
_dyCollectVideoRepository.InsertReDowns(reDownList);
// 4.2 批量删除原视频记录(使用视频实际存在的ID,避免无效删除)
var actualDeleteIds = videos.Select(v => v.Id).ToList();
var deleteCount = await _dyCollectVideoRepository.DeleteByIdsAsync(actualDeleteIds); // 建议仓储层提供异步删除方法
}, e =>
{
Serilog.Log.Error(e, "数据库事务执行失败:Ids={0}", string.Join(",", videoIds));
foreach (var group in groups)
{
var jobs = group.ToList();
var storageLabel = group.Key switch
{
MediaStorageType.WebDav => "WebDAV",
MediaStorageType.OpenList => "OpenList",
_ => "本地"
};
var title = groups.Count == 1
? $"重新下载({jobs.Count} 条)"
: $"重新下载 · {storageLabel}{jobs.Count} 条)";
var task = await _videoTasks.CreateTaskAsync(VideoTaskType.Redownload,
VideoTaskTrigger.UserAction, title, storageType: group.Key);
foreach (var job in jobs)
{
var video = videosById[job.VideoRecordId];
var item = await _videoTasks.AddItemAsync(task.Id, video.AwemeId, video.CookieId, null,
video.ViedoType, video.VideoTitle, video.Author, video.VideoSavePath,
new[] { video.VideoUrl }, video, video.FileSize);
job.TaskId = task.Id;
job.TaskItemId = item.Id;
}
if (await sqlSugarClient.Insertable(jobs).ExecuteCommandAsync() != jobs.Count)
throw new InvalidOperationException($"{storageLabel}重新下载队列写入不完整。");
}
});
// 5. 文件删除(非事务操作,失败不回滚数据库,可根据业务调整)
// 采用异步文件操作,避免同步IO阻塞线程(需.NET 5+支持)
foreach (var video in filePathsToDelete)
{
try
{
if (File.Exists(video.path))
{
File.Delete(video.path); // 异步删除,提升并发性能
Serilog.Log.Debug("视频文件删除成功:Path={0}", video.path);
if (!video.onlyImgOrMp3)//如果是纯图片或纯音频文件,则不删除所在文件夹
{
//检查这个路径所在文件夹是否还有其他视频文件,如果没有则删除这个文件夹
var dir = Path.GetDirectoryName(video.path);
bool hasMp4File = Directory.EnumerateFiles(dir, "*.mp4", SearchOption.TopDirectoryOnly).Any(); // 只要存在一个MP4文件就返回true;
if (!hasMp4File)
{
Directory.Delete(dir, true);
}
}
}
else
{
Serilog.Log.Error("视频文件不存在,跳过删除:Path={0}", video);
}
}
catch (IOException ex)
{
Serilog.Log.Error(ex, "视频文件删除失败:Path={0}", video);
}
}
if (!transaction.IsSuccess)
throw new InvalidOperationException("创建重新下载任务失败:" + transaction.ErrorMessage,
transaction.ErrorException);
//var CookieIds = reDownList.Select(x => x.CookieId).Distinct();
//foreach (var ck in CookieIds)
@@ -362,8 +587,7 @@ namespace dy.net.service
// await douyinCookieRepository.UpdateAsync(cookie);
//}
if (!forever)
Serilog.Log.Debug("重新下载视频流程执行完成:成功创建{0}条重新下载记录,删除{1}个文件,等待重新下载...", reDownList.Count, filePathsToDelete.Count);
Serilog.Log.Debug("安全重新下载任务已创建:{0} 条;旧记录和旧文件会保留到新主媒体完整写入。", reDownList.Count);
return true;
}
catch (Exception ex)
@@ -415,7 +639,8 @@ namespace dy.net.service
List<string> douyinVideoIds = new List<string>();
foreach (var v in videos)
{
if (!File.Exists(v.VideoSavePath))
if (v.OnlyImgOrOnlyMp3 && string.IsNullOrWhiteSpace(v.VideoSavePath)) continue;
if (!await _storageRouter.Resolve(v.StorageType).ExistsAsync(v.VideoSavePath))
{
douyinVideoIds.Add(v.Id);
vList.Add(new DeleteInvalidVideoDto { AwId = v.AwemeId, Title = v.VideoTitle, Path = v.VideoSavePath });
@@ -443,16 +668,26 @@ namespace dy.net.service
internal async Task<int> AddDeleteVideo(List<DouyinVideo> videos)
{
var deletes = videos.Select(video => new DouyinVideoDelete
var awemeIds = videos.Select(x => x.AwemeId).Where(x => !string.IsNullOrWhiteSpace(x)).Distinct().ToList();
var existing = await sqlSugarClient.Queryable<DouyinVideoDelete>()
.Where(x => awemeIds.Contains(x.ViedoId)).Select(x => x.ViedoId).ToListAsync();
var deletes = videos.Where(video => !existing.Contains(video.AwemeId)).Select(video => new DouyinVideoDelete
{
ViedoId = video.AwemeId,
VideoTitle = video.VideoTitle,
VideoSavePath = video.VideoSavePath,
Id = IdGener.GetLong().ToString(),
DeleteTime = DateTime.Now
DeleteTime = DateTime.Now,
CookieId = video.CookieId,
VideoType = video.ViedoType,
AuthorId = video.AuthorId,
Author = video.Author,
VideoUrl = video.VideoUrl,
RestoreSnapshotJson = JsonConvert.SerializeObject(video)
})?.ToList();
return await sqlSugarClient.Insertable<DouyinVideoDelete>(deletes).ExecuteCommandAsync();
if (deletes.Count > 0) await sqlSugarClient.Insertable<DouyinVideoDelete>(deletes).ExecuteCommandAsync();
return existing.Distinct().Count() + deletes.Count;
}
/// <summary>
@@ -470,11 +705,15 @@ namespace dy.net.service
{
if (videos.Count <= 30)
{
var deletes = await AddDeleteVideo(videos);
if (deletes < videos.Select(x => x.AwemeId).Distinct().Count())
{
Serilog.Log.Error("写入永久排除记录失败,已停止删除媒体");
return false;
}
var result = await ReDownloadViedoAsync(new ReDownViedoDto { Ids = videos.Select(x => x.Id)?.ToList() }, true);
if (result)
{
//加入删除逻辑
var deletes = await AddDeleteVideo(videos);
Serilog.Log.Debug($"批量永久删除博主{videos.FirstOrDefault()?.Author},共{deletes}条记录");
return true;
}
@@ -488,11 +727,15 @@ namespace dy.net.service
{
Task.Run(async () =>
{
var deletes = await AddDeleteVideo(videos);
if (deletes < videos.Select(x => x.AwemeId).Distinct().Count())
{
Serilog.Log.Error("写入永久排除记录失败,已停止后台删除媒体");
return;
}
var result = await ReDownloadViedoAsync(new ReDownViedoDto { Ids = videos.Select(x => x.Id)?.ToList() }, true);
if (result)
{
//加入删除逻辑
var deletes = await AddDeleteVideo(videos);
Serilog.Log.Debug($"批量永久删除博主{videos.FirstOrDefault()?.Author}{deletes}条记录");
}
else