diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..4ed5515 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +.git +.ci +artifacts +app/node_modules +app/dist +bin +obj +tests/**/bin +tests/**/obj +db +logs +storage-data +*.fpk diff --git a/.gitignore b/.gitignore index dd2a48f..4d01850 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,10 @@ logs upload data1 /db/*.sqlite -/appsettings.json +/artifacts/ +/.ci/ +/.npm-cache/ +/.nuget-packages/ +/.dotnet-cli-home/ /Properties/PublishProfiles/fn.pubxml.user /Properties/PublishProfiles/*.user diff --git a/Controllers/ConfigController.cs b/Controllers/ConfigController.cs index a2c688c..4221015 100644 --- a/Controllers/ConfigController.cs +++ b/Controllers/ConfigController.cs @@ -10,6 +10,7 @@ using Microsoft.AspNetCore.Mvc; using Newtonsoft.Json; using System.Net; using System.Text.RegularExpressions; +using dy.net.storage; namespace dy.net.Controllers { @@ -25,10 +26,12 @@ namespace dy.net.Controllers private readonly DouyinFollowService douyinFollowService; private readonly DouyinCookieService douyinCookieService; private readonly DouyinHttpClientService httpClientService; + private readonly WebDavSettingsService _webDavSettingsService; + private readonly VideoTaskService _videoTaskService; - public ConfigController(DouyinCookieService dyCookieService, DouyinCommonService commonService, DouyinQuartzJobService quartzJobService, DouyinFollowService douyinFollowService, DouyinCookieService douyinCookieService, DouyinHttpClientService httpClientService) + public ConfigController(DouyinCookieService dyCookieService, DouyinCommonService commonService, DouyinQuartzJobService quartzJobService, DouyinFollowService douyinFollowService, DouyinCookieService douyinCookieService, DouyinHttpClientService httpClientService, WebDavSettingsService webDavSettingsService, VideoTaskService videoTaskService) { this.dyCookieService = dyCookieService; this.commonService = commonService; @@ -36,6 +39,8 @@ namespace dy.net.Controllers this.douyinFollowService = douyinFollowService; this.douyinCookieService = douyinCookieService; this.httpClientService = httpClientService; + _webDavSettingsService = webDavSettingsService; + _videoTaskService = videoTaskService; } @@ -102,6 +107,11 @@ namespace dy.net.Controllers { if (cookies?.Count > 0) { + foreach (var cookie in cookies) ClearImportedSourceHealth(cookie); + if (commonService.GetConfig()?.StorageType.IsRemote() == true) + { + foreach (var cookie in cookies) _webDavSettingsService.ApplyDefaultCookiePaths(cookie); + } var imported = await douyinCookieService.ImportCookies(cookies); if (imported) Serilog.Log.Debug("抖音Cookie配置导入成功"); @@ -164,6 +174,12 @@ namespace dy.net.Controllers if (!cookieValid) return ApiResult.Fail("Cookie无效或已过期，请按照文档提示重新获取有效Cookie，不要使用插件获取cookie"); var result = await dyCookieService.FastResetCookie(dto.id, dto.cookie); + if (result) + { + if (!string.IsNullOrWhiteSpace(dto.id)) await _videoTaskService.ResetSourceHealthAsync(dto.id); + else + foreach (var item in await dyCookieService.GetAllAsync()) await _videoTaskService.ResetSourceHealthAsync(item.Id); + } return result ? ApiResult.Success() : ApiResult.Fail("cookie设置失败"); } @@ -193,8 +209,12 @@ namespace dy.net.Controllers // 1. 基础赋值 dyUserCookies.Id = IdGener.GetLong().ToString(); + var storageType = commonService.GetConfig()?.StorageType ?? StorageType.Local; + if (storageType.IsRemote()) + _webDavSettingsService.ApplyDefaultCookiePaths(dyUserCookies); + // 2. 路径权限校验 - var (Success, Message) = ValidatePaths(dyUserCookies); + var (Success, Message) = ValidatePaths(dyUserCookies, storageType); if (!Success) return ApiResult.Fail(Message); @@ -212,8 +232,28 @@ namespace dy.net.Controllers return saved ? ApiResult.Success() : ApiResult.Fail("添加失败"); } - private static (bool Success, string Message) ValidatePaths(DouyinCookie cookie) + private static (bool Success, string Message) ValidatePaths(DouyinCookie cookie, StorageType storageType) { + if (storageType.IsRemote()) + { + var remotePaths = new Dictionary + { + { "收藏 OpenList 相对路径", cookie.WebDavCollectPath }, + { "喜欢 OpenList 相对路径", cookie.WebDavFavoritePath }, + { "关注 OpenList 相对路径", cookie.WebDavFollowPath }, + { "合集 OpenList 相对路径", cookie.WebDavMixPath }, + { "短剧 OpenList 相对路径", cookie.WebDavSeriesPath } + }; + + foreach (var (label, path) in remotePaths) + { + if (string.IsNullOrWhiteSpace(path)) return (false, $"{label}不能为空"); + try { StoragePath.NormalizeRemote(path); } + catch (ArgumentException ex) { return (false, $"{label}无效：{ex.Message}"); } + } + return (true, string.Empty); + } + var pathsToCheck = new Dictionary { { "收藏存储路径", cookie.SavePath }, @@ -262,6 +302,19 @@ namespace dy.net.Controllers [HttpPost("update")] public async Task AddOrUpdateAsync([FromBody] DouyinCookie dyUserCookies) { + if (dyUserCookies == null) return ApiResult.Fail("配置不能为空"); + + var isNewCookie = string.IsNullOrWhiteSpace(dyUserCookies.Id) || dyUserCookies.Id == "0"; + var storageType = commonService.GetConfig()?.StorageType ?? StorageType.Local; + if (storageType.IsRemote()) + { + if (isNewCookie) + dyUserCookies.Id = IdGener.GetLong().ToString(); + _webDavSettingsService.ApplyDefaultCookiePaths(dyUserCookies); + var (success, message) = ValidatePaths(dyUserCookies, storageType); + if (!success) return ApiResult.Fail(message); + } + RemoveCookieLineString(dyUserCookies); var checkCk = await httpClientService.CheckCookie(dyUserCookies); @@ -271,9 +324,10 @@ namespace dy.net.Controllers } dyUserCookies.StatusCode = 0; dyUserCookies.StatusMsg = "正常"; - if (dyUserCookies.Id == "0") + if (isNewCookie) { - dyUserCookies.Id = IdGener.GetLong().ToString(); + if (string.IsNullOrWhiteSpace(dyUserCookies.Id) || dyUserCookies.Id == "0") + dyUserCookies.Id = IdGener.GetLong().ToString(); var result = await dyCookieService.Add(dyUserCookies); if (result) { @@ -287,6 +341,7 @@ namespace dy.net.Controllers var result = await dyCookieService.UpdateCookieAsync(dyUserCookies); if (result) { + await _videoTaskService.ResetSourceHealthAsync(dyUserCookies.Id); ReStartJob(); return ApiResult.Success(); } @@ -305,6 +360,18 @@ namespace dy.net.Controllers } } + private static void ClearImportedSourceHealth(DouyinCookie cookie) + { + cookie.ConsecutiveSourceForbidden = 0; + cookie.SourceCooldownUntil = null; + cookie.SourceRequiresAuthorization = false; + cookie.SourceProbePending = false; + cookie.SourceProbeInProgress = false; + cookie.LastSourceStatusCode = null; + cookie.LastSourceError = null; + cookie.SourceHealthUpdatedAt = null; + } + /// /// 批量删除用户Cookie /// @@ -351,10 +418,15 @@ namespace dy.net.Controllers //[Authorize] public async Task ExecuteJobNow() { - var config = commonService.GetConfig(); - if (config != null) - await quartzJobService.InitOrReStartAllJobs(config.Cron.ToString()); - return ApiResult.Success(); + try + { + var tasks = await quartzJobService.TriggerVideoJobsNowAsync(); + return ApiResult.Success(new { taskIds = tasks.Select(x => x.Id).ToList(), count = tasks.Count }); + } + catch (InvalidOperationException ex) + { + return ApiResult.Fail(ex.Message); + } } diff --git a/Controllers/EmailController.cs b/Controllers/EmailController.cs new file mode 100644 index 0000000..4c0135c --- /dev/null +++ b/Controllers/EmailController.cs @@ -0,0 +1,47 @@ +using dy.net.model.dto; +using dy.net.service; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; + +namespace dy.net.Controllers +{ + [Route("api/email")] + [ApiController] + [Authorize] + public sealed class EmailController : ControllerBase + { + private readonly EmailNotificationSettingsService _settings; + + public EmailController(EmailNotificationSettingsService settings) => _settings = settings; + + [HttpGet("config")] + public async Task GetConfig() + { + var settings = await _settings.GetAsync(); + return ApiResult.Success(_settings.ToDto(settings)); + } + + [HttpPut("config")] + public async Task UpdateConfig(EmailNotificationSettingsDto dto) + { + try { return ApiResult.Success(await _settings.SaveAsync(dto)); } + catch (InvalidOperationException ex) { return ApiResult.Fail(ex.Message); } + } + + [HttpPost("test")] + public async Task Test(EmailNotificationSettingsDto dto, CancellationToken cancellationToken) + { + try + { + var message = await _settings.TestAsync(dto, cancellationToken); + return ApiResult.Success(new { message }); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + var message = ex.GetBaseException().Message; + Serilog.Log.Warning(ex, "SMTP 测试邮件发送失败"); + return ApiResult.Fail(string.IsNullOrWhiteSpace(message) ? "测试邮件发送失败" : message); + } + } + } +} diff --git a/Controllers/FollowController.cs b/Controllers/FollowController.cs index 217fdfc..46cb48f 100644 --- a/Controllers/FollowController.cs +++ b/Controllers/FollowController.cs @@ -4,6 +4,7 @@ using dy.net.service; using dy.net.utils; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; +using Serilog; namespace dy.net.Controllers { @@ -14,11 +15,19 @@ namespace dy.net.Controllers { private readonly DouyinFollowService _douyinFollowService; private readonly DouyinQuartzJobService _douyinQuartzJobService; + private readonly DouyinUserLookupService _douyinUserLookupService; + private readonly DouyinLiveStatusService _douyinLiveStatusService; - public FollowController(DouyinFollowService douyinFollowService, DouyinQuartzJobService douyinQuartzJobService) + public FollowController( + DouyinFollowService douyinFollowService, + DouyinQuartzJobService douyinQuartzJobService, + DouyinUserLookupService douyinUserLookupService, + DouyinLiveStatusService douyinLiveStatusService) { this._douyinFollowService = douyinFollowService; _douyinQuartzJobService = douyinQuartzJobService; + _douyinUserLookupService = douyinUserLookupService; + _douyinLiveStatusService = douyinLiveStatusService; } @@ -52,20 +61,107 @@ namespace dy.net.Controllers [HttpGet("sync")] public async Task SyncFollowList() { - //后台异步 - _douyinQuartzJobService.StartFollowJobOnceAsync(); - await Task.Delay(1000); - return ApiResult.Success(); + try + { + var started = await _douyinQuartzJobService.StartFollowJobOnceAsync(); + return started + ? ApiResult.Success(new { message = "关注列表更新任务已启动" }) + : ApiResult.Fail("关注列表更新任务启动失败，请查看错误日志。"); + } + catch (Exception ex) + { + Log.Error(ex, "手动启动关注列表更新任务失败"); + return ApiResult.Fail("关注列表更新任务启动失败：" + ex.GetBaseException().Message); + } } [HttpPost("add")] public async Task AddFollow(DouyinFollowed followed) { + if (followed == null) + { + return ApiResult.Fail("新增信息不能为空"); + } if (string.IsNullOrWhiteSpace(followed.mySelfId)) { return ApiResult.Fail("请先配置抖音授权信息，抖音授权配置里面填写你的uid"); } - var res = await _douyinFollowService.AddAsync(followed); - return ApiResult.SuccOrFail(res, "", res ? "" : "添加失败,或者已存在相同secuid和uid"); + if (string.IsNullOrWhiteSpace(followed.SecUid)) + { + return ApiResult.Fail("博主 SecUid 不能为空"); + } + if (string.IsNullOrWhiteSpace(followed.UperId)) + { + return ApiResult.Fail("博主 UID 不能为空"); + } + if (string.IsNullOrWhiteSpace(followed.UperName)) + { + return ApiResult.Fail("博主姓名不能为空"); + } + + followed.mySelfId = followed.mySelfId.Trim(); + followed.SecUid = followed.SecUid.Trim(); + followed.UperId = followed.UperId.Trim(); + followed.UperName = followed.UperName.Trim(); + followed.DouyinNo = followed.DouyinNo?.Trim(); + followed.SavePath = followed.SavePath?.Trim(); + followed.Signature = Limit(followed.Signature, 500); + followed.UperAvatar = Limit(followed.UperAvatar, 1000); + followed.Enterprise = Limit(followed.Enterprise, 200); + followed.FullSync = followed.OpenSync && followed.FullSync; + + if (followed.mySelfId.Length > 200 || followed.SecUid.Length > 500 || + followed.UperId.Length > 100 || followed.UperName.Length > 200) + { + return ApiResult.Fail("新增信息长度超出限制，请检查 UID、SecUid 和博主姓名"); + } + if (!string.IsNullOrWhiteSpace(followed.DouyinNo) && followed.DouyinNo.Length > 100) + { + return ApiResult.Fail("抖音号长度不能超过 100 个字符"); + } + if (!string.IsNullOrWhiteSpace(followed.SavePath)) + { + if (followed.SavePath.Length > 20) + { + return ApiResult.Fail("保存文件夹名称最长 20 个字符"); + } + if (!DouyinFileNameHelper.IsValidWithoutSpecialChars(followed.SavePath)) + { + return ApiResult.Fail("保存文件夹名称只能包含字母、数字或中文"); + } + } + + var result = await _douyinFollowService.TryAddAsync(followed); + return result switch + { + AddFollowResult.Added => ApiResult.Success("新增非关注博主成功"), + AddFollowResult.AlreadyExists => ApiResult.Fail("该博主已存在，无需重复添加"), + _ => ApiResult.Fail("添加失败，请稍后重试") + }; + } + + /// + /// 使用展示在抖音资料页上的抖音号查找博主。结果需要用户确认后再调用 add。 + /// + [HttpPost("resolve-by-douyin-no")] + public async Task ResolveByDouyinNo( + DouyinFollowLookupRequest request, + CancellationToken cancellationToken) + { + try + { + var result = await _douyinUserLookupService.ResolveAsync(request, cancellationToken); + return ApiResult.Success(result); + } + catch (InvalidOperationException ex) + { + return ApiResult.Fail(ex.Message); + } + } + + private static string Limit(string value, int maxLength) + { + value = value?.Trim(); + return value != null && value.Length > maxLength ? value[..maxLength] : value; } /// @@ -107,6 +203,59 @@ namespace dy.net.Controllers return await OpenOrCloseSync(dto); } + [HttpPost("live-monitor")] + public async Task UpdateLiveMonitor( + FollowLiveMonitorUpdateDto dto, + CancellationToken cancellationToken) + { + try + { + return ApiResult.Success(await _douyinLiveStatusService.SetMonitorAsync(dto, cancellationToken)); + } + catch (Exception ex) when (ex is InvalidOperationException or KeyNotFoundException) + { + return ApiResult.Fail(ex.Message); + } + } + + [HttpPost("live-status/refresh")] + public async Task RefreshLiveStatus( + FollowLiveStatusRefreshDto dto, + CancellationToken cancellationToken) + { + try + { + if (dto == null || string.IsNullOrWhiteSpace(dto.Id)) + return ApiResult.Fail("博主记录不能为空"); + return ApiResult.Success(await _douyinLiveStatusService.RefreshAsync(dto.Id, cancellationToken)); + } + catch (Exception ex) when (ex is InvalidOperationException or KeyNotFoundException) + { + return ApiResult.Fail(ex.Message); + } + } + + [HttpPost("live-status/query")] + public async Task QueryLiveStatus(FollowLiveStatusQueryDto dto) + { + if (dto?.Ids == null || dto.Ids.Count == 0) + return ApiResult.Success(Array.Empty()); + return ApiResult.Success(await _douyinLiveStatusService.QueryAsync(dto.Ids)); + } + + [HttpPost("live-email")] + public async Task UpdateLiveEmail(FollowLiveEmailUpdateDto dto) + { + try + { + return ApiResult.Success(await _douyinLiveStatusService.SetEmailNotificationAsync(dto)); + } + catch (Exception ex) when (ex is InvalidOperationException or KeyNotFoundException) + { + return ApiResult.Fail(ex.Message); + } + } + /// /// 删除关注对象 /// diff --git a/Controllers/OpenListDirectoryRepairsController.cs b/Controllers/OpenListDirectoryRepairsController.cs new file mode 100644 index 0000000..52494a6 --- /dev/null +++ b/Controllers/OpenListDirectoryRepairsController.cs @@ -0,0 +1,75 @@ +using dy.net.model.dto; +using dy.net.service; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; + +namespace dy.net.Controllers +{ + [Route("api/storage/openlist/directory-repairs")] + [ApiController] + [Authorize] + public sealed class OpenListDirectoryRepairsController : ControllerBase + { + private readonly OpenListDirectoryRepairService _service; + + public OpenListDirectoryRepairsController(OpenListDirectoryRepairService service) => _service = service; + + [HttpPost("preflight")] + public Task Preflight( + [FromBody] OpenListDirectoryRepairPreflightRequest request, + CancellationToken cancellationToken) => ExecuteAsync(async () => + ApiResult.Success(await _service.PreflightAsync(request, cancellationToken))); + + [HttpPost] + public Task Create( + [FromBody] CreateOpenListDirectoryRepairRequest request, + CancellationToken cancellationToken) => ExecuteAsync(async () => + ApiResult.Success(await _service.CreateAsync(request, cancellationToken))); + + [HttpGet("latest")] + public Task Latest() => ExecuteAsync(async () => + ApiResult.Success(await _service.GetLatestAsync())); + + [HttpGet("{id}")] + public Task Get(string id) => ExecuteAsync(async () => + ApiResult.Success(await _service.GetAsync(id))); + + [HttpGet("{id}/items")] + public Task Items(string id, [FromQuery] OpenListDirectoryRepairItemPageRequest request) => + ExecuteAsync(async () => ApiResult.Success(await _service.GetItemsAsync(id, request))); + + [HttpPost("{id}/confirm-cleanup")] + public Task Confirm( + string id, + [FromBody] ConfirmOpenListDirectoryRepairRequest request) => ExecuteAsync(async () => + { + await _service.ConfirmCleanupAsync(id, request); + return ApiResult.Success("清理任务已提交"); + }); + + [HttpPost("{id}/cancel")] + public Task Cancel(string id) => ExecuteActionAsync(() => _service.CancelAsync(id)); + + [HttpPost("{id}/resume")] + public Task Resume(string id) => ExecuteActionAsync(() => _service.ResumeAsync(id)); + + [HttpPost("{id}/retry-failed")] + public Task RetryFailed(string id) => ExecuteActionAsync(() => _service.RetryFailedAsync(id)); + + private Task ExecuteActionAsync(Func action) => ExecuteAsync(async () => + { + await action(); + return ApiResult.Success("操作已提交"); + }); + + private static async Task ExecuteAsync(Func> action) + { + try { return await action(); } + catch (Exception ex) when (ex is InvalidOperationException or KeyNotFoundException + or HttpRequestException or TimeoutException) + { + return ApiResult.Fail(ex.GetBaseException().Message); + } + } + } +} diff --git a/Controllers/StorageController.cs b/Controllers/StorageController.cs new file mode 100644 index 0000000..0464d65 --- /dev/null +++ b/Controllers/StorageController.cs @@ -0,0 +1,154 @@ +using dy.net.model.dto; +using dy.net.service; +using dy.net.storage; +using jzc.http.lib; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; + +namespace dy.net.Controllers +{ + [Route("api/[controller]")] + [ApiController] + [Authorize] + public class StorageController : ControllerBase + { + private readonly DouyinCommonService _commonService; + private readonly OpenListSettingsService _openListSettingsService; + private readonly OpenListMediaStorage _openListStorage; + private readonly DouyinQuartzJobService _quartzJobService; + private readonly DouyinVideoService _videoService; + + public StorageController( + DouyinCommonService commonService, + OpenListSettingsService openListSettingsService, + OpenListMediaStorage openListStorage, + DouyinQuartzJobService quartzJobService, + DouyinVideoService videoService) + { + _commonService = commonService; + _openListSettingsService = openListSettingsService; + _openListStorage = openListStorage; + _quartzJobService = quartzJobService; + _videoService = videoService; + } + + [HttpGet("inventory")] + public async Task GetInventory() + { + var type = _commonService.GetConfig()?.StorageType ?? StorageType.Local; + return ApiResult.Success(await _videoService.GetStorageInventoryAsync(type)); + } + + [HttpGet("config")] + public async Task GetConfig() + { + var type = _commonService.GetConfig()?.StorageType ?? StorageType.Local; + var inventory = await _videoService.GetStorageInventoryAsync(type); + var saved = await _openListSettingsService.HasSavedSettingsAsync(); + var dto = await _openListSettingsService.ToDtoAsync(type); + dto.RequiresCutover = type == StorageType.WebDav || inventory.WebDavRecordCount > 0; + dto.LegacyWebDavRecordCount = inventory.WebDavRecordCount; + dto.SuggestedFromLegacy = !saved; + return ApiResult.Success(dto); + } + + [HttpPost("test")] + public async Task Test( + [FromBody] StorageTestRequest request, CancellationToken cancellationToken) + { + if (request == null) return ApiResult.Fail("测试请求不能为空"); + + if (request.StorageType != StorageType.OpenList) + return ApiResult.Fail("新远端存储只支持 OpenList 原生 API。"); + var candidate = await _openListSettingsService.BuildCandidateAsync( + new OpenListTestRequest + { + Endpoint = request.Endpoint, + BasePath = request.BasePath, + LocalStagingPath = request.LocalStagingPath, + SourcePath = request.SourcePath, + UserName = request.UserName, + Password = request.Password + }); + var result = await _openListStorage.ProbeAsync(candidate, cancellationToken); + return result.Success + ? ApiResult.Success(new { message = result.Message }) + : ApiResult.Fail(result.Message); + } + + [HttpPost("openlist/directories")] + public async Task ListOpenListDirectories( + [FromBody] OpenListDirectoryRequest request, + CancellationToken cancellationToken) + { + if (request == null) return ApiResult.Fail("目录请求不能为空"); + try + { + var candidate = await _openListSettingsService.BuildCandidateAsync(request); + return ApiResult.Success(await _openListStorage.ListDirectoriesAsync( + candidate, request.Path, cancellationToken)); + } + catch (Exception ex) when (ex is InvalidOperationException or HttpRequestException) + { return ApiResult.Fail(ex.GetBaseException().Message); } + } + + [HttpPut("config")] + public async Task Update( + [FromBody] StorageSettingsDto dto, CancellationToken cancellationToken) + { + if (dto == null) return ApiResult.Fail("配置不能为空"); + + if (dto.StorageType == StorageType.WebDav) + return ApiResult.Fail("WebDAV 已停止作为新存储目标，请改用 OpenList 原生 API。"); + + if (dto.StorageType == StorageType.OpenList) + { + var candidate = await _openListSettingsService.BuildCandidateAsync( + new OpenListTestRequest + { + Endpoint = dto.Endpoint, + BasePath = dto.BasePath, + LocalStagingPath = dto.LocalStagingPath, + SourcePath = dto.SourcePath, + UserName = dto.UserName, + Password = dto.Password + }); + + var result = await _openListStorage.ProbeAsync(candidate, cancellationToken); + if (!result.Success) return ApiResult.Fail("无法启用 OpenList：" + result.Message); + + await _openListSettingsService.SaveAsync(candidate, true, result.Message); + await _openListSettingsService.InitializeCookiePathsAsync(); + + var config = _commonService.GetConfig(); + if (config == null) return ApiResult.Fail("系统配置不存在"); + config.StorageType = dto.StorageType; + if (!await _commonService.UpdateConfig(config)) + return ApiResult.Fail("存储模式保存失败"); + await _quartzJobService.InitOrReStartAllJobs(config.Cron.ToString()); + + var response = await _openListSettingsService.ToDtoAsync(config.StorageType); + var inventory = await _videoService.GetStorageInventoryAsync(config.StorageType); + response.RequiresCutover = inventory.WebDavRecordCount > 0; + response.LegacyWebDavRecordCount = inventory.WebDavRecordCount; + return ApiResult.Success(response); + } + + if (dto.StorageType == StorageType.Local) + { + var config = _commonService.GetConfig(); + if (config == null) return ApiResult.Fail("系统配置不存在"); + config.StorageType = StorageType.Local; + if (!await _commonService.UpdateConfig(config)) + return ApiResult.Fail("存储模式保存失败"); + await _quartzJobService.InitOrReStartAllJobs(config.Cron.ToString()); + var response = await _openListSettingsService.ToDtoAsync(config.StorageType); + var inventory = await _videoService.GetStorageInventoryAsync(config.StorageType); + response.RequiresCutover = inventory.WebDavRecordCount > 0; + response.LegacyWebDavRecordCount = inventory.WebDavRecordCount; + return ApiResult.Success(response); + } + return ApiResult.Fail("不支持的存储类型"); + } + } +} diff --git a/Controllers/StorageMigrationsController.cs b/Controllers/StorageMigrationsController.cs new file mode 100644 index 0000000..ff611e3 --- /dev/null +++ b/Controllers/StorageMigrationsController.cs @@ -0,0 +1,88 @@ +using dy.net.model.dto; +using dy.net.service; +using jzc.http.lib; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; + +namespace dy.net.Controllers +{ + [Route("api/storage/migrations")] + [ApiController] + [Authorize] + public class StorageMigrationsController : ControllerBase + { + private readonly StorageMigrationService _service; + + public StorageMigrationsController(StorageMigrationService service) => _service = service; + + [HttpPost("preflight")] + public async Task Preflight([FromBody] StorageMigrationPreflightRequest request, CancellationToken cancellationToken) => + ApiResult.Success(await _service.PreflightAsync(request?.VerifyCapabilities ?? true, cancellationToken)); + + [HttpPost] + public async Task Create([FromBody] CreateStorageMigrationRequest request, CancellationToken cancellationToken) => + await ExecuteAsync(async () => ApiResult.Success(await _service.CreateAsync(request, cancellationToken))); + + [HttpGet("latest")] + public async Task Latest() => ApiResult.Success(await _service.GetLatestAsync()); + + [HttpGet("failed-records/preview")] + public async Task FailedRecordsPreview() => + ApiResult.Success(await _service.PreviewFailedRecordRemovalAsync()); + + [HttpPost("failed-records/remove")] + public async Task RemoveFailedRecords([FromBody] RemoveFailedMigrationRecordsRequest request) => + await ExecuteAsync(async () => ApiResult.Success(await _service.RemoveFailedRecordsAsync(request))); + + [HttpPost("history/archive")] + public async Task ArchiveHistory() => + await ExecuteAsync(async () => ApiResult.Success(await _service.ArchiveAllSettledAsync())); + + [HttpGet("{id}")] + public async Task Get(string id) => + await ExecuteAsync(async () => ApiResult.Success(await _service.GetAsync(id))); + + [HttpGet("{id}/items")] + public async Task Items(string id, [FromQuery] StorageMigrationItemPageRequest request) => + await ExecuteAsync(async () => ApiResult.Success(await _service.GetItemsAsync(id, request))); + + [HttpPost("{id}/pause")] + public Task Pause(string id) => ExecuteActionAsync(() => _service.PauseAsync(id)); + + [HttpPost("{id}/resume")] + public Task Resume(string id) => ExecuteActionAsync(() => _service.ResumeAsync(id)); + + [HttpPost("{id}/cancel")] + public Task Cancel(string id) => ExecuteActionAsync(() => _service.CancelAsync(id)); + + [HttpPost("{id}/retry-failed")] + public Task RetryFailed(string id) => ExecuteActionAsync(() => _service.RetryFailedAsync(id)); + + [HttpPost("{id}/cleanup")] + public Task Cleanup(string id, CancellationToken cancellationToken) => + ExecuteActionAsync(() => _service.CleanupAsync(id, cancellationToken)); + + [HttpPost("{id}/rollback")] + public Task Rollback(string id, CancellationToken cancellationToken) => + ExecuteActionAsync(() => _service.RollbackAsync(id, cancellationToken)); + + [HttpPost("{id}/archive")] + public Task Archive(string id) => ExecuteActionAsync(() => _service.ArchiveAsync(id)); + + private async Task ExecuteActionAsync(Func action) => + await ExecuteAsync(async () => + { + await action(); + return ApiResult.Success("操作已提交"); + }); + + private static async Task ExecuteAsync(Func> action) + { + try { return await action(); } + catch (Exception ex) when (ex is InvalidOperationException or KeyNotFoundException) + { + return ApiResult.Fail(ex.Message); + } + } + } +} diff --git a/Controllers/TasksController.cs b/Controllers/TasksController.cs new file mode 100644 index 0000000..9c92746 --- /dev/null +++ b/Controllers/TasksController.cs @@ -0,0 +1,100 @@ +using dy.net.model.dto; +using dy.net.service; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; + +namespace dy.net.Controllers +{ + [Route("api/tasks")] + [ApiController] + [Authorize] + public class TasksController : ControllerBase + { + private readonly VideoTaskService _tasks; + private readonly DouyinQuartzJobService _quartz; + private readonly DouyinVideoService _videos; + + public TasksController(VideoTaskService tasks, DouyinQuartzJobService quartz, DouyinVideoService videos) + { + _tasks = tasks; + _quartz = quartz; + _videos = videos; + } + + [HttpPost("sync")] + public async Task StartSync([FromQuery] VideoTypeEnum? videoType = null) => await ExecuteAsync(async () => + { + var tasks = await _quartz.TriggerVideoJobsNowAsync(videoType); + return ApiResult.Success(new { taskIds = tasks.Select(x => x.Id).ToList(), count = tasks.Count }); + }); + + [HttpGet] + public async Task List([FromQuery] VideoTaskPageRequest request) => + ApiResult.Success(await _tasks.GetTasksAsync(request)); + + [HttpGet("summary")] + public async Task Summary() => ApiResult.Success(await _tasks.GetSummaryAsync()); + + [HttpGet("{type}/{id}")] + public async Task Detail(VideoTaskType type, string id) => + await ExecuteAsync(async () => ApiResult.Success(await _tasks.GetTaskAsync(type, id))); + + [HttpGet("{type}/{id}/items")] + public async Task Items(VideoTaskType type, string id, [FromQuery] VideoTaskItemPageRequest request) => + await ExecuteAsync(async () => ApiResult.Success(await _tasks.GetItemsAsync(type, id, request))); + + [HttpPost("{type}/{id}/retry-failed")] + public async Task RetryFailed(VideoTaskType type, string id) => + await ExecuteAsync(async () => + { + await _tasks.RetryFailedAsync(type, id); + return ApiResult.Success("重试任务已提交"); + }); + + [HttpPost("{type}/{id}/items/{itemId}/retry")] + public async Task RetryItem(VideoTaskType type, string id, string itemId) => + await ExecuteAsync(async () => + { + await _tasks.RetryItemAsync(type, id, itemId); + return ApiResult.Success("重试任务已提交"); + }); + + [HttpPost("{type}/{id}/items/{itemId}/retry-cleanup")] + public async Task RetryCleanup(VideoTaskType type, string id, string itemId) => + await ExecuteAsync(async () => + { + if (type != VideoTaskType.Sync) + throw new InvalidOperationException("仅普通同步任务支持重试旧本地文件清理。"); + var deleted = await _videos.RetryTaskItemLocalCleanupAsync(id, itemId); + return ApiResult.Success(new + { + deletedCount = deleted, + message = deleted > 0 ? $"已清理 {deleted} 个旧本地文件。" : "旧本地文件已不存在或仍被共享引用，清理状态已确认。" + }); + }); + + [HttpPost("{type}/{id}/actions/{action}")] + public async Task Action(VideoTaskType type, string id, string action, CancellationToken cancellationToken) => + await ExecuteAsync(async () => + { + await _tasks.ExecuteTaskActionAsync(type, id, action, cancellationToken); + return ApiResult.Success("操作已提交"); + }); + + [HttpPost("storage-health/probe")] + public async Task Probe(CancellationToken cancellationToken) + { + var result = await _tasks.ProbeStorageAsync(cancellationToken); + return result.Success ? ApiResult.Success(new { message = result.Message }) : ApiResult.Fail(result.Message); + } + + private static async Task ExecuteAsync(Func> action) + { + try { return await action(); } + catch (Exception ex) when (ex is InvalidOperationException or KeyNotFoundException) + { + return ApiResult.Fail(ex.Message); + } + } + } +} diff --git a/Controllers/VideoController.cs b/Controllers/VideoController.cs index 2e20d5f..7798a65 100644 --- a/Controllers/VideoController.cs +++ b/Controllers/VideoController.cs @@ -4,6 +4,7 @@ using dy.net.service; using dy.net.utils; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; +using dy.net.storage; namespace dy.net.Controllers { @@ -13,11 +14,13 @@ namespace dy.net.Controllers { private readonly DouyinVideoService douyinVideoService; private readonly DouyinCommonService douyinCommonService; + private readonly MediaStorageRouter _storageRouter; - public VideoController(DouyinVideoService dyCollectVideoService, DouyinCommonService douyinCommonService) + public VideoController(DouyinVideoService dyCollectVideoService, DouyinCommonService douyinCommonService, MediaStorageRouter storageRouter) { this.douyinVideoService = dyCollectVideoService; this.douyinCommonService = douyinCommonService; + _storageRouter = storageRouter; } /// /// 分页查询收藏视频 @@ -88,6 +91,9 @@ namespace dy.net.Controllers return ApiResult.Fail("视频信息不能为空"); } + if (video.StorageType.IsRemote()) + return await PlayRemoteVideoAsync(video); + // 1. 获取完整物理路径并校验 string videoFullPath = video.VideoSavePath; if (string.IsNullOrWhiteSpace(videoFullPath)) @@ -153,6 +159,34 @@ namespace dy.net.Controllers } } + private async Task PlayRemoteVideoAsync(DouyinVideo video) + { + if (string.IsNullOrWhiteSpace(video.VideoSavePath)) return ApiResult.Fail("远端视频路径不能为空"); + + long? from = null; + long? to = null; + var range = Request.Headers.Range.ToString(); + if (!string.IsNullOrWhiteSpace(range) && range.StartsWith("bytes=", StringComparison.OrdinalIgnoreCase)) + { + var parts = range[6..].Split('-', 2); + if (long.TryParse(parts[0], out var parsedFrom)) from = parsedFrom; + if (parts.Length > 1 && long.TryParse(parts[1], out var parsedTo)) to = parsedTo; + } + + await using var remote = await _storageRouter.Resolve(video.StorageType) + .OpenReadAsync(video.VideoSavePath, from, to, HttpContext.RequestAborted); + + Response.StatusCode = remote.StatusCode; + Response.ContentType = remote.ContentType ?? GetContentType(video.VideoSavePath); + Response.Headers.AcceptRanges = "bytes"; + if (remote.ContentLength.HasValue) Response.ContentLength = remote.ContentLength.Value; + if (!string.IsNullOrWhiteSpace(remote.ContentRange)) Response.Headers.ContentRange = remote.ContentRange; + + await remote.Stream.CopyToAsync(Response.Body, 81920, HttpContext.RequestAborted); + await Response.Body.FlushAsync(HttpContext.RequestAborted); + return new EmptyResult(); + } + /// /// 处理分片请求（Range），异步安全处理流 /// @@ -275,6 +309,7 @@ namespace dy.net.Controllers /// /// /// + [Authorize] [HttpPost("redown")] public async Task ReDownload(ReDownViedoDto dto) { @@ -301,6 +336,7 @@ namespace dy.net.Controllers /// /// /// + [Authorize] [HttpPost("vdelete/batch")] public async Task BathRealDelete(ReDownViedoDto dto) { @@ -321,6 +357,7 @@ namespace dy.net.Controllers /// /// /// + [Authorize] [HttpGet("vdelete/{vid}")] public async Task DeleteVideo([FromRoute] string vid) { @@ -346,6 +383,7 @@ namespace dy.net.Controllers /// 查询已删除视频列表 /// /// + [Authorize] [HttpGet("vdelete/get")] public async Task GetDeleteVideo() { @@ -356,6 +394,7 @@ namespace dy.net.Controllers /// /// /// + [Authorize] [HttpGet("vdelete/byauthor/{uperUid}")] public async Task DeleteByAuthor([FromRoute] string uperUid) { @@ -396,6 +435,7 @@ namespace dy.net.Controllers /// 删除无效视频记录 /// /// + [Authorize] [HttpGet("removeInvalid")] public async Task RemoveInvalidVideo() { @@ -463,6 +503,7 @@ namespace dy.net.Controllers } } + [Authorize] [HttpGet("/Move")] public async Task Move() { diff --git a/Controllers/VideoExclusionsController.cs b/Controllers/VideoExclusionsController.cs new file mode 100644 index 0000000..8d1ebbe --- /dev/null +++ b/Controllers/VideoExclusionsController.cs @@ -0,0 +1,31 @@ +using dy.net.model.dto; +using dy.net.service; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; + +namespace dy.net.Controllers +{ + [Route("api/video/exclusions")] + [ApiController] + [Authorize] + public class VideoExclusionsController : ControllerBase + { + private readonly VideoExclusionService _service; + + public VideoExclusionsController(VideoExclusionService service) => _service = service; + + [HttpGet] + public async Task List([FromQuery] VideoExclusionPageRequest request) => + ApiResult.Success(await _service.GetPageAsync(request)); + + [HttpPost("unexclude")] + public async Task Unexclude([FromBody] UnexcludeVideosRequest request) + { + try { return ApiResult.Success(await _service.UnexcludeAsync(request)); } + catch (Exception ex) when (ex is InvalidOperationException or KeyNotFoundException) + { + return ApiResult.Fail(ex.Message); + } + } + } +} diff --git a/Dockerfile b/Dockerfile index 3339e34..1d4090e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,35 @@ -#See https://aka.ms/customizecontainer to learn how to customize your debug container and how Visual Studio uses this Dockerfile to build your images for faster debugging. +ARG NODE_IMAGE=node:20-bookworm-slim +ARG DOTNET_SDK_IMAGE=mcr.microsoft.com/dotnet/sdk:6.0 +ARG DOTNET_RUNTIME_IMAGE=mcr.microsoft.com/dotnet/aspnet:6.0 -FROM mcr.microsoft.com/dotnet/aspnet:6.0 AS base +FROM ${NODE_IMAGE} AS frontend-build +WORKDIR /src/app +COPY app/package.json app/package-lock.json ./ +RUN npm ci +COPY app/ ./ +RUN npm run build + +FROM ${DOTNET_SDK_IMAGE} AS publish +WORKDIR /src +COPY dy.net.csproj dy.net.sln ./ +COPY lib/ ./lib/ +RUN dotnet restore dy.net.csproj --disable-parallel +COPY . ./ +COPY --from=frontend-build /src/app/dist/ ./app/dist/ +RUN dotnet publish dy.net.csproj -c Release --no-restore \ + -p:DebugType=None -p:DebugSymbols=false -o /out /maxcpucount:1 + +FROM ${DOTNET_RUNTIME_IMAGE} +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl ffmpeg \ + && rm -rf /var/lib/apt/lists/* WORKDIR /app +COPY --from=publish /out/ ./ +ENV ASPNETCORE_URLS=http://0.0.0.0:10101 \ + DYSYNC_LISTEN_URL=http://0.0.0.0:10101 \ + TZ=Asia/Shanghai +VOLUME ["/data"] EXPOSE 10101 - - -RUN echo "deb http://mirrors.aliyun.com/debian/ bookworm main non-free contrib" > /etc/apt/sources.list && \ - echo "deb http://mirrors.aliyun.com/debian-security/ bookworm-security main" >> /etc/apt/sources.list && \ - echo "deb http://mirrors.aliyun.com/debian/ bookworm-updates main non-free contrib" >> /etc/apt/sources.list && \ - echo "deb http://mirrors.aliyun.com/debian/ bookworm-backports main non-free contrib" >> /etc/apt/sources.list && \ - apt-get update && \ - apt-get install -y --no-install-recommends ffmpeg && \ - rm -rf /var/lib/apt/lists/* - -RUN ffmpeg -version - -COPY . . -ENV ASPNETCORE_URLS=http://*:10101 -ENV TZ=Asia/Shanghai -ENTRYPOINT ["dotnet", "dy.net.dll"] +HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ + CMD curl -fsS http://127.0.0.1:10101/ >/dev/null || exit 1 +ENTRYPOINT ["dotnet", "dy.net.dll", "/data"] diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..d1c08e2 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,222 @@ +pipeline { + agent { + node { + label '构建机1' + customWorkspace '/home/nanxunai/goujian/workspace/douyin-release' + } + } + + options { + timestamps() + disableConcurrentBuilds() + skipDefaultCheckout(true) + timeout(time: 180, unit: 'MINUTES') + buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '20')) + } + + environment { + PATH = '/home/nanxunai/.local/bin:/usr/local/bin:/usr/bin:/bin' + DOTNET = '/home/nanxunai/.local/bin/dotnet' + FNPACK_BIN = '/home/nanxunai/.local/bin/fnpack' + REGISTRY_URL = 'reg.nxsir.cn' + IMAGE_REPO = 'reg.nxsir.cn/douyin' + HARBOR_CREDENTIALS = 'douyin_key' + OPENLIST_CREDENTIALS = 'openlist_key' + NODE_CREDENTIALS = 'bbb939ea-4f01-4b47-aecb-c5ee2a551ef4' + OPENLIST_BASE_URL = 'https://openlist.nxsir.cn' + OPENLIST_REMOTE_DIR = '/yidongpan/构建产物/douyin' + HTTP_PROXY_URL = 'http://192.168.5.200:7890' + NO_PROXY_HOSTS = '127.0.0.1,localhost,reg.nxsir.cn,gitea.nxsir.cn,openlist.nxsir.cn,mcr.microsoft.com,docker.m.daocloud.io' + BUILDER_NAME = "douyin-${BUILD_NUMBER}" + DOCKER_CONFIG = "${WORKSPACE}/.ci/docker" + NPM_CONFIG_CACHE = "${WORKSPACE}/.ci/npm-cache" + NUGET_PACKAGES = "${WORKSPACE}/.ci/nuget-packages" + DOTNET_CLI_HOME = "${WORKSPACE}/.ci/dotnet-home" + DYSYNC_BUILD_TMPDIR = "${WORKSPACE}/.ci/fnos-tmp" + DYSYNC_VERIFY_TMPDIR = "${WORKSPACE}/.ci/fnos-verify" + DYSYNC_SMOKE_TMPDIR = "${WORKSPACE}/.ci/fnos-smoke" + } + + stages { + stage('Checkout') { + steps { + deleteDir() + checkout scm + } + } + + stage('Metadata And Preflight') { + steps { + script { + env.APP_VERSION = sh(script: "sed -n 's/^version[[:space:]]*=[[:space:]]*//p' fnos/manifest | head -n 1", returnStdout: true).trim() + env.SHORT_SHA = sh(script: 'git rev-parse --short=8 HEAD', returnStdout: true).trim() + env.FNOS_VERSION = sh(script: ''' + set -euo pipefail + base="$APP_VERSION" + major=${base%%.*}; remainder=${base#*.} + minor=${remainder%%.*}; patch=${remainder#*.} + test "$base" = "$major.$minor.$patch" + test "$BUILD_NUMBER" -lt 100000 + printf '%s.%s.%s\n' "$major" "$minor" "$((patch * 100000 + BUILD_NUMBER))" + ''', returnStdout: true).trim() + env.IMMUTABLE_TAG = "${env.APP_VERSION}-b${env.BUILD_NUMBER}-${env.SHORT_SHA}" + env.IMAGE_REF = "${env.IMAGE_REPO}:${env.IMMUTABLE_TAG}" + env.FPK_BASENAME = "dy.net-${env.FNOS_VERSION}-x86_64.fpk" + env.FPK_PATH = "${env.WORKSPACE}/artifacts/${env.FPK_BASENAME}" + env.MANIFEST_PATH = "${env.WORKSPACE}/artifacts/douyin-${env.IMMUTABLE_TAG}-build-manifest.json" + currentBuild.displayName = "#${env.BUILD_NUMBER} ${env.IMMUTABLE_TAG}" + currentBuild.description = "fnOS ${env.FNOS_VERSION}" + } + sh ''' + set -euo pipefail + test "$(uname -m)" = x86_64 + for command_name in git sudo docker curl python3 npm node sha256sum tar; do command -v "$command_name" >/dev/null; done + test -x "$DOTNET" && test -x "$FNPACK_BIN" + mkdir -p "$DOCKER_CONFIG" "$NPM_CONFIG_CACHE" "$NUGET_PACKAGES" "$DOTNET_CLI_HOME" \ + "$DYSYNC_BUILD_TMPDIR" "$DYSYNC_VERIFY_TMPDIR" "$DYSYNC_SMOKE_TMPDIR" artifacts + chmod 700 "$DOCKER_CONFIG" + available_kb=$(df -Pk "$WORKSPACE" | awk 'NR == 2 { print $4 }') + test "$available_kb" -ge 3145728 + ''' + } + } + + stage('Restore And Test') { + steps { + sh ''' + set -euo pipefail + npm ci --prefix app + npm run test:mobile-contract --prefix app + npm run build --prefix app + "$DOTNET" restore tests/dy.net.Tests/dy.net.Tests.csproj --disable-parallel + "$DOTNET" test tests/dy.net.Tests/dy.net.Tests.csproj -c Release --no-restore \ + --logger 'console;verbosity=normal' /maxcpucount:1 + ''' + } + } + + stage('Build And Smoke fnOS x64') { + steps { + sh ''' + set -euo pipefail + PACKAGE_VERSION="$FNOS_VERSION" FNPACK="$FNPACK_BIN" DOTNET="$DOTNET" \ + ./scripts/build-fnos-package.sh artifacts + test -s "$FPK_PATH" && test -s "$FPK_PATH.sha256" + ./scripts/verify-fnos-package.sh "$FPK_PATH" "$FNOS_VERSION" + ./scripts/smoke-fnos-package.sh "$FPK_PATH" "$DYSYNC_SMOKE_TMPDIR" + ''' + } + } + + stage('Build And Smoke Docker amd64') { + steps { + withCredentials([usernamePassword(credentialsId: "${NODE_CREDENTIALS}", usernameVariable: 'JENKINS_NODE_USERNAME', passwordVariable: 'JENKINS_NODE_PASSWORD')]) { + sh ''' + set -euo pipefail + ./scripts/ci-docker.sh buildx rm -f "$BUILDER_NAME" >/dev/null 2>&1 || true + ./scripts/ci-docker.sh buildx create --name "$BUILDER_NAME" --driver docker-container --driver-opt network=host --use + ./scripts/ci-docker.sh buildx inspect "$BUILDER_NAME" --bootstrap + ./scripts/ci-docker.sh buildx build --builder "$BUILDER_NAME" --platform linux/amd64 --network host \ + --progress=plain --provenance=false --no-cache --load \ + --build-arg HTTP_PROXY="$HTTP_PROXY_URL" --build-arg HTTPS_PROXY="$HTTP_PROXY_URL" \ + --build-arg NO_PROXY="$NO_PROXY_HOSTS" -t "$IMAGE_REF" . + smoke_name="douyin-smoke-$BUILD_NUMBER" + mkdir -p .ci/docker-smoke + ./scripts/ci-docker.sh rm -f "$smoke_name" >/dev/null 2>&1 || true + ./scripts/ci-docker.sh run -d --name "$smoke_name" -p 127.0.0.1::10101 \ + -v "$WORKSPACE/.ci/docker-smoke:/data" "$IMAGE_REF" + host_port=$(./scripts/ci-docker.sh port "$smoke_name" 10101/tcp | sed -n 's/.*://p' | head -n 1) + test -n "$host_port" + ready=0 + for attempt in $(seq 1 60); do + if curl -fsS "http://127.0.0.1:$host_port/" >/dev/null; then ready=1; break; fi + sleep 2 + done + test "$ready" = 1 || { ./scripts/ci-docker.sh logs "$smoke_name" || true; exit 1; } + ./scripts/ci-docker.sh restart "$smoke_name" >/dev/null + sleep 3 + curl -fsS "http://127.0.0.1:$host_port/" >/dev/null + ./scripts/ci-docker.sh rm -f "$smoke_name" + ./scripts/ci-docker.sh buildx prune --builder "$BUILDER_NAME" --all --force + ''' + } + } + } + + stage('Push Immutable And Describe') { + steps { + withCredentials([ + usernamePassword(credentialsId: "${NODE_CREDENTIALS}", usernameVariable: 'JENKINS_NODE_USERNAME', passwordVariable: 'JENKINS_NODE_PASSWORD'), + usernamePassword(credentialsId: "${HARBOR_CREDENTIALS}", usernameVariable: 'HARBOR_USERNAME', passwordVariable: 'HARBOR_PASSWORD') + ]) { + sh ''' + set -euo pipefail + auth=$(printf '%s:%s' "$HARBOR_USERNAME" "$HARBOR_PASSWORD" | base64 -w0) + printf '{"auths":{"%s":{"auth":"%s"}}}\n' "$REGISTRY_URL" "$auth" >"$DOCKER_CONFIG/config.json" + chmod 600 "$DOCKER_CONFIG/config.json" + ./scripts/ci-docker.sh push "$IMAGE_REF" + digest=$(./scripts/ci-docker.sh image inspect --format '{{index .RepoDigests 0}}' "$IMAGE_REF" | sed -n 's/.*@//p') + test -n "$digest"; export digest + fpk_sha=$(awk '{print $1}' "$FPK_PATH.sha256"); export fpk_sha + python3 - <<'PY' +import json, os, pathlib, subprocess +path = pathlib.Path(os.environ['MANIFEST_PATH']) +payload = { + 'project': 'douyin', 'buildNumber': os.environ['BUILD_NUMBER'], + 'commit': subprocess.check_output(['git','rev-parse','HEAD'], text=True).strip(), + 'productVersion': os.environ['APP_VERSION'], 'fnosVersion': os.environ['FNOS_VERSION'], + 'fpk': pathlib.Path(os.environ['FPK_PATH']).name, 'fpkSha256': os.environ['fpk_sha'], + 'image': os.environ['IMAGE_REF'], 'digest': os.environ['digest'], 'platform': 'linux/amd64' +} +path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') +PY + sha256sum "$MANIFEST_PATH" >"$MANIFEST_PATH.sha256" + ''' + } + } + } + + stage('Upload Artifacts') { + steps { + withCredentials([usernamePassword(credentialsId: "${OPENLIST_CREDENTIALS}", usernameVariable: 'OPENLIST_USERNAME', passwordVariable: 'OPENLIST_PASSWORD')]) { + sh ''' + set -euo pipefail + for artifact in "$FPK_PATH" "$FPK_PATH.sha256" "$MANIFEST_PATH" "$MANIFEST_PATH.sha256"; do + ./scripts/upload-openlist-artifact.sh "$artifact" "$OPENLIST_REMOTE_DIR" + done + ''' + } + } + } + + stage('Publish Stable Image Tags') { + steps { + withCredentials([usernamePassword(credentialsId: "${NODE_CREDENTIALS}", usernameVariable: 'JENKINS_NODE_USERNAME', passwordVariable: 'JENKINS_NODE_PASSWORD')]) { + sh ''' + set -euo pipefail + for tag in "$APP_VERSION" latest; do + ./scripts/ci-docker.sh tag "$IMAGE_REF" "$IMAGE_REPO:$tag" + ./scripts/ci-docker.sh push "$IMAGE_REPO:$tag" + done + ''' + } + } + } + } + + post { + success { archiveArtifacts artifacts: 'artifacts/*.fpk,artifacts/*.sha256,artifacts/*.json', fingerprint: true } + always { + withCredentials([usernamePassword(credentialsId: "${NODE_CREDENTIALS}", usernameVariable: 'JENKINS_NODE_USERNAME', passwordVariable: 'JENKINS_NODE_PASSWORD')]) { + sh ''' + set +e + ./scripts/ci-docker.sh rm -f "douyin-smoke-$BUILD_NUMBER" >/dev/null 2>&1 || true + ./scripts/ci-docker.sh buildx rm -f "$BUILDER_NAME" >/dev/null 2>&1 || true + for tag in "$IMMUTABLE_TAG" "$APP_VERSION" latest; do ./scripts/ci-docker.sh rmi "$IMAGE_REPO:$tag" >/dev/null 2>&1 || true; done + : >"$DOCKER_CONFIG/config.json" 2>/dev/null || true + ''' + } + cleanWs(deleteDirs: true, notFailBuild: true) + } + } +} diff --git a/Program.cs b/Program.cs index 68be463..3bc1fd9 100644 --- a/Program.cs +++ b/Program.cs @@ -4,13 +4,15 @@ using dy.net.utils; using Serilog; using System.Reflection; using System.Text; +using Microsoft.AspNetCore.DataProtection; +using dy.net.storage; namespace dy.net { public class Program { // 常量定义 - private static readonly string DefaultListenUrl = "http://*:10101"; + private static readonly string DefaultListenUrl = Environment.GetEnvironmentVariable("DYSYNC_LISTEN_URL") ?? "http://*:10101"; private const string SpaRootPath = "app/dist"; private const string SpaSourcePath = "app/"; private const string SwaggerDocTitle = "dysync.net WebApi Docs"; @@ -29,6 +31,9 @@ namespace dy.net ConfigureHost(builder, isDevelopment); // 配置服务--数据库保存路径从命令行参数传入的第一个参数 string dbPath = args.Length > 0 ? args[0] : ""; + // The backup is deliberately completed before DI can construct SqlSugar and run CodeFirst. + // A failed schema update therefore leaves the previous database, WAL/SHM and key ring recoverable. + var upgradeBackup = DatabaseUpgradeBackup.Prepare(dbPath); ConfigureServices(builder.Services, builder.Configuration, builder.Environment, dbPath); // 构建应用 var app = builder.Build(); @@ -37,6 +42,7 @@ namespace dy.net Log.Debug($"dy.sync app is started successfully on {DefaultListenUrl}"); // 初始化应用服务 await InitApplicationServices(app, isDevelopment, dbPath); + upgradeBackup.MarkSchemaReady(); Console.WriteLine(); await app.RunAsync(); @@ -124,6 +130,16 @@ namespace dy.net //PrintApp(); services.AddSingleton(new Appsettings(config)); + + // WebDAV 密码使用 Data Protection 加密；密钥与 SQLite 一同持久化，容器重启后仍可解密。 + var databaseRoot = string.IsNullOrWhiteSpace(dbPath) + ? Path.Combine(Environment.CurrentDirectory, "db") + : Path.Combine(dbPath, "db"); + var keyRingPath = Path.Combine(databaseRoot, "keys"); + Directory.CreateDirectory(keyRingPath); + services.AddDataProtection() + .SetApplicationName("dysync.net") + .PersistKeysToFileSystem(new DirectoryInfo(keyRingPath)); // 雪花ID生成器 services.AddSnowFlakeId(options => options.WorkId = new Random().Next(1, 100)); @@ -143,6 +159,17 @@ namespace dy.net services.AddServicesFromNamespace("dy.net.repository") .AddServicesFromNamespace("dy.net.service"); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddSingleton(); + services.AddScoped(); + services.AddHostedService(); + services.AddHostedService(); + services.AddHostedService(); + services.AddHostedService(); + services.AddHostedService(); + services.AddScoped(); // SPA静态文件支持 @@ -263,8 +290,10 @@ namespace dy.net catch (Exception ex) { Serilog.Log.Error(ex, "Failed to initialize services on startup"); + // Never listen with a partially migrated schema. fnOS can retain/restore the pre-upgrade backup. + throw; } } } -} \ No newline at end of file +} diff --git a/Properties/AssemblyInfo.cs b/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..c5ecbe3 --- /dev/null +++ b/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("dy.net.Tests")] diff --git a/README.md b/README.md index 9e57f62..04ff23d 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,9 @@ - [1.1 提取抖音 Cookie](#11-提取抖音-cookie) - [1.2 提取 `sec_user_id`（个人/指定博主）](#12-提取-sec_user_id个人指定博主) - [2. 路径映射规则（核心！错配会导致无法访问/数据丢失）](#2-路径映射规则核心错配会导致无法访问数据丢失) + - [2.1 直接同步到 WebDAV（AList/OpenList）](#21-直接同步到-webdavalistopenlist) + - [2.2 升级后将本地视频迁移到 WebDAV](#22-升级后将本地视频迁移到-webdav) + - [2.3 WebDAV 自动化与真实服务测试](#23-webdav-自动化与真实服务测试) - [3. 默认账号密码（首次登录用）](#3-默认账号密码首次登录用) - [4. 运行方式（推荐 Docker Compose）](#4-运行方式推荐-docker-compose) - [镜像版本](#镜像版本) @@ -80,6 +83,92 @@ Cookie 及 `sec_user_id` 是同步功能的核心，需严格按步骤获取， --- +### 2.1 通过 OpenList 原生 API 同步到远端 + +`0.2.22` 不再使用 WebDAV 上传新内容。应用先把完整文件写入本地共享中转目录，再调用 OpenList `/api/fs/copy` 在服务端复制，并通过暂存目录、移动和长度校验原子提升到最终路径。OpenList 返回外部云盘签名地址时不会向该域名转发 OpenList Token，避免移动云 Range 校验返回 400，并防止登录凭据跨主机发送。 + +`0.2.23` 将“检测存储”固定显示在任务中心顶部；存储正常时可随时主动检测，熔断后会自动变为“重新检测存储”。移动端按钮独占一行，不再依赖警告框的操作区域。 + +`0.2.24` 修复 OpenList 新视频目录尚未创建时被误判为存储故障的问题。应用会继续保留已有目录的服务端真实大小写，并把缺失的尾部路径交给安全传输流程创建。抖音作品列表和媒体连接超时会进行两次带退避的有限重试；切换到 OpenList 后遗留的旧 WebDAV 同步任务会在启动恢复阶段安全终结并隐藏，不删除视频记录或媒体文件。普通同步成功切换存储但旧本地文件清理失败时，任务条目会显示“重试清理旧文件”，再次验证远端主媒体后才执行清理。 + +`0.2.25` 补齐 OpenList 对象检查的强制刷新分支：远端尚未创建的新视频目录会稳定返回“不存在”并进入下载，不再在 `ExistsAsync` 的第二次检查中误报 `object not found`。新建目录后会进行总计最多约 7.5 秒的有限刷新确认，以兼容远端云盘的可见性延迟；若服务端确实自动改名，仍会停止写入以避免生成重复目录。 + +当底层云盘目录大小写不敏感时，应用会根据 OpenList 目录列表解析真实名称。例如逻辑路径 `/collect/Kk` 会稳定复用现有 `/collect/KK`，不会再触发 `Kk_日期_时间` 自动改名。任务中心的“修复异常目录”会严格筛选时间后缀候选，逐个检查并等待人工确认；删除前还会复检，只删除空目录。 + +1. 在 fnOS 或宿主机创建应用可写的本地中转目录。 +2. 在 OpenList 中添加一个本地存储驱动，让“源挂载目录”指向同一个物理目录。应用与 OpenList 看到的路径名称可以不同，但内容必须完全对应。 +3. 进入“系统配置 → 媒体存储”，选择 **OpenList 原生 API**。地址填写站点根地址，例如 `http://192.168.1.2:5244`，不要填写 `/dav`。 +4. 填写本地中转目录、OpenList 源挂载目录和目标基础目录，然后执行“测试完整复制链路”。测试会验证登录、本地文件可见、服务端复制、Range 读取和删除。 +5. 测试通过后启用 OpenList，再进入“抖音授权”配置收藏、喜欢、关注、合集和短剧路径。这些字段都是目标基础目录之后的相对路径；留空保存时会按账号生成默认值。 + +注意事项： + +- OpenList 账号需要源挂载目录的读取权限，以及目标基础目录的读取、写入、移动、重命名和删除权限。 +- 新写入不会调用 `/api/fs/put` 或 WebDAV。服务端复制完成并确认最终文件长度后，本地中转副本才会清理；失败任务会持久化并按退避规则恢复。 +- 切换存储模式只影响之后的新同步内容，不会自动移动或删除历史文件。历史 WebDAV 驱动仅用于旧记录继续播放、删除和回滚。 +- OpenList 密码使用 Data Protection 加密。密钥位于数据库目录的 `keys` 子目录，因此 Docker 部署必须持续映射并备份 `/app/db`。 +- 图文和动态视频仍会先在本地完成 FFmpeg 合成，再进入同一 OpenList 中转与服务端复制流程。 + +### 2.2 升级后将旧视频接管到 OpenList + +`0.2.25` 支持从官方 `0.2.x` 原地升级，保留数据库、Cookie、配置、Data Protection 密钥及历史媒体路径。系统配置会分别显示本地、历史 WebDAV、OpenList 和“不在当前存储”的记录数量；旧 WebDAV 迁移任务只保留审计信息，可直接归档隐藏。 + +> 升级 → 配置并测试 OpenList → 启用 OpenList → 迁移预检 → 手动创建任务 → 后台迁移 → 选择回滚或清理旧文件 + +1. 安装 `0.2.25 x86_64` FPK 覆盖升级。升级不会自动迁移视频，也不会覆盖 fnOS 数据共享目录；启动前会备份 SQLite、WAL/SHM 和 Data Protection 密钥，并只保留最近三份升级备份。 +2. 按上一节配置共享中转目录和 OpenList，完成完整复制链路测试后再启用。此后新同步内容直接写入 OpenList，旧记录仍按原存储类型工作。 +3. 打开“系统配置 → 存储迁移”执行预检。页面会统计本地记录、历史 WebDAV、可直接接管、需复制或回源、缺失文件、总字节数、无效 Cookie、未配置路径和目标冲突；存在冲突时不会创建任务。 +4. 同一 OpenList 实例中可见且长度匹配的历史 WebDAV 文件会直接改标接管，不会重复复制。其余记录优先读取安全范围内的本地文件；本地主媒体缺失时才尝试记录 URL 和抖音全量列表回源。 +5. 选择并发数 `1–3`（默认 `1`）并手动创建任务。升级、保存配置和启用 OpenList 都不会自动启动迁移。配置地址、源挂载、基础目录或账号变化时，任务会暂停并要求重新预检。 +6. 任务支持暂停、恢复、取消和失败项重试；应用重启后会恢复中断项。单条记录只有在远端最终文件通过长度校验且数据库事务成功后才会改为 OpenList，失败项仍指向原存储。 +7. 任务结束后可以整批回滚或清理成功项。清理前会再次检查数据库与远端文件；本地来源只删除不再被引用的旧文件，直接接管的 WebDAV 文件不会被误删。清理开始后不再允许回滚。 +8. 已结束且无需继续清理或回滚的迁移历史可在系统配置或任务中心隐藏；归档只影响显示，不删除视频、文件或审计条目。 + +建议在抽样播放确认无误后再清理旧文件，并始终保留 `/app/db` 与原媒体目录的外部备份。 + +### 2.3 媒体来源 403、429 与重新授权 + +任务中心会把抖音/CDN 来源错误与 OpenList 存储错误分开显示。任务条目只记录安全的来源域名、HTTP 状态和重试时间，不保存或输出带签名的完整媒体 URL、响应正文与 Cookie。 + +- 下载会优先尝试所选清晰度，再去重轮换其他清晰度地址；403、404 和 410 会立即切换候选地址。 +- 同一抖音授权连续 3 个作品的全部候选地址均返回 403 时，只暂停该授权 15 分钟，其他授权继续同步。冷却结束后后台只用最早的等待条目探测一次。 +- 探测成功会自动恢复该授权的其余等待条目；探测仍返回 403 或请求直接返回 401 时，任务中心和“抖音授权”页会提示重新授权。 +- 保存新的 Cookie 会清除来源锁并安排一次探测。429 会遵循服务端 `Retry-After` 进入冷却，但不会直接判定 Cookie 失效。 +- 404/410 只标记当前作品失败，通常表示作品下架、私密或签名地址已失效，不会暂停整个授权，也不会触发存储熔断。 + +如果任务中心显示“需要重新授权”，请进入“抖音授权”保留原入口并更新 Cookie；无需修改 OpenList 配置，也不要删除已有视频记录或文件。 + +### 2.4 关注博主直播监测与邮箱通知 + +- 在“关注列表”中为每个博主独立开启“直播监测”。开启后会立即检查一次，之后后台每 5 分钟检查；单个博主也可手动刷新，30 秒内重复点击不会再次请求抖音。 +- 页面每分钟读取一次本地状态，不会因此额外请求抖音。直播中可直接进入直播间；检查失败时保留上次成功状态并显示失败或过期提示。 +- 同一授权账号下的博主串行检查并加入随机间隔；遇到 403、429 或验证响应时按账号进入 15–360 分钟递增冷却，避免持续请求扩大风控影响。 +- 在“系统配置 → 邮箱通知”配置 SMTP 地址、端口、安全方式、账号、授权码、发件人和收件人。支持无加密、STARTTLS 和 SSL/TLS，建议优先使用邮箱服务商提供的授权码并先发送测试邮件。 +- SMTP 密码使用 Data Protection 加密保存，不会通过读取配置接口返回。密码留空保存表示继续使用原密码；可填写多个收件人，使用逗号、分号或换行分隔。 +- 全局邮箱启用并配置完整后，可在每个博主卡片上独立开启“开播邮件”。同一 `web_rid` 或直播房间只通知一次，邮件失败不会覆盖已获取的直播状态，15 分钟后才会重试；检测到下播后才会为下一场直播重新准备通知。 + +### 2.5 OpenList 自动化与真实服务测试 + +本地自动化测试覆盖 OpenList Token 缓存与 401 刷新、Unicode 路径、服务端复制、Range 读取、重启恢复、数据库升级，以及历史 WebDAV 兼容行为： + +```bash +dotnet test tests/dy.net.Tests/dy.net.Tests.csproj +``` + +AList 和 OpenList 的历史 WebDAV 兼容实测默认跳过；需要验证旧记录读取与删除时，可分别设置以下环境变量： + +| AList | OpenList | 说明 | +|---|---|---| +| `DYSYNC_TEST_ALIST_ENDPOINT` | `DYSYNC_TEST_OPENLIST_ENDPOINT` | WebDAV 地址，例如 `https://host/dav` | +| `DYSYNC_TEST_ALIST_BASE_PATH` | `DYSYNC_TEST_OPENLIST_BASE_PATH` | 专用测试目录，必须是包含 `dysync-test` 的非根路径 | +| `DYSYNC_TEST_ALIST_USERNAME` | `DYSYNC_TEST_OPENLIST_USERNAME` | 测试账号用户名 | +| `DYSYNC_TEST_ALIST_PASSWORD` | `DYSYNC_TEST_OPENLIST_PASSWORD` | 测试账号密码 | +| `DYSYNC_TEST_ALIST_ALLOW_INVALID_CERTIFICATE` | `DYSYNC_TEST_OPENLIST_ALLOW_INVALID_CERTIFICATE` | 可选，仅可信内网自签证书设置为 `true` | + +历史兼容测试只会在指定基础目录下创建随机子目录，验证完成后自动删除；不会读取网站中保存的生产配置，也不会输出密码。OpenList 原生写入还依赖应用与 OpenList 共享同一物理中转目录，因此请直接使用“系统配置 → 媒体存储 → 测试完整复制链路”完成部署环境实测。 + +--- + ## 3. 默认账号密码（首次登录用） 首次访问后台管理页面时，使用以下默认账号密码： - **用户名**：`douyin` @@ -301,4 +390,4 @@ services: ## 4. 声明的修改与生效 - 本免责声明的最终解释权归项目开发者所有，开发者保留随时修改本声明的权利。 - 修改后的声明将在项目仓库中更新，建议使用者定期查阅。 -- 一旦使用本项目的代码、功能或相关资源，即视为您已充分理解并同意本免责声明的全部条款。如不同意，请立即停止使用。 \ No newline at end of file +- 一旦使用本项目的代码、功能或相关资源，即视为您已充分理解并同意本免责声明的全部条款。如不同意，请立即停止使用。 diff --git a/app/index.html b/app/index.html index 302ff6d..1b111e9 100644 --- a/app/index.html +++ b/app/index.html @@ -35,7 +35,7 @@ } /* 2. Vue 挂载后：#stepin-app 非空（包含 Vue 组件），覆盖为普通块级元素，解除约束 */ - #stepin-app:not(:has(.breath-ring)) { + #stepin-app.app-mounted { /* 核心：取消 flex 布局，改为普通块级元素 */ display: block !important; /* 覆盖原有 flex 相关属性，解除居中约束 */ @@ -146,4 +146,4 @@ - \ No newline at end of file + diff --git a/app/package.json b/app/package.json index d1be31d..946c355 100644 --- a/app/package.json +++ b/app/package.json @@ -1,12 +1,13 @@ { "name": "stepin-template", "private": true, - "version": "1.2.0-preview", + "version": "0.2.25", "type": "module", "scripts": { "dev": "vite", "api": "node service/index.js", "build": "vue-tsc --noEmit && vite build", + "test:mobile-contract": "node --test tests/mobile-contract.test.mjs", "preview": "vite preview", "deploy": "yarn build && gh-pages -d dist -b pages -r https://gitee.com/stepui/stepin-template.git", "deploy:github": "yarn build --mode github && gh-pages -d dist -b master -r git@github.com:stepui/stepui.github.io.git", diff --git a/app/src/App.vue b/app/src/App.vue index b2d021c..f5647cd 100644 --- a/app/src/App.vue +++ b/app/src/App.vue @@ -16,11 +16,13 @@ + + + + diff --git a/app/src/components/OpenListDirectoryRepairModal.vue b/app/src/components/OpenListDirectoryRepairModal.vue new file mode 100644 index 0000000..6262dfd --- /dev/null +++ b/app/src/components/OpenListDirectoryRepairModal.vue @@ -0,0 +1,193 @@ + + + + + diff --git a/app/src/components/layout/MobileBottomNav.vue b/app/src/components/layout/MobileBottomNav.vue new file mode 100644 index 0000000..6b94311 --- /dev/null +++ b/app/src/components/layout/MobileBottomNav.vue @@ -0,0 +1,210 @@ + + + + + diff --git a/app/src/components/layout/MobileTopBar.vue b/app/src/components/layout/MobileTopBar.vue new file mode 100644 index 0000000..bead875 --- /dev/null +++ b/app/src/components/layout/MobileTopBar.vue @@ -0,0 +1,68 @@ + + + + + diff --git a/app/src/main.ts b/app/src/main.ts index 493a87c..371144b 100644 --- a/app/src/main.ts +++ b/app/src/main.ts @@ -24,3 +24,4 @@ app.config.errorHandler = function (err) { console.error('未捕获的异常，', err); }; app.mount('#stepin-app'); +document.getElementById('stepin-app')?.classList.add('app-mounted'); diff --git a/app/src/pages/cok/CookieTable.vue b/app/src/pages/cok/CookieTable.vue index 6f04a75..d1b011c 100644 --- a/app/src/pages/cok/CookieTable.vue +++ b/app/src/pages/cok/CookieTable.vue @@ -1,7 +1,7 @@