diff --git a/Controllers/CollectController.cs b/Controllers/CollectController.cs index 6b3dc42..280f1a1 100644 --- a/Controllers/CollectController.cs +++ b/Controllers/CollectController.cs @@ -46,19 +46,6 @@ namespace dy.net.Controllers pageSize = dto.PageSize }); } - /// - /// 重新同步-单次 - /// - /// - [HttpGet("sync")] - public async Task SyncFollowList() - { - //后台异步 - _douyinQuartzJobService.StartFollowJobOnceAsync(); - await Task.Delay(1000); - return ApiResult.Success(); - } - /// /// 修改关注同步状态 diff --git a/app/package.json b/app/package.json index 0431eff..d1be31d 100644 --- a/app/package.json +++ b/app/package.json @@ -52,6 +52,7 @@ "autoprefixer": "^10.4.7", "babel-preset-es2015": "^6.24.1", "babel-register": "^6.26.0", + "baseline-browser-mapping": "^2.9.15", "body-parser": "^1.20.1", "crypto-js": "^4.2.0", "gh-pages": "^3.1.0", diff --git a/job/DouYinCollectSyncJob.cs b/job/DouYinCollectSyncJob.cs index e73c990..dd64ab2 100644 --- a/job/DouYinCollectSyncJob.cs +++ b/job/DouYinCollectSyncJob.cs @@ -9,14 +9,9 @@ namespace dy.net.job { public class DouyinCollectSyncJob : DouyinBasicSyncJob { - public DouyinCollectSyncJob( - DouyinCookieService douyinCookieService, - DouyinHttpClientService douyinHttpClientService, - DouyinVideoService douyinVideoService, - DouyinCommonService douyinCommonService,DouyinFollowService douyinFollowService,DouyinMergeVideoService douyinMergeVideoService) - : base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService,douyinFollowService, douyinMergeVideoService) { } - - + public DouyinCollectSyncJob(DouyinCookieService douyinCookieService, DouyinHttpClientService douyinHttpClientService, DouyinVideoService douyinVideoService, DouyinCommonService douyinCommonService, DouyinFollowService douyinFollowService, DouyinMergeVideoService douyinMergeVideoService, DouyinCollectCateService douyinCollectCateService) : base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService, douyinCollectCateService) + { + } protected override VideoTypeEnum VideoType => VideoTypeEnum.dy_collects; protected override async Task BeforeProcessCookies() @@ -24,30 +19,14 @@ namespace dy.net.job var now = DateTime.Now; if (now.Hour == 1 && now.Minute < 30) { - //douyinCommonService.UpdateAllCookieSyncedToZero(); - //顺手清理下日志文件 LogFileCleaner.CleanOldLogFiles(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "logs"), 1); await Task.Delay(200); } } - protected override async Task> GetValidCookies() - { - return await douyinCookieService.GetAllOpendAsync(x => !string.IsNullOrWhiteSpace(x.SavePath)); - } - - protected override bool IsCookieValid(DouyinCookie cookie) + protected override async Task FetchVideoData(DouyinCookie cookie, string cursor, DouyinFollowed followed, DouyinCollectCate cate) { - return !string.IsNullOrWhiteSpace(cookie.Cookies)&& !string.IsNullOrWhiteSpace(cookie.SavePath); - } - - protected override async Task FetchVideoData(DouyinCookie cookie, string cursor, DouyinFollowed followed,DouyinCollectItem collectItem=null) - { - if (cookie.UseCollectFolder && collectItem != null && !string.IsNullOrWhiteSpace(collectItem.CollectsId)) - { - return await douyinHttpClientService.SyncCollectVideosByCollectId(cursor, count, cookie.Cookies, collectItem); - } return await douyinHttpClientService.SyncCollectVideos(cursor, count, cookie.Cookies); } @@ -56,16 +35,6 @@ namespace dy.net.job return data != null && data.HasMore == 1 && cookie.CollHasSyncd == 0; } - protected override string GetNextCursor(DouyinVideoInfoResponse data) - { - return data?.Cursor ?? "0"; - } - - protected override string GetAuthorAvatarBasePath(DouyinCookie cookie) - { - return Path.Combine(cookie.SavePath, "author"); - } - protected override async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount,DouyinFollowed followed) { if (syncCount > 0) @@ -81,31 +50,15 @@ namespace dy.net.job } - protected override VideoEntityDifferences GetVideoEntityDifferences(DouyinCookie cookie, Aweme item) - { - return new VideoEntityDifferences(); - } - protected override string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed, DouyinCollectItem collectItem) + protected override string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed, DouyinCollectCate cate) { - if(collectItem!=null&&cookie.UseCollectFolder &&!string.IsNullOrWhiteSpace(collectItem.CollectsId)) - { - var safeCollectName = DouyinFileNameHelper.SanitizeLinuxFileName(collectItem.CollectsName, "", true); - var folderCollect = Path.Combine(cookie.SavePath, safeCollectName); - var safeVideoDesc = DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId, true); - var folder = Path.Combine(folderCollect, safeVideoDesc); - if (!Directory.Exists(folder)) Directory.CreateDirectory(folder); - return folder; - } - else - { - var (tag1, _, _) = GetVideoTags(item); - var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : DouyinFileNameHelper.SanitizeLinuxFileName(tag1, "", true); - var folder = Path.Combine(cookie.SavePath, safeTag1, $"{DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId, true)}"); - if (!Directory.Exists(folder)) Directory.CreateDirectory(folder); - return folder; - } - + var (tag1, _, _) = GetVideoTags(item); + var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : DouyinFileNameHelper.SanitizeLinuxFileName(tag1, "", true); + var folder = Path.Combine(cookie.SavePath, safeTag1, $"{DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId, true)}"); + if (!Directory.Exists(folder)) Directory.CreateDirectory(folder); + return folder; + } } } \ No newline at end of file diff --git a/job/DouYinFavoritSyncJob.cs b/job/DouYinFavoritSyncJob.cs index 56bda3a..89beef8 100755 --- a/job/DouYinFavoritSyncJob.cs +++ b/job/DouYinFavoritSyncJob.cs @@ -8,29 +8,19 @@ namespace dy.net.job { public class DouyinFavoritSyncJob : DouyinBasicSyncJob { - public DouyinFavoritSyncJob( - DouyinCookieService douyinCookieService, - DouyinHttpClientService douyinHttpClientService, - DouyinVideoService douyinVideoService, - DouyinCommonService douyinCommonService,DouyinFollowService douyinFollowService,DouyinMergeVideoService douyinMergeVideoService) - : base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService) { } + public DouyinFavoritSyncJob(DouyinCookieService douyinCookieService, DouyinHttpClientService douyinHttpClientService, DouyinVideoService douyinVideoService, DouyinCommonService douyinCommonService, DouyinFollowService douyinFollowService, DouyinMergeVideoService douyinMergeVideoService, DouyinCollectCateService douyinCollectCateService) : base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService, douyinCollectCateService) + { + } protected override VideoTypeEnum VideoType => VideoTypeEnum.dy_favorite; - protected override async Task> GetValidCookies() + protected override async Task> GetSyncCookies() { - return await douyinCookieService.GetAllOpendAsync(x=> !string.IsNullOrWhiteSpace(x.FavSavePath)); + return await douyinCookieService.GetOpendCookiesAsync(x=> !string.IsNullOrWhiteSpace(x.FavSavePath)&&!string.IsNullOrWhiteSpace(x.SecUserId)); } - protected override bool IsCookieValid(DouyinCookie cookie) - { - return !string.IsNullOrWhiteSpace(cookie.Cookies) && cookie.Cookies.Length >= 1000 && - !string.IsNullOrWhiteSpace(cookie.FavSavePath) && - !string.IsNullOrWhiteSpace(cookie.SecUserId) && cookie.SecUserId.Length >= 10; - } - - protected override async Task FetchVideoData(DouyinCookie cookie, string cursor,DouyinFollowed followed, DouyinCollectItem collectId) + protected override async Task FetchVideoData(DouyinCookie cookie, string cursor,DouyinFollowed followed, DouyinCollectCate cate) { return await douyinHttpClientService.SyncFavoriteVideos(count, cursor, cookie.SecUserId, cookie.Cookies); } @@ -40,16 +30,6 @@ namespace dy.net.job return data != null && data.HasMore == 1 && cookie.FavHasSyncd == 0; } - protected override string GetNextCursor(DouyinVideoInfoResponse data) - { - return data?.MaxCursor ?? "0"; - } - - - protected override string GetAuthorAvatarBasePath(DouyinCookie cookie) - { - return Path.Combine(cookie.FavSavePath, "author"); - } protected override async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount, DouyinFollowed followed) { @@ -65,7 +45,7 @@ namespace dy.net.job } } - protected override string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed, DouyinCollectItem collectItem) + protected override string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed,DouyinCollectCate cate) { var (tag1, _, _) = GetVideoTags(item); var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : DouyinFileNameHelper.SanitizeLinuxFileName(tag1,"",true); diff --git a/job/DouyinBasicSyncJob.cs b/job/DouyinBasicSyncJob.cs index 7422132..563210f 100644 --- a/job/DouyinBasicSyncJob.cs +++ b/job/DouyinBasicSyncJob.cs @@ -24,17 +24,14 @@ namespace dy.net.job /// 抖音Cookie服务，用于获取和管理用户Cookie /// protected readonly DouyinCookieService douyinCookieService; - /// /// 抖音HTTP客户端服务，用于发送HTTP请求 /// protected readonly DouyinHttpClientService douyinHttpClientService; - /// /// 抖音视频服务，用于视频信息的数据库操作 /// protected readonly DouyinVideoService douyinVideoService; - /// /// 抖音通用服务，用于获取应用配置等 /// @@ -43,20 +40,20 @@ namespace dy.net.job /// 抖音关注列表 /// private readonly DouyinFollowService douyinFollowService; - /// /// 图文合成视频 /// private readonly DouyinMergeVideoService douyinMergeVideoService; /// + /// 收藏夹、短剧、合集 + /// + private readonly DouyinCollectCateService douyinCollectCateService; + /// /// 随机数生成器，用于生成随机延迟，模拟人类操作 /// protected readonly Random _random = new Random(); - - - /// - /// 每页请求的视频数量，可通过配置文件修改 + /// 每页请求的视频数量.不可修改 /// protected string count = "18"; @@ -79,12 +76,6 @@ namespace dy.net.job #region 抽象属性 - /// - /// 任务类型名称，用于日志记录和区分不同的同步任务 - /// 子类必须实现此属性并返回具体的任务类型 - /// - //protected abstract string VideoType { get; } - protected abstract VideoTypeEnum VideoType { get; } #endregion @@ -100,13 +91,15 @@ namespace dy.net.job /// 抖音通用服务 /// 抖音关注的 /// 视频合成 + /// protected DouyinBasicSyncJob( DouyinCookieService douyinCookieService, DouyinHttpClientService douyinHttpClientService, DouyinVideoService douyinVideoService, DouyinCommonService douyinCommonService, DouyinFollowService douyinFollowService, - DouyinMergeVideoService douyinMergeVideoService) + DouyinMergeVideoService douyinMergeVideoService, + DouyinCollectCateService douyinCollectCateService) { this.douyinCookieService = douyinCookieService ?? throw new ArgumentNullException(nameof(douyinCookieService)); this.douyinHttpClientService = douyinHttpClientService ?? throw new ArgumentNullException(nameof(douyinHttpClientService)); @@ -114,6 +107,7 @@ namespace dy.net.job this.douyinCommonService = douyinCommonService ?? throw new ArgumentNullException(nameof(douyinCommonService)); this.douyinFollowService = douyinFollowService; this.douyinMergeVideoService = douyinMergeVideoService; + this.douyinCollectCateService = douyinCollectCateService; } #endregion @@ -135,22 +129,21 @@ namespace dy.net.job var config = douyinCommonService.GetConfig(); if (config == null) { - Log.Debug($"{VideoType}-未获取到系统配置，任务终止!!!"); + Log.Debug($"{VideoType}-系统配置错误，任务终止!!!"); return; } - // 在处理Cookie之前执行的预处理 await BeforeProcessCookies(); // 获取所有有效的Cookie - var cookies = await GetValidCookies(); + var cookies = await GetSyncCookies(); if (cookies == null || !cookies.Any()) { - Log.Debug($"{VideoType}-无有效cookie或{VideoType}已关闭同步，任务终止!!!"); + Log.Debug($"{VideoType}-Cookie无效或{VideoType}已关闭同步，任务终止!!!"); return; } - Log.Debug($"{VideoType}-共发现{cookies.Count}个有效的cookie，同步即将开始..."); + Log.Debug($"{VideoType}:共发现{cookies.Count}个有效Cookie，同步开始..."); // 遍历每个有效的Cookie，执行同步 foreach (var cookie in cookies) @@ -174,16 +167,9 @@ namespace dy.net.job /// 子类必须实现此方法，根据具体任务类型筛选有效的Cookie /// /// 有效的Cookie列表 - protected abstract Task> GetValidCookies(); - - /// - /// 检查指定的Cookie是否有效 - /// - /// 要检查的Cookie - /// 如果Cookie有效，则为true；否则为false - protected virtual bool IsCookieValid(DouyinCookie cookie) + protected virtual async Task> GetSyncCookies() { - return !string.IsNullOrWhiteSpace(cookie.Cookies); + return await douyinCookieService.GetOpendCookiesAsync(x => !string.IsNullOrWhiteSpace(x.SavePath)); } /// @@ -192,14 +178,13 @@ namespace dy.net.job /// /// 用户Cookie /// 分页游标，用于获取下一页数据 - /// 关注的人 - /// 收藏夹ID + /// 关注的人 + /// 自定义收藏夹、合集、短剧 /// 视频信息对象，包含视频列表和分页信息 - protected abstract Task FetchVideoData(DouyinCookie cookie, string cursor, DouyinFollowed uperUid , DouyinCollectItem collectId); + protected abstract Task FetchVideoData(DouyinCookie cookie, string cursor, DouyinFollowed followed, DouyinCollectCate cate); /// /// 判断是否应该继续同步下一页数据 - /// 子类必须实现此方法，根据API返回的分页信息判断是否还有更多数据 /// /// 用户Cookie /// 当前获取到的视频数据 @@ -209,23 +194,29 @@ namespace dy.net.job /// /// 获取下一页数据的游标 - /// 子类必须实现此方法，从当前视频数据中提取下一页的游标 /// /// 当前获取到的视频数据 /// 下一页数据的游标 - protected abstract string GetNextCursor(DouyinVideoInfoResponse data); + private static string GetNextCursor(DouyinVideoInfoResponse data) + { + return data?.Cursor ?? (data?.MaxCursor ?? "0"); + } /// /// 创建视频保存文件夹 - /// 子类必须实现此方法，根据具体的存储策略创建文件夹 /// /// 用户Cookie /// 视频信息 /// 关注用户 - /// 收藏夹Item + /// /// 应用配置 /// 创建的视频保存文件夹路径 - protected abstract string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed,DouyinCollectItem collectItem); + protected virtual string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed,DouyinCollectCate cate) + { + var folder = Path.Combine(cookie.SavePath, DouyinFileNameHelper.SanitizeLinuxFileName(cate.Name, item.AwemeId, true)); + if (!Directory.Exists(folder)) Directory.CreateDirectory(folder); + return folder; + } /// /// 获取视频文件名,默认就用id作文件名 @@ -246,21 +237,29 @@ namespace dy.net.job /// /// 用户Cookie /// 作者头像保存的基础路径 - protected abstract string GetAuthorAvatarBasePath(DouyinCookie cookie); + //protected abstract string GetAuthorAvatarBasePath(DouyinCookie cookie); /// /// 处理同步完成后的操作 - /// 子类必须实现此方法，可用于更新同步状态、发送通知等 /// /// 用户Cookie /// 本次同步成功的视频数量 /// /// 一个表示异步操作的任务 - protected abstract Task HandleSyncCompletion(DouyinCookie cookie, int syncCount, DouyinFollowed followed = null); + protected virtual async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount, DouyinFollowed followed = null) + { + if (syncCount > 0) + { + Serilog.Log.Debug($"{VideoType}-Cookie-[{cookie.UserName}],本次共同步成功{syncCount}条视频"); + } + else + { + Serilog.Log.Debug($"{VideoType}-Cookie-[{cookie.UserName}],没有可以同步的新视频"); + } + } /// /// 获取视频实体的差异信息 - /// 子类必须实现此方法，用于区分不同类型的视频（如普通视频、图片合成视频等） /// /// 用户Cookie /// 视频信息 @@ -294,98 +293,64 @@ namespace dy.net.job { try { - // 检查Cookie是否有效 - if (!IsCookieValid(cookie)) - { - Log.Debug($"{VideoType}-Cookie[{cookie.UserName}]无效，任务终止!!!"); - return; - } Log.Debug($"{VideoType}- Cookie-[{cookie.UserName}]开始同步..."); - //up主上传视频特殊处理 - if (VideoType == VideoTypeEnum.dy_follows) + switch (VideoType) { - int syncCount = 0; // 本次同步成功的视频数量 - string cursor = "0"; // 初始游标 - bool hasMore = true; // 是否还有更多数据 - - //查询关注列表开启了同步的关注 - var follows = await douyinFollowService.GetSyncFollows(cookie.MyUserId); - - var firstUp = follows?.Where(x => !string.IsNullOrWhiteSpace(x.SecUid)).FirstOrDefault(); - if (firstUp == null) - { - return; - } - if (string.IsNullOrWhiteSpace(firstUp.SecUid)) - { - Log.Debug($"{VideoType}-Cookie[{firstUp.UperName}]无效,没有sec_userid，任务终止!!!"); - return; - } - - - foreach (var followed in follows) - { - cursor = "0"; // 初始游标 - await GetViedos(cookie, config, syncCount, cursor, hasMore, followed); - hasMore = true; - // 处理同步完成后的操作 - await HandleSyncCompletion(cookie, syncCount, followed); - } - } - else - { - // 仅在视频类型为抖音收藏时执行对应逻辑 - if (VideoType == VideoTypeEnum.dy_collects) - { - // 统一声明变量，避免分散定义，提升可读性 - int syncCount = 0; // 本次同步成功的视频数量 - string cursor = "0"; // 初始游标 - bool hasMore = true; // 是否还有更多数据 - - // 处理「使用收藏文件夹」和「不使用收藏文件夹」的统一逻辑 - if (cookie.UseCollectFolder) + case VideoTypeEnum.dy_follows: { - // 获取收藏文件夹列表 - var folders = await douyinHttpClientService.SyncCollectFolderList(cookie.Cookies,""); - Log.Debug("获取收藏文件夹列表完成，共计{Count}个包含视频的收藏夹", folders?.CollectsList?.Count(x => x.TotalNumber > 0) ?? 0); - // 文件夹为空时，执行默认同步逻辑（与不使用文件夹逻辑一致） - if (folders?.CollectsList == null || !folders.CollectsList.Any(x => x.TotalNumber > 0)) + int syncCount = 0; // 本次同步成功的视频数量 + //查询关注列表开启了同步的关注 + var follows = await douyinFollowService.GetSyncFollows(cookie.MyUserId); + if (follows != null && follows.Any()) { - (syncCount, cursor, hasMore) = await GetViedos(cookie, config, syncCount, cursor, hasMore); - await HandleSyncCompletion(cookie, syncCount); - } - // 文件夹不为空时，遍历每个文件夹同步视频 - else - { - foreach (var item in folders.CollectsList.Where(x=>x.TotalNumber>0)) + foreach (var followed in follows) { - // 重置游标，每个文件夹独立从头同步 - string folderCursor = "0"; - await GetViedos(cookie, config, syncCount, folderCursor, hasMore, null, item); + string cursor = "0"; + bool hasMore = true; + (syncCount, cursor, hasMore) = await GetAndSaveViedos(cookie, config, syncCount, cursor, hasMore, followed); + await HandleSyncCompletion(cookie, syncCount, followed); + } + } + } + break; + case VideoTypeEnum.dy_favorite: + case VideoTypeEnum.dy_collects: + { + int syncCount = 0; + string cursor = "0"; + bool hasMore = true; + + (syncCount, cursor, hasMore) = await GetAndSaveViedos(cookie, config, syncCount, cursor, hasMore); + await HandleSyncCompletion(cookie, syncCount); + } + break; + + case VideoTypeEnum.dy_mix: + case VideoTypeEnum.dy_series: + case VideoTypeEnum.dy_custom_collect: + { + int syncCount = 0; // 本次同步成功的视频数量 + + var cates = await douyinCollectCateService.GetSyncCates(cookie.Id, VideoType); + if (cates != null && cates.Any()) + { + foreach (var cate in cates) + { + string cursor = "0"; + bool hasMore = true; + (syncCount, cursor, hasMore) = await GetAndSaveViedos(cookie, config, syncCount, cursor, hasMore, null); await HandleSyncCompletion(cookie, syncCount, null); } } } - // 不使用收藏文件夹，直接执行默认同步逻辑 - else - { - (syncCount, cursor, hasMore) = await GetViedos(cookie, config, syncCount, cursor, hasMore); - await HandleSyncCompletion(cookie, syncCount); - } - } - // 非抖音收藏类型，执行统一的默认同步逻辑 - else - { - // 提取重复变量，与上方逻辑保持一致命名 - int syncCount = 0; - string cursor = "0"; - bool hasMore = true; - - (syncCount, cursor, hasMore) = await GetViedos(cookie, config, syncCount, cursor, hasMore); - await HandleSyncCompletion(cookie, syncCount); - } + break; + case VideoTypeEnum.ImageVideo: + default: + break; } + + } catch (Exception ex) { @@ -393,30 +358,25 @@ namespace dy.net.job } } - private async Task<(int syncCount, string cursor, bool hasMore)> GetViedos(DouyinCookie cookie, AppConfig config, int syncCount, string cursor, bool hasMore, DouyinFollowed followed = null,DouyinCollectItem collectItem=null) + private async Task<(int syncCount, string cursor, bool hasMore)> GetAndSaveViedos(DouyinCookie cookie, AppConfig config, int syncCount, string cursor, bool hasMore, DouyinFollowed followed = null, DouyinCollectCate cate = null) { // 循环获取视频数据 while (hasMore) { // 获取视频数据 - var data = await FetchVideoData(cookie, cursor, followed,collectItem); - if (data == null) + var data = await FetchVideoData(cookie, cursor, followed, cate); + if (data == null|| data.AwemeList == null || !data.AwemeList.Any()) { - Log.Debug($"{VideoType}-Cookie[{cookie.UserName}]同步数据失败!!!"); break; } - // 判断是否还有更多数据 hasMore = ShouldContinueSync(cookie, data, followed); + // 获取下一页游标 cursor = GetNextCursor(data); - // 如果没有视频数据，退出循环 - if (data.AwemeList == null || !data.AwemeList.Any()) - break; - // 处理视频列表 - var videos = await ProcessVideoList(cookie, data, config, followed, collectItem); + var videos = await ProcessVideoList(cookie, data, config, followed, cate); // 保存视频信息到数据库 syncCount += await SaveVideos(videos); @@ -426,7 +386,7 @@ namespace dy.net.job break; } //随机等待 - await Task.Delay(_random.Next(5, 10) * 1000); + await Task.Delay(_random.Next(2, 10) * 1000); } return (syncCount, cursor, hasMore); @@ -474,9 +434,9 @@ namespace dy.net.job /// 视频信息对象 /// 应用配置 /// 关注的 - /// 收藏夹Item + /// 收藏夹、合集、短剧 /// 处理后的视频实体列表 - protected async Task> ProcessVideoList(DouyinCookie cookie, DouyinVideoInfoResponse data, AppConfig config, DouyinFollowed followed = null,DouyinCollectItem collectItem=null) + protected async Task> ProcessVideoList(DouyinCookie cookie, DouyinVideoInfoResponse data, AppConfig config, DouyinFollowed followed = null, DouyinCollectCate cate = null) { var videos = new List(); foreach (var item in data.AwemeList) @@ -485,48 +445,54 @@ namespace dy.net.job //{ // continue; //} - //if (!item.Desc.Contains("小房车正式启程")) - //{ - // continue; - //} - //判断视频是否是强制删除且不再下载的视频 - var deleteVideo = await douyinCommonService.ExistDeleteVideo(item.AwemeId); - if (deleteVideo) - { - //Log.Debug($"{VideoType}-视频-{item.AwemeId}-[{item.Desc}]已被标记为强制删除，跳过下载"); - continue; - } - - //判断是否存在视频，是否根据去重规则进行去重处理。。 // 1. 查询数据库中是否已存在该视频（通过 AwemeId 唯一标识） var exitVideo = await douyinVideoService.GetByAwemeId(item.AwemeId); - bool Goon = await AutoDistinct(config, item, cookie, exitVideo); - if (!Goon) + if (cate == null) { - continue; - } - //如果存在，但是因为去重规则，选择的最高优先级变更了，需要删除原来的，重新下载。 - if (exitVideo != null) - { - await douyinVideoService.DeleteById(exitVideo.Id); - } - var uper = await douyinFollowService.GetByUperId(item.AuthorUserId.ToString(), cookie.MyUserId); - if (uper != null) - { - if (uper.FullSync) + //判断视频是否是强制删除且不再下载的视频 + var deleteVideo = await douyinCommonService.ExistDeleteVideo(item.AwemeId); + if (deleteVideo) { - followed ??= uper; + //Log.Debug($"{VideoType}-视频-{item.AwemeId}-[{item.Desc}]已被标记为强制删除，跳过下载"); + continue; + } + + + bool Goon = await AutoDistinct(config, exitVideo); + if (!Goon) + { + continue; + } + //如果存在，但是因为去重规则，选择的最高优先级变更了，需要删除原来的，重新下载。 + if (exitVideo != null) + { + await douyinVideoService.DeleteById(exitVideo.Id); + } + var uper = await douyinFollowService.GetByUperId(item.AuthorUserId.ToString(), cookie.MyUserId); + if (uper != null) + { + if (uper.FullSync) + { + followed ??= uper; + } } } + else + { + if (exitVideo != null && exitVideo.ViedoType == cate.CateType &&File.Exists(exitVideo.VideoSavePath)) + { + continue; + } + } + // 处理单个视频 - var video = await ProcessSingleVideo(cookie, item, data, config, followed,collectItem); + var video = await ProcessSingleVideo(cookie, item, data, config, followed); if (video != null) videos.Add(video); else { - //处理多个视频-组合的图文视频--类似动图。 List dynamicVideoUrls = new List(); // 当需要下载动态视频时，获取其他URL @@ -557,7 +523,7 @@ namespace dy.net.job } // 处理核心逻辑 - if (config.DownDynamicVideo && dynamicVideoUrls.Count > 0) + if (dynamicVideoUrls.Count > 0) { // 处理动态视频 var dynamicVideo = await ProcessDynamicVideo(dynamicVideoUrls, cookie, item, data, config); @@ -569,15 +535,15 @@ namespace dy.net.job Log.Debug($"{VideoType}-动态视频[{item.Desc}]，下载成功 ,共{dynamicVideos?.Count}个视频..."); if (dynamicVideos != null && dynamicVideos.Count > 0) { - - var savePath = DouyinFileNameHelper.RemoveNumberSuffix(dynamicVideo.VideoSavePath); - - //音频文件下载地址 - var mp3Url= item?.Music?.PlayUrl?.UrlList?.FirstOrDefault(); - - var outPath = await douyinMergeVideoService.MergeMultipleVideosAsync(dynamicVideos,mp3Url, savePath,cookie.Cookies); - if(!string.IsNullOrWhiteSpace(outPath.mp4Path)) + var savePath = DouyinFileNameHelper.RemoveNumberSuffix(dynamicVideo.VideoSavePath); + + //音频文件下载地址 + var mp3Url = item?.Music?.PlayUrl?.UrlList?.FirstOrDefault(); + + + var outPath = await douyinMergeVideoService.MergeMultipleVideosAsync(dynamicVideos, mp3Url, savePath, cookie.Cookies); + if (!string.IsNullOrWhiteSpace(outPath.mp4Path)) { if (File.Exists(outPath.mp4Path)) { @@ -608,10 +574,10 @@ namespace dy.net.job } else { - // 统一处理图文视频逻辑 + // 处理图文视频逻辑 if (config.DownImageVideo || config.DownMp3 || config.DownImage) { - var mergevideo = await ProcessImageSetAndMergeToVideo(cookie, item, data, config, followed, collectItem); + var mergevideo = await ProcessImageSetAndMergeToVideo(cookie, item, config, followed,cate); if (mergevideo != null) videos.Add(mergevideo); } @@ -621,7 +587,7 @@ namespace dy.net.job return videos; } - private async Task AutoDistinct(AppConfig config, Aweme item, DouyinCookie cookie, DouyinVideo exitVideo) + private async Task AutoDistinct(AppConfig config, DouyinVideo exitVideo) { // 去重，检查视频是否已存在（按优先级下载） if (config.AutoDistinct) @@ -806,9 +772,9 @@ namespace dy.net.job /// 视频信息对象 /// 应用配置 /// 关注 - /// 收藏夹Item + /// /// 处理后的视频实体，如果处理失败则为null - protected async Task ProcessSingleVideo(DouyinCookie cookie, Aweme item, DouyinVideoInfoResponse data, AppConfig config, DouyinFollowed followed = null,DouyinCollectItem collectItem=null) + protected async Task ProcessSingleVideo(DouyinCookie cookie, Aweme item, DouyinVideoInfoResponse data, AppConfig config, DouyinFollowed followed = null,DouyinCollectCate cate=null) { // 检查视频数据是否有效 if (!IsAwemeValid(item)) return null; @@ -824,7 +790,7 @@ namespace dy.net.job // 获取视频标签 var (tag1, tag2, tag3) = GetVideoTags(item); // 创建保存文件夹 - var saveFolder = CreateSaveFolder(cookie, item, config, followed,collectItem); + var saveFolder = CreateSaveFolder(cookie, item, config, followed, cate); // 获取视频文件名 var fileName = GetVideoFileName(cookie, item, config); // 拼接视频保存路径 @@ -860,24 +826,20 @@ namespace dy.net.job // 下载视频封面 await DownVideoCover(item, saveFolder, cookie, config); // 下载作者头像 - var (avatarSavePath, avatarUrl) = await DownAuthorAvatar(cookie, item); + //var (avatarSavePath, avatarUrl) = await DownAuthorAvatar(cookie, item); // 创建视频实体 - return await CreateVideoEntity(config, cookie, item, v, savePath, saveFolder, tag1, tag2, tag3, avatarSavePath, avatarUrl, data); + return await CreateVideoEntity(config, cookie, item, v, savePath, saveFolder, tag1, tag2, tag3); } - - - - - protected async Task ProcessDynamicVideo(List dynamicUrls, DouyinCookie cookie, Aweme item, DouyinVideoInfoResponse data, AppConfig config, DouyinFollowed followed = null,DouyinCollectItem collectItem=null) + protected async Task ProcessDynamicVideo(List dynamicUrls, DouyinCookie cookie, Aweme item, DouyinVideoInfoResponse data, AppConfig config, DouyinFollowed followed = null,DouyinCollectCate cate=null) { // 获取视频标签 var (tag1, tag2, tag3) = GetVideoTags(item); // 创建保存文件夹 - var saveFolder = CreateSaveFolder(cookie, item, config, followed,collectItem); + var saveFolder = CreateSaveFolder(cookie, item, config, followed, cate); // 获取视频文件名 //var fileName = DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId) + ".mp4"; @@ -894,7 +856,7 @@ namespace dy.net.job { // 下载动态视频 var dynamicSavePath = savePath.Replace(".mp4", $"_00{i}.mp4"); - DouyinDynamicVideoDto v = new DouyinDynamicVideoDto() { Height = dynamicUrl.Height, Width = dynamicUrl.Width,Path= dynamicSavePath }; + DouyinDynamicVideoDto v = new DouyinDynamicVideoDto() { Height = dynamicUrl.Height, Width = dynamicUrl.Width, Path = dynamicSavePath }; i++; // 如果文件已存在，跳过 @@ -932,19 +894,13 @@ namespace dy.net.job { Width = item?.Images?.FirstOrDefault()?.Width ?? 0, Height = item?.Images?.FirstOrDefault()?.Height ?? 0, - DataSize = DouyinFileUtils.GetTotalFileSize(dynamicSavePaths.Select(x=>x.Path).ToList()) // 合成视频的文件大小 + DataSize = DouyinFileUtils.GetTotalFileSize(dynamicSavePaths.Select(x => x.Path).ToList()) // 合成视频的文件大小 } }; - return await CreateVideoEntity(config, cookie, item, virtualBitRate, dynamicSavePaths.FirstOrDefault()?.Path, saveFolder, tag1, tag2, tag3, "", "", data, dynamicSavePaths); + return await CreateVideoEntity(config, cookie, item, virtualBitRate, dynamicSavePaths.FirstOrDefault()?.Path, saveFolder, tag1, tag2, tag3, dynamicSavePaths); } - - - - - - /// /// 寻找其他视频地址下载... /// @@ -1010,8 +966,9 @@ namespace dy.net.job /// 视频信息对象 /// 应用配置 /// 应用配置 + /// /// 合成后的视频实体，如果处理失败则为null - protected async Task ProcessImageSetAndMergeToVideo(DouyinCookie cookie, Aweme item, DouyinVideoInfoResponse data, AppConfig config, DouyinFollowed followed,DouyinCollectItem collectItem) + protected async Task ProcessImageSetAndMergeToVideo(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed,DouyinCollectCate cate) { try { @@ -1043,7 +1000,7 @@ namespace dy.net.job } else { - fileNamefolder = CreateSaveFolder(cookie, item, config, followed,collectItem); + fileNamefolder = CreateSaveFolder(cookie, item, config, followed, cate); } if (!Directory.Exists(fileNamefolder)) Directory.CreateDirectory(fileNamefolder); @@ -1113,7 +1070,7 @@ namespace dy.net.job // 下载视频封面（使用第一张图片作为封面） await DownVideoCover(imageUrls.FirstOrDefault(), fileNamefolder, cookie, item, config); // 下载作者头像 - var (avatarSavePath, avatarUrl) = await DownAuthorAvatar(cookie, item); + //var (avatarSavePath, avatarUrl) = await DownAuthorAvatar(cookie, item); // 获取视频标签 var (tag1, tag2, tag3) = GetVideoTags(item); @@ -1131,7 +1088,7 @@ namespace dy.net.job // 创建视频实体 var videoEntity = await CreateVideoEntity(config, cookie, item, virtualBitRate, savePath, fileNamefolder, - tag1, tag2, tag3, avatarSavePath, avatarUrl, data); + tag1, tag2, tag3); // 特殊处理合成视频的字段 videoEntity.FileHash = string.Empty; // 合成视频没有原始文件哈希 @@ -1171,8 +1128,6 @@ namespace dy.net.job } } - - /// /// 下载视频封面 /// 从视频信息中提取封面URL并下载到指定文件夹 @@ -1196,25 +1151,25 @@ namespace dy.net.job /// 用户Cookie /// 视频信息 /// 一个元组，包含头像保存路径和头像URL - protected async Task<(string savePath, string url)> DownAuthorAvatar(DouyinCookie cookie, Aweme item) - { - if (item.Author == null) return (null, null); - // 优先获取高清头像 - var avatarUrl = item.Author.AvatarLarger?.UrlList?.FirstOrDefault() ?? item.Author.AvatarThumb?.UrlList?.FirstOrDefault(); - if (string.IsNullOrWhiteSpace(avatarUrl)) return (null, null); + //protected async Task<(string savePath, string url)> DownAuthorAvatar(DouyinCookie cookie, Aweme item) + //{ + // if (item.Author == null) return (null, null); + // // 优先获取高清头像 + // var avatarUrl = item.Author.AvatarLarger?.UrlList?.FirstOrDefault() ?? item.Author.AvatarThumb?.UrlList?.FirstOrDefault(); + // if (string.IsNullOrWhiteSpace(avatarUrl)) return (null, null); - // 拼接头像保存路径 - var avatarSavePath = Path.Combine(GetAuthorAvatarBasePath(cookie), $"{item.Author.Uid}.jpg"); - var avatarDir = Path.GetDirectoryName(avatarSavePath); - // 创建头像保存文件夹 - if (!Directory.Exists(avatarDir)) Directory.CreateDirectory(avatarDir); - // 如果头像文件不存在，则下载 - if (!File.Exists(avatarSavePath)) - { - await douyinHttpClientService.DownloadAsync(avatarUrl, avatarSavePath, cookie.Cookies); - } - return (avatarSavePath, avatarUrl); - } + // // 拼接头像保存路径 + // var avatarSavePath = Path.Combine(GetAuthorAvatarBasePath(cookie), $"{item.Author.Uid}.jpg"); + // var avatarDir = Path.GetDirectoryName(avatarSavePath); + // // 创建头像保存文件夹 + // if (!Directory.Exists(avatarDir)) Directory.CreateDirectory(avatarDir); + // // 如果头像文件不存在，则下载 + // if (!File.Exists(avatarSavePath)) + // { + // await douyinHttpClientService.DownloadAsync(avatarUrl, avatarSavePath, cookie.Cookies); + // } + // return (avatarSavePath, avatarUrl); + //} #endregion @@ -1226,7 +1181,7 @@ namespace dy.net.job /// /// 视频信息 /// 如果视频数据有效，则为true；否则为false - private bool IsAwemeValid(Aweme item) => item != null && item.Video != null && item.Video.BitRate != null; + private static bool IsAwemeValid(Aweme item) => item != null && item.Video != null && item.Video.BitRate != null; /// /// 获取视频标签 @@ -1326,17 +1281,17 @@ namespace dy.net.job /// 创建的视频实体对象 private async Task CreateVideoEntity(AppConfig config, DouyinCookie cookie, Aweme item, VideoBitRate bitRate, string savePath, string saveFolder, - string tag1, string tag2, string tag3, string avatarSavePath, string avatarUrl, DouyinVideoInfoResponse data, List dynamicVideos = null) + string tag1, string tag2, string tag3, List dynamicVideos = null) { var diffs = GetVideoEntityDifferences(cookie, item); - + var avatarUrl = item.Author.AvatarLarger?.UrlList?.FirstOrDefault() ?? item.Author.AvatarThumb?.UrlList?.FirstOrDefault(); var video = new DouyinVideo { ViedoType = VideoType, AwemeId = item.AwemeId, Author = item.Author?.Nickname, AuthorId = item.Author?.Uid, - AuthorAvatar = avatarSavePath, + AuthorAvatar = "", AuthorAvatarUrl = avatarUrl, CreateTime = DateTimeUtil.Convert10BitTimestamp(item.CreateTime), VideoTitle = string.IsNullOrWhiteSpace(item.Desc) ? $"{item.Author?.Nickname}-{item.CreateTime}" : item.Desc, @@ -1377,11 +1332,6 @@ namespace dy.net.job /// public class VideoEntityDifferences { - /// - /// 视频类型 - /// - //public VideoTypeEnum VideoType { get; set; } - /// /// 简化的视频标题 /// diff --git a/job/DouyinCollectCatesSyncJob.cs b/job/DouyinCollectCatesSyncJob.cs deleted file mode 100644 index f69b708..0000000 --- a/job/DouyinCollectCatesSyncJob.cs +++ /dev/null @@ -1,124 +0,0 @@ -using dy.net.model.entity; -using dy.net.model.response; -using dy.net.service; -using Quartz; -using System.Collections.Generic; - -namespace dy.net.job -{ - [DisallowConcurrentExecution] // 禁止并发执行，确保同一时间只有一个实例在运行 - public class DouyinCollectCatesSyncJob : IJob - { - - - /// - /// 抖音Cookie服务，用于获取和管理用户Cookie - /// - protected readonly DouyinCookieService _dyCookieService; - - /// - /// 抖音HTTP客户端服务，用于发送HTTP请求 - /// - protected readonly DouyinHttpClientService _douyinService; - - /// - /// 抖音关注服务，用于管理关注数据 - /// - protected readonly DouyinFollowService _followService; - - protected readonly DouyinCommonService douyinCommonService; - - public DouyinCollectCatesSyncJob(DouyinCookieService dyCookieService, DouyinHttpClientService douyinService, DouyinFollowService followService, DouyinCommonService douyinCommonService) - { - _dyCookieService = dyCookieService; - _douyinService = douyinService; - _followService = followService; - this.douyinCommonService = douyinCommonService; - } - - public async Task Execute(IJobExecutionContext context) - { - var cookies = await _dyCookieService.GetAllOpendAsync(); - if (cookies != null && cookies.Any()) - { - AppConfig conf = douyinCommonService.GetConfig(); - - foreach (var ck in cookies) - { - if (string.IsNullOrWhiteSpace(ck.SecUserId)) - { - Serilog.Log.Debug($"同步关注列表博主[{ck.UserName}]-未设置secuserid,跳过."); - continue; - } - string count = "20"; - string offset = "0"; - bool hasmore = true; - int total= 0; - List follows = new List(); - while (hasmore) - { - try - { - var data = await _douyinService.SyncMyFollows(count, offset, ck.SecUserId, ck.Cookies, async err => - { - if (err.StatusCode != 0) - { - Serilog.Log.Error($"同步关注列表博主[{ck.UserName}]关注列表时发生错误，错误信息：{err.StatusMsg}，状态码：{err.StatusCode}"); - } - // 如果是未登录错误，则跳出循环 - if (err.StatusCode == 8) - { - hasmore = false; - } - ck.StatusMsg = err.StatusCode == 8 ? "无效" : "正常"; - ck.StatusCode = err.StatusCode; - await _dyCookieService.UpdateAsync(ck); - }); - - if (data != null) - { - // 绑定我的用户ID--之前没有这个字段 - if (string.IsNullOrWhiteSpace(ck.MyUserId)) - { - ck.MyUserId = data.MySelfUserId; - await _dyCookieService.UpdateAsync(ck); - } - total = data.Total; - hasmore = data.HasMore; - offset = data.Offset.ToString(); - if (data.Followings != null && data.Followings.Count > 0) - { - follows.AddRange(data.Followings); - } - } - - // 只有第一次启动时，才进行全部同步 - if (!conf.IsFirstRunning) - { - hasmore = false; - } - } - catch (Exception ex) - { - Serilog.Log.Error($"关注列表同步失败,{ex.Message}"); - hasmore = false; - break; - } - } - - if (follows.Count > 0) - { - var (add, update, succ) = await _followService.Sync(follows, ck); - if (!succ) hasmore = false; - else - { - hasmore = add > 100|| update > 100;//一次最多100条数据 - } - } - - } - await douyinCommonService.SetConfigNotFirstRunning(); - } - } - } -} diff --git a/job/DouyinCollectCustomSyncJob.cs b/job/DouyinCollectCustomSyncJob.cs new file mode 100644 index 0000000..7fbcf84 --- /dev/null +++ b/job/DouyinCollectCustomSyncJob.cs @@ -0,0 +1,29 @@ +using dy.net.model.dto; +using dy.net.model.entity; +using dy.net.model.response; +using dy.net.service; +using dy.net.utils; +using System; + +namespace dy.net.job +{ + public class DouyinCollectCustomSyncJob : DouyinBasicSyncJob + { + public DouyinCollectCustomSyncJob(DouyinCookieService douyinCookieService, DouyinHttpClientService douyinHttpClientService, DouyinVideoService douyinVideoService, DouyinCommonService douyinCommonService, DouyinFollowService douyinFollowService, DouyinMergeVideoService douyinMergeVideoService, DouyinCollectCateService douyinCollectCateService) : base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService, douyinCollectCateService) + { + } + + protected override VideoTypeEnum VideoType => VideoTypeEnum.dy_custom_collect; + + protected override async Task FetchVideoData(DouyinCookie cookie, string cursor, DouyinFollowed followed, DouyinCollectCate cate) + { + return await douyinHttpClientService.SyncCollectVideos(cursor, count, cookie.Cookies); + } + + protected override bool ShouldContinueSync(DouyinCookie cookie, DouyinVideoInfoResponse data, DouyinFollowed followed = null) + { + return data != null && data.HasMore == 1 && cookie.CollHasSyncd == 0; + } + + } +} \ No newline at end of file diff --git a/job/DouyinFollowedUsersSyncJob.cs b/job/DouyinFollowedUsersSyncJob.cs deleted file mode 100644 index 584a444..0000000 --- a/job/DouyinFollowedUsersSyncJob.cs +++ /dev/null @@ -1,124 +0,0 @@ -using dy.net.model.entity; -using dy.net.model.response; -using dy.net.service; -using Quartz; -using System.Collections.Generic; - -namespace dy.net.job -{ - [DisallowConcurrentExecution] // 禁止并发执行，确保同一时间只有一个实例在运行 - public class DouyinFollowedUsersSyncJob : IJob - { - - - /// - /// 抖音Cookie服务，用于获取和管理用户Cookie - /// - protected readonly DouyinCookieService _dyCookieService; - - /// - /// 抖音HTTP客户端服务，用于发送HTTP请求 - /// - protected readonly DouyinHttpClientService _douyinService; - - /// - /// 抖音关注服务，用于管理关注数据 - /// - protected readonly DouyinFollowService _followService; - - protected readonly DouyinCommonService douyinCommonService; - - public DouyinFollowedUsersSyncJob(DouyinCookieService dyCookieService, DouyinHttpClientService douyinService, DouyinFollowService followService, DouyinCommonService douyinCommonService) - { - _dyCookieService = dyCookieService; - _douyinService = douyinService; - _followService = followService; - this.douyinCommonService = douyinCommonService; - } - - public async Task Execute(IJobExecutionContext context) - { - var cookies = await _dyCookieService.GetAllOpendAsync(); - if (cookies != null && cookies.Any()) - { - AppConfig conf = douyinCommonService.GetConfig(); - - foreach (var ck in cookies) - { - if (string.IsNullOrWhiteSpace(ck.SecUserId)) - { - Serilog.Log.Debug($"同步关注列表博主[{ck.UserName}]-未设置secuserid,跳过."); - continue; - } - string count = "20"; - string offset = "0"; - bool hasmore = true; - int total= 0; - List follows = new List(); - while (hasmore) - { - try - { - var data = await _douyinService.SyncMyFollows(count, offset, ck.SecUserId, ck.Cookies, async err => - { - if (err.StatusCode != 0) - { - Serilog.Log.Error($"同步关注列表博主[{ck.UserName}]关注列表时发生错误，错误信息：{err.StatusMsg}，状态码：{err.StatusCode}"); - } - // 如果是未登录错误，则跳出循环 - if (err.StatusCode == 8) - { - hasmore = false; - } - ck.StatusMsg = err.StatusCode == 8 ? "无效" : "正常"; - ck.StatusCode = err.StatusCode; - await _dyCookieService.UpdateAsync(ck); - }); - - if (data != null) - { - // 绑定我的用户ID--之前没有这个字段 - if (string.IsNullOrWhiteSpace(ck.MyUserId)) - { - ck.MyUserId = data.MySelfUserId; - await _dyCookieService.UpdateAsync(ck); - } - total = data.Total; - hasmore = data.HasMore; - offset = data.Offset.ToString(); - if (data.Followings != null && data.Followings.Count > 0) - { - follows.AddRange(data.Followings); - } - } - - // 只有第一次启动时，才进行全部同步 - if (!conf.IsFirstRunning) - { - hasmore = false; - } - } - catch (Exception ex) - { - Serilog.Log.Error($"关注列表同步失败,{ex.Message}"); - hasmore = false; - break; - } - } - - if (follows.Count > 0) - { - var (add, update, succ) = await _followService.Sync(follows, ck); - if (!succ) hasmore = false; - else - { - hasmore = add > 100|| update > 100;//一次最多100条数据 - } - } - - } - await douyinCommonService.SetConfigNotFirstRunning(); - } - } - } -} diff --git a/job/DouyinFollowedViedoSyncJob.cs b/job/DouyinFollowedViedoSyncJob.cs index 05ff45d..82aeb36 100644 --- a/job/DouyinFollowedViedoSyncJob.cs +++ b/job/DouyinFollowedViedoSyncJob.cs @@ -13,24 +13,18 @@ namespace dy.net.job { public class DouyinFollowedViedoSyncJob : DouyinBasicSyncJob { - public DouyinFollowedViedoSyncJob(DouyinCookieService douyinCookieService, DouyinHttpClientService douyinHttpClientService, DouyinVideoService douyinVideoService, DouyinCommonService douyinCommonService, DouyinFollowService douyinFollowService, DouyinMergeVideoService douyinMergeVideoService) : base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService) + public DouyinFollowedViedoSyncJob(DouyinCookieService douyinCookieService, DouyinHttpClientService douyinHttpClientService, DouyinVideoService douyinVideoService, DouyinCommonService douyinCommonService, DouyinFollowService douyinFollowService, DouyinMergeVideoService douyinMergeVideoService, DouyinCollectCateService douyinCollectCateService) : base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService, douyinCollectCateService) { } - protected override VideoTypeEnum VideoType => VideoTypeEnum.dy_follows; - protected override async Task> GetValidCookies() + protected override async Task> GetSyncCookies() { - return await douyinCookieService.GetAllOpendAsync(x => !string.IsNullOrWhiteSpace(x.UpSavePath)); + return await douyinCookieService.GetOpendCookiesAsync(x => !string.IsNullOrWhiteSpace(x.UpSavePath)); } - protected override bool IsCookieValid(DouyinCookie cookie) - { - return !string.IsNullOrWhiteSpace(cookie.Cookies)&&!string.IsNullOrWhiteSpace(cookie.UpSavePath); - } - - protected override async Task FetchVideoData(DouyinCookie cookie, string cursor, DouyinFollowed followed, DouyinCollectItem collectId) + protected override async Task FetchVideoData(DouyinCookie cookie, string cursor, DouyinFollowed followed,DouyinCollectCate cate) { return await douyinHttpClientService.SyncUpderPostVideos(count, cursor, followed.SecUid, cookie.Cookies); } @@ -40,10 +34,6 @@ namespace dy.net.job return data != null && data.HasMore == 1 && cookie.UperSyncd == 0 && followed.FullSync; } - protected override string GetNextCursor(DouyinVideoInfoResponse data) - { - return data?.MaxCursor ?? "0"; - } /// /// 关注用户特殊处理文件夹存储路径，用户可自定义保存路径 @@ -53,7 +43,7 @@ namespace dy.net.job /// /// /// - protected override string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed, DouyinCollectItem collectItem) + protected override string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed,DouyinCollectCate cate) { #region 默认使用UP主名称作为文件夹名称，若关注列表中有自定义保存路径则使用自定义路径 // 1. 优先获取有效的作者名称（遵循原有优先级：followed.UperName > item.Author.Nickname > 默认值） @@ -170,11 +160,6 @@ namespace dy.net.job } } - protected override string GetAuthorAvatarBasePath(DouyinCookie cookie) - { - return Path.Combine(cookie.UpSavePath, "author"); - } - protected override async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount, DouyinFollowed followed) { if (syncCount > 0) diff --git a/job/DouyinFollowsAndCollnectsSyncJob.cs b/job/DouyinFollowsAndCollnectsSyncJob.cs new file mode 100644 index 0000000..5c11943 --- /dev/null +++ b/job/DouyinFollowsAndCollnectsSyncJob.cs @@ -0,0 +1,347 @@ +using dy.net.model.dto; +using dy.net.model.entity; +using dy.net.model.response; +using dy.net.service; +using Quartz; + +namespace dy.net.job +{ + [DisallowConcurrentExecution] // 禁止并发执行，确保同一时间只有一个实例在运行 + public class DouyinFollowsAndCollnectsSyncJob : IJob + { + + + /// + /// 抖音收藏夹服务 + /// + protected readonly DouyinCollectCateService douyinCollectCateService; + /// + /// 抖音Cookie服务，用于获取和管理用户Cookie + /// + protected readonly DouyinCookieService _dyCookieService; + + /// + /// 抖音HTTP客户端服务，用于发送HTTP请求 + /// + protected readonly DouyinHttpClientService _douyinService; + + /// + /// 抖音关注服务，用于管理关注数据 + /// + protected readonly DouyinFollowService _followService; + + protected readonly DouyinCommonService douyinCommonService; + + public DouyinFollowsAndCollnectsSyncJob(DouyinCookieService dyCookieService, DouyinHttpClientService douyinService, DouyinFollowService followService, DouyinCommonService douyinCommonService, DouyinCollectCateService douyinCollectCateService) + { + _dyCookieService = dyCookieService; + _douyinService = douyinService; + _followService = followService; + this.douyinCommonService = douyinCommonService; + this.douyinCollectCateService = douyinCollectCateService; + } + + // 第一步：定义常量和枚举（建议放在单独的常量类中，此处为内联演示） + private const string DEFAULT_FOLLOW_COUNT = "20"; + private const int INVALID_COOKIE_STATUS_CODE = 8; + private const string LOG_TAG_COLLECT = "收藏夹同步"; + private const string LOG_TAG_MIX = "合集列表同步"; + private const string LOG_TAG_SERIES = "短剧列表同步"; + private const string LOG_TAG_FOLLOW = "关注列表同步"; + + /// + /// 抖音数据同步任务执行方法（优化版） + /// + /// 任务执行上下文 + public async Task Execute(IJobExecutionContext context) + { + // 1. 获取有效Cookie列表，提前做空值判断 + var cookies = await _dyCookieService.GetOpendCookiesAsync(); + if (cookies == null || !cookies.Any()) + { + Serilog.Log.Debug("当前无可用的抖音Cookie，同步任务跳过"); + return; + } + + // 2. 获取应用配置 + AppConfig conf = douyinCommonService.GetConfig(); + + // 3. 遍历每个Cookie，执行各类数据同步 + foreach (var ck in cookies) + { + Serilog.Log.Information($"开始处理Cookie用户：[{ck.UserName}]（ID：{ck.Id}）"); + + // 3.1 同步自定义收藏夹（UseCollectFolder=true时） + if (ck.UseCollectFolder) + { + await SyncCollectDataAsync( + cookie: ck, + cateType: VideoTypeEnum.dy_custom_collect, + dataFetchFunc: async (offset) => await _douyinService.SyncCollectFolderList(ck.Cookies, offset), + entityConvertFunc: (collectItem) => new DouyinCollectCate + { + CookieId = ck.Id, + CateType = VideoTypeEnum.dy_custom_collect, + Name = collectItem.CollectsName, + Sync = false, + XId = collectItem.CollectsId + }, + hasMoreFunc: (data) => data?.HasMore ?? false, + cursorFunc: (data) => data?.Cursor.ToString() ?? "0", + dataListFunc: (data) => data?.CollectsList, + logTag: LOG_TAG_COLLECT + ); + } + + // 3.2 同步收藏夹合集（DownMix=true时） + if (ck.DownMix) + { + await SyncCollectDataAsync( + cookie: ck, + cateType: VideoTypeEnum.dy_mix, + dataFetchFunc: async (offset) => await _douyinService.SyncMixList(ck.Cookies, offset), + entityConvertFunc: (mixItem) => new DouyinCollectCate + { + CookieId = ck.Id, + CateType = VideoTypeEnum.dy_mix, + Name = mixItem.MixName, + Sync = false, + XId = mixItem.MixId, + CoverUrl = mixItem.CoverUrl?.UrlList?.FirstOrDefault() // 空值防护：避免CoverUrl为null + }, + hasMoreFunc: (data) => (data?.HasMore ?? 0) == 1, + cursorFunc: (data) => data?.Cursor.ToString() ?? "0", + dataListFunc: (data) => data?.MixInfos, + logTag: LOG_TAG_MIX + ); + } + + // 3.3 同步收藏夹短剧（DownSeries=true时） + if (ck.DownSeries) + { + await SyncCollectDataAsync( + cookie: ck, + cateType: VideoTypeEnum.dy_series, + dataFetchFunc: async (offset) => await _douyinService.SyncSeriesList(ck.Cookies, offset), + entityConvertFunc: (seriesItem) => new DouyinCollectCate + { + CookieId = ck.Id, + CateType = VideoTypeEnum.dy_series, + Name = seriesItem.SeriesName, + Sync = false, + XId = seriesItem.SeriesId, + CoverUrl = seriesItem.CoverImage?.ImageUrlList?.FirstOrDefault() // 空值防护：避免CoverImage为null + }, + hasMoreFunc: (data) => (data?.HasMore ?? 0) == 1, + cursorFunc: (data) => data?.Cursor.ToString() ?? "0", + dataListFunc: (data) => data?.SeriesList, + logTag: LOG_TAG_SERIES + ); + } + + // 3.4 同步关注列表（单独处理，逻辑特殊不纳入通用方法） + await SyncFollowListAsync(ck, conf); + + Serilog.Log.Debug($"完成Cookie用户：[{ck.UserName}]（ID：{ck.Id}）的所有收藏夹信息、合集信息、关注列表信息、短剧信息同步"); + } + + // 4. 标记首次运行完成（仅执行一次） + await douyinCommonService.SetConfigNotFirstRunning(); + } + + #region 私有通用辅助方法 + /// + /// 通用收藏类数据同步方法（泛型封装，消除冗余） + /// + /// 抖音返回的分页数据类型 + /// 抖音返回的列表项类型 + /// 当前Cookie信息 + /// 分类类型 + /// 分页数据获取委托 + /// 抖音项转换为DouyinCollectCate的委托 + /// 判断是否还有更多数据的委托 + /// 获取下一页游标值的委托 + /// 从分页数据中提取列表的委托 + /// 日志标签 + private async Task SyncCollectDataAsync( + DouyinCookie cookie, + VideoTypeEnum cateType, + Func> dataFetchFunc, + Func entityConvertFunc, + Func hasMoreFunc, + Func cursorFunc, + Func> dataListFunc, + string logTag) + { + string offset = "0"; + bool hasMore = true; + List collectDataList = new List(); + + try + { + // 1. 分页获取抖音数据 + while (hasMore) + { + var pageData = await dataFetchFunc(offset); + if (pageData == null) + { + //Serilog.Log.Warning($"[{cookie.UserName}] - {logTag}：获取分页数据为空，停止分页"); + break; + } + + // 2. 更新分页状态 + hasMore = hasMoreFunc(pageData); + offset = cursorFunc(pageData); + + // 3. 提取当前页列表数据并转换 + var currentPageItems = dataListFunc(pageData); + if (currentPageItems != null && currentPageItems.Any()) + { + var convertItems = currentPageItems.Select(entityConvertFunc).ToList(); + collectDataList.AddRange(convertItems); + Serilog.Log.Debug($"[{cookie.UserName}] - {logTag}：获取到{currentPageItems.Count}条数据，累计{collectDataList.Count}条"); + } + } + + // 4. 调用Sync方法同步到数据库 + if (collectDataList.Any()) + { + var (add, update, delete, succ) = await douyinCollectCateService.Sync( + collectDataList, + cookie.Id, + cateType); + + Serilog.Log.Debug($"[{cookie.UserName}] - {logTag}：同步完成，新增{add}条，更新{update}条，删除{delete}条，是否成功：{succ}"); + } + else + { + Serilog.Log.Debug($"[{cookie.UserName}] - {logTag}：无有效数据需要同步"); + } + } + catch (Exception ex) + { + Serilog.Log.Error(ex, $"[{cookie.UserName}] - {logTag}：分页同步失败，异常信息：{ex.Message}"); + } + } + + /// + /// 关注列表单独同步方法（逻辑特殊，单独封装） + /// + /// 当前Cookie信息 + /// 应用配置 + private async Task SyncFollowListAsync(DouyinCookie cookie, AppConfig config) + { + // 1. 校验SecUserId是否有效 + if (string.IsNullOrWhiteSpace(cookie.SecUserId)) + { + Serilog.Log.Debug($"[{cookie.UserName}] - {LOG_TAG_FOLLOW}：未设置SecUserId，跳过"); + return; + } + + string offset = "0"; + bool hasMore = true; + int total = 0; + List followList = new List(); + + try + { + // 2. 分页获取关注列表数据 + while (hasMore) + { + var (data, error) = await FetchFollowPageDataAsync(cookie, offset); + + // 3. 处理错误信息 + if (error != null && error.StatusCode != 0) + { + Serilog.Log.Error($"[{cookie.UserName}] - {LOG_TAG_FOLLOW}：发生错误，状态码：{error.StatusCode}，错误信息：{error.StatusMsg}"); + // 无效Cookie（未登录）直接停止分页 + if (error.StatusCode == INVALID_COOKIE_STATUS_CODE) + { + cookie.StatusMsg = "无效"; + cookie.StatusCode = INVALID_COOKIE_STATUS_CODE; + await _dyCookieService.UpdateAsync(cookie); + break; + } + } + + // 4. 处理有效数据 + if (data != null) + { + // 5. 更新MyUserId（若未设置） + if (string.IsNullOrWhiteSpace(cookie.MyUserId)) + { + cookie.MyUserId = data.MySelfUserId; + await _dyCookieService.UpdateAsync(cookie); + } + + // 6. 更新分页状态和数据 + total = data.Total; + hasMore = data.HasMore; + offset = data.Offset.ToString(); + + if (data.Followings != null && data.Followings.Any()) + { + followList.AddRange(data.Followings); + Serilog.Log.Debug($"[{cookie.UserName}] - {LOG_TAG_FOLLOW}：获取到{data.Followings.Count}条关注数据，累计{followList.Count}条"); + } + + // 7. 非首次运行时，仅同步第一页数据 + if (!config.IsFirstRunning) + { + hasMore = false; + //Serilog.Log.Debug($"[{cookie.UserName}] - {LOG_TAG_FOLLOW}：非首次运行，停止分页获取"); + } + } + else + { + Serilog.Log.Warning($"[{cookie.UserName}] - {LOG_TAG_FOLLOW}：获取分页数据为空，停止分页"); + break; + } + } + + // 8. 同步关注列表到数据库 + if (followList.Any()) + { + var (add, update, succ) = await _followService.Sync(followList, cookie); + Serilog.Log.Information($"[{cookie.UserName}] - {LOG_TAG_FOLLOW}：同步完成，新增{add}条，更新{update}条，是否成功：{succ}，总关注数：{total}"); + } + else + { + Serilog.Log.Debug($"[{cookie.UserName}] - {LOG_TAG_FOLLOW}：无有效关注数据需要同步"); + } + } + catch (Exception ex) + { + Serilog.Log.Error(ex, $"[{cookie.UserName}] - {LOG_TAG_FOLLOW}：同步失败，异常信息：{ex.Message}"); + } + } + + /// + /// 分页获取关注列表数据（封装回调逻辑，简化代码） + /// + /// 当前Cookie信息 + /// 分页偏移量 + private async Task<(DouyinFollowInfoResponse Data, FollowErrorDto Error)> FetchFollowPageDataAsync(DouyinCookie cookie, string offset) + { + DouyinFollowInfoResponse resultData = null; + FollowErrorDto resultError = null; + resultData = await _douyinService.SyncMyFollows( + DEFAULT_FOLLOW_COUNT, + offset, + cookie.SecUserId, + cookie.Cookies, + async (err) => + { + resultError = err; + // 更新Cookie状态（异步不阻塞） + cookie.StatusMsg = err.StatusCode == INVALID_COOKIE_STATUS_CODE ? "无效" : "正常"; + cookie.StatusCode = err.StatusCode; + await _dyCookieService.UpdateAsync(cookie); + }); + + // 若原方法是回调式无返回值，需调整封装逻辑 + return (resultData, resultError); + } + #endregion + } +} diff --git a/job/DouyinMixSyncJob.cs b/job/DouyinMixSyncJob.cs new file mode 100644 index 0000000..a6839c0 --- /dev/null +++ b/job/DouyinMixSyncJob.cs @@ -0,0 +1,31 @@ +using dy.net.model.dto; +using dy.net.model.entity; +using dy.net.model.response; +using dy.net.service; +using dy.net.utils; + +namespace dy.net.job +{ + public class DouyinMixSyncJob : DouyinBasicSyncJob + { + public DouyinMixSyncJob(DouyinCookieService douyinCookieService, DouyinHttpClientService douyinHttpClientService, DouyinVideoService douyinVideoService, DouyinCommonService douyinCommonService, DouyinFollowService douyinFollowService, DouyinMergeVideoService douyinMergeVideoService, DouyinCollectCateService douyinCollectCateService) : base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService, douyinCollectCateService) + { + } + + + protected override VideoTypeEnum VideoType => VideoTypeEnum.dy_favorite; + + + protected override async Task FetchVideoData(DouyinCookie cookie, string cursor,DouyinFollowed followed,DouyinCollectCate cate) + { + return await douyinHttpClientService.SyncFavoriteVideos(count, cursor, cookie.SecUserId, cookie.Cookies); + } + + protected override bool ShouldContinueSync(DouyinCookie cookie, DouyinVideoInfoResponse data, DouyinFollowed followed=null) + { + return data != null && data.HasMore == 1 && cookie.FavHasSyncd == 0; + } + + + } +} \ No newline at end of file diff --git a/job/DouyinSeriesSyncJob.cs b/job/DouyinSeriesSyncJob.cs new file mode 100644 index 0000000..b083499 --- /dev/null +++ b/job/DouyinSeriesSyncJob.cs @@ -0,0 +1,29 @@ +using dy.net.model.dto; +using dy.net.model.entity; +using dy.net.model.response; +using dy.net.service; +using dy.net.utils; +using System.Net; + +namespace dy.net.job +{ + public class DouyinSeriesSyncJob : DouyinBasicSyncJob + { + public DouyinSeriesSyncJob(DouyinCookieService douyinCookieService, DouyinHttpClientService douyinHttpClientService, DouyinVideoService douyinVideoService, DouyinCommonService douyinCommonService, DouyinFollowService douyinFollowService, DouyinMergeVideoService douyinMergeVideoService, DouyinCollectCateService douyinCollectCateService) : base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService, douyinCollectCateService) + { + } + + + protected override VideoTypeEnum VideoType => VideoTypeEnum.dy_series; + + protected override async Task FetchVideoData(DouyinCookie cookie, string cursor,DouyinFollowed followed, DouyinCollectCate cate) + { + return await douyinHttpClientService.SyncSeriesViedosByMSeriesId(cursor, count, cookie.Cookies,cate.XId); + } + + protected override bool ShouldContinueSync(DouyinCookie cookie, DouyinVideoInfoResponse data, DouyinFollowed followed=null) + { + return data != null && data.HasMore == 1; + } + } +} \ No newline at end of file diff --git a/model/dto/DouyinVideoPageRequestDto.cs b/model/dto/DouyinVideoPageRequestDto.cs index 758d559..356ce18 100644 --- a/model/dto/DouyinVideoPageRequestDto.cs +++ b/model/dto/DouyinVideoPageRequestDto.cs @@ -59,6 +59,6 @@ namespace dy.net.model.dto { public string cookieId { get; set; } - public int cateType { get; set; } + public VideoTypeEnum cateType { get; set; } } } diff --git a/model/dto/VideoTypeEnum.cs b/model/dto/VideoTypeEnum.cs index b678809..3b30636 100644 --- a/model/dto/VideoTypeEnum.cs +++ b/model/dto/VideoTypeEnum.cs @@ -23,7 +23,24 @@ namespace dy.net.model.dto /// 图文视频，无用了，但是不能删... /// [Description("图片视频")] - ImageVideo = 4 + ImageVideo = 4, + + /// + /// 自定义收藏夹 + /// + [Description("收藏短剧")] + dy_custom_collect = 5, + /// + /// 收藏合集 + /// + [Description("收藏合集")] + dy_mix = 6, + /// + /// 收藏短剧 + /// + [Description("收藏短剧")] + dy_series = 7, + } } diff --git a/model/entity/DouyinCollectCate.cs b/model/entity/DouyinCollectCate.cs index d1253ef..403ddcf 100644 --- a/model/entity/DouyinCollectCate.cs +++ b/model/entity/DouyinCollectCate.cs @@ -1,4 +1,5 @@ -using SqlSugar; +using dy.net.model.dto; +using SqlSugar; namespace dy.net.model.entity { @@ -26,7 +27,7 @@ namespace dy.net.model.entity [SugarColumn(Length = 500, IsNullable = false)] public string Name { get; set; } /// - /// 收藏夹Id + /// 收藏夹Id、合集Id、短剧Id /// [SugarColumn(Length =60,IsNullable =false)] public string XId { get; set; } @@ -45,12 +46,16 @@ namespace dy.net.model.entity public bool Sync { get; set; } /// - /// 收藏夹类型（0：自定义收藏夹，1：合集，2：短剧） + /// 收藏夹类型（5：自定义收藏夹，6：合集，7：短剧） /// - public int CateType { get; set; } + public VideoTypeEnum CateType { get; set; } public DateTime CreateTime { get; set; } public DateTime UpdateTime { get; set; } + /// + /// 是否已完结 + /// + public bool IsEnd { get; set; } } } diff --git a/model/entity/DouyinVideo.cs b/model/entity/DouyinVideo.cs index 78efeb1..5287338 100644 --- a/model/entity/DouyinVideo.cs +++ b/model/entity/DouyinVideo.cs @@ -159,5 +159,11 @@ namespace dy.net.model.entity /// 是否仅下载图片或仅下载音频 /// public bool OnlyImgOrOnlyMp3 { get; set; } + + /// + /// 自定义收藏夹、合集、短剧 绑定的Id + /// + [SugarColumn(Length =200,IsNullable =true)] + public string CateId { get; set; } } } diff --git a/model/response/DouyinSeriesListResponse.cs b/model/response/DouyinSeriesListResponse.cs index 9b32a86..2c75198 100644 --- a/model/response/DouyinSeriesListResponse.cs +++ b/model/response/DouyinSeriesListResponse.cs @@ -23,7 +23,7 @@ namespace dy.net.model.response //public object ShortDramaSingleInfo { get; set; } // 原始值为null，兼容扩展 [JsonProperty("series_infos")] - public List ShortDramaList { get; set; } + public List SeriesList { get; set; } [JsonProperty("status_code")] public int ResponseStatusCode { get; set; } @@ -57,7 +57,7 @@ namespace dy.net.model.response /// /// 抖音短剧核心信息 /// - public class DouyinShortDramaInfo + public class DouyinSeriesInfo { [JsonProperty("actors")] public List Actors { get; set; } @@ -78,7 +78,7 @@ namespace dy.net.model.response //public DouyinImageResource DarkModeIcon { get; set; } [JsonProperty("desc")] - public string DramaDescription { get; set; } + public string SeriesDescription { get; set; } //[JsonProperty("directors")] //public List Directors { get; set; } @@ -139,7 +139,7 @@ namespace dy.net.model.response /// 短剧ID /// [JsonProperty("series_id")] - public string DramaId { get; set; } + public string SeriesId { get; set; } //[JsonProperty("series_interactive")] //public DouyinDramaInteractiveConfig InteractiveConfig { get; set; } @@ -147,13 +147,13 @@ namespace dy.net.model.response /// 短剧名称 /// [JsonProperty("series_name")] - public string DramaName { get; set; } + public string SeriesName { get; set; } //[JsonProperty("series_paid_type_list")] //public object PaidTypeList { get; set; } // 原始值为null，兼容扩展 [JsonProperty("series_price")] - public int DramaPrice { get; set; } + public int SeriesPrice { get; set; } //[JsonProperty("series_rank_info")] //public DouyinDramaRankInfo RankInfo { get; set; } diff --git a/repository/DouyinCollectCateRepository.cs b/repository/DouyinCollectCateRepository.cs index 83f621c..6e1f395 100644 --- a/repository/DouyinCollectCateRepository.cs +++ b/repository/DouyinCollectCateRepository.cs @@ -63,7 +63,7 @@ namespace dy.net.repository return await Db.Updateable(cates).ExecuteCommandAsync() > 0; } - internal async Task<(int add, int update, int delete, bool succ)> Sync(List cates, string ckId,int cateType) + internal async Task<(int add, int update, int delete, bool succ)> Sync(List cates, string ckId, VideoTypeEnum cateType) { // 初始化返回结果 int addCount = 0; @@ -83,56 +83,60 @@ namespace dy.net.repository cates = new List(); // 空列表则执行删除所有操作 } - // 2. 开启SqlSugar事务 await Db.Ado.BeginTranAsync(); - // 3. 查询数据库中当前用户的所有分类 + // 3. 查询数据库中当前用户的所有分类，并转为Dictionary优化查找效率（核心优化1） var dbCates = await Db.Queryable() .Where(c => c.CookieId == ckId && c.CateType == cateType) .ToListAsync(); + // 以XId为Key构建字典，避免循环内FirstOrDefault的低效查找 + var dbCateDict = dbCates.ToDictionary(c => c.XId, c => c); + + // 4. 处理新增和【批量】更新逻辑 + var xIds = cates.Select(c => c.XId).ToList(); + // 定义批量更新的待处理集合（核心优化2） + var toUpdateCates = new List(); - // 4. 处理新增和更新逻辑 - var cateIds = cates.Select(c => c.Id).ToList(); foreach (var cate in cates) { - - cate.UpdateTime = DateTime.Now; // 更新时间戳 - - // 检查是否已存在 - if (string.IsNullOrWhiteSpace(cate.Id)) + // 用Dictionary查找，效率更高 + if (dbCateDict.TryGetValue(cate.XId, out var existCate)) { - cate.CreateTime = DateTime.Now; - cate.Id = IdGener.GetLong().ToString(); - // 新增 - await Db.Insertable(cate).ExecuteCommandAsync(); - addCount++; - } - else - { - var existCate = dbCates.FirstOrDefault(x => x.Id == cate.Id); - if (existCate == null) - { - Serilog.Log.Error($" cate {cate.Id} id 不存在"); - continue; - } // 对比字段是否有变化（根据实际业务字段调整） bool isChanged = !string.Equals(existCate.Name, cate.Name, StringComparison.Ordinal); if (isChanged) { - // 更新（只更新有变化的字段，或全量更新） - await Db.Updateable(cate) - .IgnoreColumns(ignoreAllNullColumns: true) // 忽略空值字段 - .Where(c => c.Id == cate.Id) - .ExecuteCommandAsync(); - updateCount++; + // 补充更新信息，加入批量更新集合 + cate.UpdateTime = DateTime.Now; + // 确保更新时的Id正确（从数据库已有记录中获取） + cate.Id = existCate.Id; + toUpdateCates.Add(cate); } } + else + { + // 新增逻辑保持不变 + cate.CreateTime = DateTime.Now; + cate.Id = IdGener.GetLong().ToString(); + await Db.Insertable(cate).ExecuteCommandAsync(); + addCount++; + } } - // 5. 处理删除逻辑：数据库中有但传入列表中没有的分类 - var deleteCates = dbCates.Where(c => !cateIds.Contains(c.Id)).ToList(); + // 5. 执行批量更新（核心优化3：批量提交，减少数据库IO） + if (toUpdateCates.Any()) + { + updateCount = await Db.Updateable(toUpdateCates) + .IgnoreColumns(x => new { x.Sync, x.CreateTime, x.CookieId }) + .IgnoreNullColumns() // 忽略空值字段 + .Where(c => c.Id == c.Id) // 批量更新默认按主键匹配，此处显式指定（也可省略，SqlSugar自动识别主键） + .ExecuteCommandAsync(); + } + + // 6. 处理删除逻辑：数据库中有但传入列表中没有的分类 + var deleteCates = dbCates.Where(c => !xIds.Contains(c.XId)).ToList(); if (deleteCates.Any()) { var deleteIds = deleteCates.Select(c => c.Id).ToList(); @@ -141,20 +145,20 @@ namespace dy.net.repository .ExecuteCommandAsync(); } - // 6. 提交事务 + // 7. 提交事务 await Db.Ado.CommitTranAsync(); isSuccess = true; } catch (Exception ex) { - // 异常时回滚事务 - await Db.Ado.RollbackTranAsync(); + // 异常时回滚事务（补充：避免Db.Ado未初始化事务导致的空引用） + await Db.Ado?.RollbackTranAsync(); isSuccess = false; - // throw; - Serilog.Log.Error($"同步收藏夹:类型-{cateType}异常，{ex.StackTrace}" ); + Serilog.Log.Error($"同步收藏夹:类型-{cateType}异常，{ex.StackTrace}"); } - // 7. 返回最终结果 + //Serilog.Log.Debug($"本次收藏夹{cateType}同步结果,新增:{addCount},更新:{updateCount},删除:{deleteCount}"); + // 8. 返回最终结果 return (addCount, updateCount, deleteCount, isSuccess); } } diff --git a/repository/DouyinCookieRepository.cs b/repository/DouyinCookieRepository.cs index 1891254..eda2cc5 100644 --- a/repository/DouyinCookieRepository.cs +++ b/repository/DouyinCookieRepository.cs @@ -16,7 +16,8 @@ namespace dy.net.repository { // 1. 初始化查询：先加固定条件 Status == 1 var query = Db.Queryable() - .Where(x => x.Status == 1); // 固定条件（必选） + .Where(x => x.Status == 1) + .Where(x=>!string.IsNullOrWhiteSpace(x.Cookies)); // 固定条件（必选） // 2. 若传入自定义条件，叠加 Where（自动 AND 组合） if (whereExpression != null) diff --git a/repository/DouyinFollowRepository.cs b/repository/DouyinFollowRepository.cs index b89cdda..a0214cb 100644 --- a/repository/DouyinFollowRepository.cs +++ b/repository/DouyinFollowRepository.cs @@ -84,7 +84,7 @@ namespace dy.net.repository public async Task> GetSyncFollows(string userId) { return await this.Db.Queryable() - .Where(x => x.OpenSync == true).Where(x => x.mySelfId == userId) + .Where(x => x.OpenSync == true).Where(x => x.mySelfId == userId).Where(x=>!string.IsNullOrWhiteSpace(x.SecUid)) .ToListAsync(); } diff --git a/service/DouyinCollectCateService.cs b/service/DouyinCollectCateService.cs index 8a63863..77cdfe3 100644 --- a/service/DouyinCollectCateService.cs +++ b/service/DouyinCollectCateService.cs @@ -24,6 +24,8 @@ namespace dy.net.service { return await douyinCollectCateRepository.InsertAsync(cate); } + + /// @@ -43,7 +45,7 @@ namespace dy.net.service /// /// /// - public async Task<(int add, int update,int delete, bool succ)> Sync(List cates, string ckId,int cateType) + public async Task<(int add, int update,int delete, bool succ)> Sync(List cates, string ckId,VideoTypeEnum cateType) { return await douyinCollectCateRepository.Sync(cates, ckId, cateType); } @@ -66,7 +68,7 @@ namespace dy.net.service /// 获取需要同步的合集、短剧、收藏夹 /// /// - public async Task> GetSyncCates(string cookieId,int cateType) + public async Task> GetSyncCates(string cookieId, VideoTypeEnum cateType) { return await douyinCollectCateRepository.GetListAsync(x => x.CookieId == cookieId && x.CateType == cateType); } diff --git a/service/DouyinCommonService.cs b/service/DouyinCommonService.cs index 7d947c6..c06119c 100644 --- a/service/DouyinCommonService.cs +++ b/service/DouyinCommonService.cs @@ -108,75 +108,6 @@ namespace dy.net.service } - /// - /// 初始化一些数据，兼容旧版数据结构。 - /// - public void UpdateCollectViedoType() - { - //更新视频类型字段-兼容老版本 - string sql = @"UPDATE dy_collect_video - SET ViedoType = CASE - WHEN ViedoType = '0' THEN 0 - WHEN ViedoType = '1' THEN 1 - WHEN ViedoType = '2' THEN 2 - WHEN ViedoType = '3' THEN 3 - WHEN ViedoType = '4' THEN 4 - ELSE NULL - END;"; - - sqlSugarClient.Ado.ExecuteCommand(sql); - - //更新关注表的IsNoFollowed字段为空的数据为0--兼容老版本-新加的字段 - string followUpdateSql = @"Update dy_follow SET IsNoFollowed=0 WHERE IsNoFollowed is NULL"; - sqlSugarClient.Ado.ExecuteCommand(followUpdateSql); - - //更新图片视频的合并状态 - string updateIsMergeVideoSql = @"UPDATE dy_collect_video SET IsMergeVideo = 1 WHERE IsMergeVideo IS NULL and ViedoType='4';"; - sqlSugarClient.Ado.ExecuteCommand(updateIsMergeVideoSql); - - //更新非图片视频的合并状态 - string updateNoIsMergeVideoSql = @"UPDATE dy_collect_video SET IsMergeVideo = 0 WHERE IsMergeVideo IS NULL and ViedoType<>'4';"; - sqlSugarClient.Ado.ExecuteCommand(updateNoIsMergeVideoSql); - - //强制开启去重 - sqlSugarClient.Updateable().SetColumns(x => new AppConfig { AutoDistinct = true }).Where(x => !string.IsNullOrWhiteSpace(x.Id)).ExecuteCommand(); - //重新根据保存路径更新图片视频的类型为喜欢，收藏或关注 - var collectViedos = sqlSugarClient.Queryable().Where(x => x.ViedoType == VideoTypeEnum.ImageVideo).ToList(); - if (collectViedos != null && collectViedos.Any()) - { - var cookies = sqlSugarClient.Queryable().ToList(); - collectViedos.ForEach(x => - { - var ck = cookies.FirstOrDefault(c => c.Id == x.CookieId); - if (ck != null) - { - - var savePath = x.VideoSavePath; - if (savePath != null) - { - if (savePath.StartsWith(ck.SavePath)) - { - x.ViedoType = VideoTypeEnum.dy_collects; - } - else if (savePath.StartsWith(ck.UpSavePath)) - { - x.ViedoType = VideoTypeEnum.dy_follows; - } - else if (savePath.StartsWith(ck.FavSavePath)) - { - x.ViedoType = VideoTypeEnum.dy_favorite; - } - else - { - x.ViedoType = VideoTypeEnum.dy_collects; - } - } - } - }); - //只更新图片视频的类型 - sqlSugarClient.Updateable(collectViedos).UpdateColumns(x => new DouyinVideo { ViedoType = x.ViedoType }).IgnoreColumns(true).ExecuteCommand(); - } - } /// /// 重置所有Cookie的同步状态为0 /// diff --git a/service/DouyinCookieService.cs b/service/DouyinCookieService.cs index 8ea3368..de6080c 100644 --- a/service/DouyinCookieService.cs +++ b/service/DouyinCookieService.cs @@ -18,7 +18,7 @@ namespace dy.net.service _cookieRepository = cookieRepository; } - public Task> GetAllOpendAsync(Expression> whereExpression = null) + public Task> GetOpendCookiesAsync(Expression> whereExpression = null) { return _cookieRepository.GetAllCookies(whereExpression); } diff --git a/service/DouyinHttpClientService.cs b/service/DouyinHttpClientService.cs index 86527e3..733146d 100644 --- a/service/DouyinHttpClientService.cs +++ b/service/DouyinHttpClientService.cs @@ -110,7 +110,10 @@ namespace dy.net.service if (respose.IsSuccessStatusCode) { var data = await respose.Content.ReadAsStringAsync(); - return JsonConvert.DeserializeObject(data); + var model= JsonConvert.DeserializeObject(data); + if (model == null) + Serilog.Log.Error($"SyncCollectVideos fail: {data}"); + return model; } else { @@ -160,6 +163,8 @@ namespace dy.net.service { var data = await respose.Content.ReadAsStringAsync(); var model = JsonConvert.DeserializeObject(data); + if (model == null) + Serilog.Log.Error($"SyncCollectFolderList fail: {data}"); return model; } else @@ -181,15 +186,15 @@ namespace dy.net.service /// /// /// - /// + /// /// /// - public async Task SyncCollectVideosByCollectId(string cursor, string count, string cookie, DouyinCollectItem collect) + public async Task SyncCollectVideosByCollectId(string cursor, string count, string cookie, string collectsId) { #region 检查参数 - if (collect is null || string.IsNullOrWhiteSpace(collect.CollectsId)) + if (string.IsNullOrWhiteSpace(collectsId)) { - throw new ArgumentException($"“{nameof(collect)}”不能为 null 或空。", nameof(collect)); + throw new ArgumentException($"“{nameof(collectsId)}”不能为 null 或空。", nameof(collectsId)); } if (string.IsNullOrWhiteSpace(cursor)) { @@ -217,7 +222,7 @@ namespace dy.net.service { requestParameters["cursor"] = cursor; requestParameters["count"] = count; - requestParameters["collects_id"] = collect.CollectsId; + requestParameters["collects_id"] = collectsId; } var respose = await GetHttpResponseMessage(HttpMethod.Post, requestUrl, requestParameters, refererValue, cookie); @@ -225,7 +230,10 @@ namespace dy.net.service if (respose.IsSuccessStatusCode) { var data = await respose.Content.ReadAsStringAsync(); - return JsonConvert.DeserializeObject(data); + var model= JsonConvert.DeserializeObject(data); + if (model == null) + Serilog.Log.Error($"SyncCollectVideosByCollectId fail: {data}"); + return model; } else { @@ -266,7 +274,7 @@ namespace dy.net.service var requestParameters = DouyinRequestParamManager.DouyinMixListParams; { // 添加动态参数 - requestParameters["count"] = "20"; + requestParameters["count"] = "100"; requestParameters["cursor"] = cursor; //页码 } @@ -275,6 +283,8 @@ namespace dy.net.service { var data = await respose.Content.ReadAsStringAsync(); var model = JsonConvert.DeserializeObject(data); + if (model == null) + Serilog.Log.Error($"SyncMixList fail: {data}"); return model; } else @@ -300,12 +310,12 @@ namespace dy.net.service /// /// /// - public async Task SyncMixViedosByMixId(string cursor, string count, string cookie, DouyinMixInfo mix) + public async Task SyncMixViedosByMixId(string cursor, string count, string cookie, string mixId) { #region 检查参数 - if (mix is null ||string.IsNullOrWhiteSpace(mix.MixId)) + if (string.IsNullOrWhiteSpace(mixId)) { - throw new ArgumentException($"“{nameof(mix)}”不能为 null 或空。", nameof(mix)); + throw new ArgumentException($"“{nameof(mixId)}”不能为 null 或空。", nameof(mixId)); } if (string.IsNullOrWhiteSpace(cursor)) { @@ -333,7 +343,7 @@ namespace dy.net.service { requestParameters["cursor"] = cursor; requestParameters["count"] = count; - requestParameters["mix_id"] = mix.MixId; + requestParameters["mix_id"] = mixId; } var respose = await GetHttpResponseMessage(HttpMethod.Post, requestUrl, requestParameters, refererValue, cookie); @@ -341,7 +351,10 @@ namespace dy.net.service if (respose.IsSuccessStatusCode) { var data = await respose.Content.ReadAsStringAsync(); - return JsonConvert.DeserializeObject(data); + var model = JsonConvert.DeserializeObject(data); + if (model == null) + Serilog.Log.Error($"SyncMixViedosByMixId fail: {data}"); + return model; } else { @@ -365,7 +378,7 @@ namespace dy.net.service /// /// /// - public async Task SyncShortList(string cookie, string cursor) + public async Task SyncSeriesList(string cookie, string cursor) { if (string.IsNullOrWhiteSpace(cookie)) { @@ -381,7 +394,7 @@ namespace dy.net.service var requestParameters = DouyinRequestParamManager.DouyinSeriesListParams; { // 添加动态参数 - requestParameters["count"] = "20"; + requestParameters["count"] = "100"; requestParameters["cursor"] = cursor; //页码 } @@ -390,6 +403,8 @@ namespace dy.net.service { var data = await respose.Content.ReadAsStringAsync(); var model = JsonConvert.DeserializeObject(data); + if (model == null) + Serilog.Log.Error($"SyncShortList fail: {data}"); return model; } else @@ -412,15 +427,15 @@ namespace dy.net.service /// /// /// - /// + /// /// /// - public async Task SyncSeriesViedosByMSeriesId(string cursor, string count, string cookie, DouyinShortDramaInfo series) + public async Task SyncSeriesViedosByMSeriesId(string cursor, string count, string cookie, string seriesId) { #region 检查参数 - if (series is null || string.IsNullOrWhiteSpace(series.DramaId)) + if (string.IsNullOrWhiteSpace(seriesId)) { - throw new ArgumentException($"“{nameof(series)}”不能为 null 或空。", nameof(series)); + throw new ArgumentException($"“{nameof(seriesId)}”不能为 null 或空。", nameof(seriesId)); } if (string.IsNullOrWhiteSpace(cursor)) { @@ -448,7 +463,7 @@ namespace dy.net.service { requestParameters["cursor"] = cursor; requestParameters["count"] = count; - requestParameters["series_id"] = series.DramaId; + requestParameters["series_id"] = seriesId; } var respose = await GetHttpResponseMessage(HttpMethod.Post, requestUrl, requestParameters, refererValue, cookie); @@ -456,7 +471,10 @@ namespace dy.net.service if (respose.IsSuccessStatusCode) { var data = await respose.Content.ReadAsStringAsync(); - return JsonConvert.DeserializeObject(data); + var model= JsonConvert.DeserializeObject(data); + if (model == null) + Serilog.Log.Error($"SyncSeriesViedosByMSeriesId fail: {data}"); + return model; } else { @@ -525,7 +543,10 @@ namespace dy.net.service if (respose.IsSuccessStatusCode) { var data = await respose.Content.ReadAsStringAsync(); - return JsonConvert.DeserializeObject(data); + var model= JsonConvert.DeserializeObject(data); + if (model == null) + Serilog.Log.Error($"SyncFavoriteVideos fail: {data}"); + return model; } else { @@ -594,7 +615,10 @@ namespace dy.net.service if (respose.IsSuccessStatusCode) { var data = await respose.Content.ReadAsStringAsync(); - return JsonConvert.DeserializeObject(data); + var model= JsonConvert.DeserializeObject(data); + if (model == null) + Serilog.Log.Error($"SyncUpderPostVideos fail: {data}"); + return model; } else { diff --git a/service/DouyinQuartzJobService.cs b/service/DouyinQuartzJobService.cs index 3859f86..8590a08 100644 --- a/service/DouyinQuartzJobService.cs +++ b/service/DouyinQuartzJobService.cs @@ -57,7 +57,7 @@ namespace dy.net.service { "follow_user", new JobConfig( - typeof(DouyinFollowedUsersSyncJob), + typeof(DouyinFollowsAndCollnectsSyncJob), "dy.job.key.follow_user", "dy.trigger.key.follow_user", "抖音关注同步任务") @@ -65,7 +65,7 @@ namespace dy.net.service { "follow_user_once", new JobConfig( - typeof(DouyinFollowedUsersSyncJob), + typeof(DouyinFollowsAndCollnectsSyncJob), "dy.job.key.follow_user_once", "dy.trigger.key.follow_user_once", "抖音关注同步任务(单次执行)")