From 830b6769f936ada626ff97157df0c9fba7474f21 Mon Sep 17 00:00:00 2001 From: lijianyou Date: Mon, 1 Dec 2025 12:06:41 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=A7=86=E9=A2=91=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=BA=93=E6=95=B0=E6=8D=AE=E4=BF=9D=E5=AD=98=E9=80=BB?= =?UTF-8?q?=E8=BE=91=EF=BC=8C=E6=AF=8F=E6=AC=A1=E4=BB=BB=E5=8A=A1=E6=89=A7?= =?UTF-8?q?=E8=A1=8C=E4=BF=9D=E5=AD=98=E8=A7=86=E9=A2=91=E6=95=B0=E9=87=8F?= =?UTF-8?q?=E8=B6=85=E8=BF=87=E8=AE=BE=E7=BD=AE=E7=9A=84=E6=AF=8F=E6=AC=A1?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E6=95=B0=E9=87=8F=E6=97=B6=20=E8=B7=B3?= =?UTF-8?q?=E5=87=BA=E5=BD=93=E5=89=8D=E5=BE=AA=E7=8E=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- job/DouyinBasicSyncJob.cs | 55 ++++++++++------------- model/AppConfig.cs | 1 - repository/DouyinVideoRepository.cs | 2 +- service/DouyinVideoService.cs | 69 ++++++++++++++++++++--------- 4 files changed, 73 insertions(+), 54 deletions(-) diff --git a/job/DouyinBasicSyncJob.cs b/job/DouyinBasicSyncJob.cs index 9e54808..8e93267 100644 --- a/job/DouyinBasicSyncJob.cs +++ b/job/DouyinBasicSyncJob.cs @@ -72,7 +72,7 @@ namespace dy.net.job /// /// 是否下载图片并合成视频 /// - private bool _downImageVideo; + //private bool _downImageVideo; #endregion @@ -138,13 +138,10 @@ namespace dy.net.job if (config.BatchCount > 0) count = config.BatchCount.ToString(); - // 3. 初始化是否下载图片视频的设置 - InitializeDownImageVideoSetting(config); - - // 4. 在处理Cookie之前执行的预处理操作（子类可重写） + // 3. 在处理Cookie之前执行的预处理操作 await BeforeProcessCookies(); - // 5. 获取所有有效的Cookie + // 4. 获取所有有效的Cookie var cookies = await GetValidCookies(); if (cookies == null || !cookies.Any()) { @@ -314,10 +311,10 @@ namespace dy.net.job int syncCount = 0; // 本次同步成功的视频数量 string cursor = "0"; // 初始游标 bool hasMore = true; // 是否还有更多数据 + + //查询关注列表开启了同步的关注 var follows = await douyinFollowService.GetSyncFollows(cookie.MyUserId); - - //var ups = JsonConvert.DeserializeObject>(cookie.UpSecUserIds); var firstUp = follows?.Where(x => !string.IsNullOrWhiteSpace(x.SecUid)).FirstOrDefault(); if (firstUp == null) { @@ -385,6 +382,13 @@ namespace dy.net.job // 保存视频信息到数据库 syncCount += await SaveVideos(videos); + //当syncCount达到上限时，跳出循环 + if (config.BatchCount > 0 && syncCount >= config.BatchCount) + { + Log.Debug($"{JobType}-Cookie[{cookie.UserName}]本次同步达到上限{config.BatchCount}，停止同步!!!"); + break; + } + // 随机延迟，模拟人类操作，避免请求过快 await Task.Delay(_random.Next(5, 10) * 1000); } @@ -414,7 +418,7 @@ namespace dy.net.job { if (File.Exists(exitVideo.VideoSavePath)) { - Serilog.Log.Debug($"视频-{exitVideo.AwemeId}-[{exitVideo.VideoTitle}]已存在，跳过"); + Log.Debug($"视频-{exitVideo.AwemeId}-[{exitVideo.VideoTitle}]已存在，跳过"); continue;// 已存在则跳过 } } @@ -434,14 +438,11 @@ namespace dy.net.job videos.Add(video); // 如果配置了下载图片视频，则处理图片集并合成视频 - if (_downImageVideo) + if (config.DownImageVideo || config.DownMp3 || config.DownImage) { - if (config.DownImageVideo || config.DownMp3 || config.DownImage) - { - var mergevideo = await ProcessImageSetAndMergeToVideo(cookie, item, data, config, followed); - if (mergevideo != null) - videos.Add(mergevideo); - } + var mergevideo = await ProcessImageSetAndMergeToVideo(cookie, item, data, config, followed); + if (mergevideo != null) + videos.Add(mergevideo); } } return videos; @@ -480,7 +481,11 @@ namespace dy.net.job var savePath = Path.Combine(saveFolder, fileName); // 如果文件已存在，跳过 - if (File.Exists(savePath)) return null; + if (File.Exists(savePath)) + { + Log.Debug($"{JobType}-视频[{DouyinFileNameHelper.SanitizePath(item.Desc)}]已存在，跳过下载."); + return null; + } Log.Debug($"{JobType}-视频[{DouyinFileNameHelper.SanitizePath(item.Desc)}]开始下载..."); // 随机延迟，模拟人类操作 @@ -656,7 +661,7 @@ namespace dy.net.job if (!videos.Any()) return 0; try { - await douyinVideoService.batchInsert(videos); + await douyinVideoService.BatchInsertOrUpdate(videos); var redowns = await douyinCommonService.GetAllRedown(); if (redowns != null && redowns.Any()) @@ -798,20 +803,6 @@ namespace dy.net.job ); } - /// - /// 初始化是否下载图片视频的设置 - /// 从环境变量和配置文件中读取设置，环境变量优先级更高 - /// - /// 应用配置 - private void InitializeDownImageVideoSetting(AppConfig config) - { - var downImageVideoConfig = Appsettings.Get("DOWN_IMGVIDEO"); - if (!string.IsNullOrWhiteSpace(downImageVideoConfig)) - { - downImageVideoConfig = downImageVideoConfig.ToLower(); - _downImageVideo = config.DownImageVideo && downImageVideoConfig == "1"; - } - } /// /// 清理保存失败的视频文件 diff --git a/model/AppConfig.cs b/model/AppConfig.cs index 1f3d9a4..1bfd513 100644 --- a/model/AppConfig.cs +++ b/model/AppConfig.cs @@ -73,6 +73,5 @@ namespace dy.net.model /// 自动去重-逻辑是遇到相同ID的视频直接跳过 /// public bool AutoDistinct { get; set; } - } } diff --git a/repository/DouyinVideoRepository.cs b/repository/DouyinVideoRepository.cs index 90dc9db..2005f6a 100644 --- a/repository/DouyinVideoRepository.cs +++ b/repository/DouyinVideoRepository.cs @@ -40,7 +40,7 @@ namespace dy.net.repository var where = this.Db.Queryable() //.WhereIF(!string.IsNullOrWhiteSpace(title), x => x.VideoTitle.Contains(title)) .WhereIF(!string.IsNullOrWhiteSpace(dto.Title), x => x.VideoTitle.Contains(dto.Title)) - .WhereIF(!string.IsNullOrWhiteSpace(dto.Author), x => x.Author == dto.Author) + .WhereIF(!string.IsNullOrWhiteSpace(dto.Author), x => x.Author.Contains(dto.Author)) .WhereIF(start.HasValue, x => x.SyncTime >= start.Value) .WhereIF(end.HasValue, x => x.SyncTime <= end.Value) .WhereIF(start2.HasValue, x => x.CreateTime >= start2.Value) diff --git a/service/DouyinVideoService.cs b/service/DouyinVideoService.cs index 96d26eb..18a1720 100644 --- a/service/DouyinVideoService.cs +++ b/service/DouyinVideoService.cs @@ -3,6 +3,7 @@ using dy.net.dto; using dy.net.model; using dy.net.repository; using dy.net.utils; +using Newtonsoft.Json; using System.ComponentModel; using System.Threading.Tasks; @@ -21,38 +22,66 @@ namespace dy.net.service } - public async Task batchInsert(List videos) + public async Task BatchInsertOrUpdate(List videos) { - - // 边界处理：如果传入的列表为空，直接返回成功（或根据业务返回false） + // 边界处理：传入列表为空直接返回成功 if (videos == null || !videos.Any()) return true; - // 1. 提取待插入的所有AwemeId（去重，减少数据库查询压力） - var newAwemeIds = videos.Select(x => x.AwemeId) - .Distinct() - .ToList(); + // 1. 提取所有AwemeId（无需去重，用户保证无重复） + var allAwemeIds = videos.Select(v => v.AwemeId).ToList(); - // 2. 查询数据库中已存在的AwemeId（只查需要的字段，提高效率） - var existingAwemeIds = await _dyCollectVideoRepository - .Query(x => newAwemeIds.Contains(x.AwemeId)) // 使用Query方法构建查询 - .Select(x => x.AwemeId) // 只获取AwemeId，减少数据传输 + // 2. 查询数据库中已存在的视频记录（用于后续更新） + var existingVideos = await _dyCollectVideoRepository + .Query(x => allAwemeIds.Contains(x.AwemeId)) .ToListAsync(); - // 3. 过滤出数据库中不存在的视频（只保留新记录） + // 3. 分拆数据集：不存在的（插入）、已存在的（更新） + var existingAwemeIdSet = existingVideos.Select(v => v.AwemeId).ToHashSet(); var videosToInsert = videos - .Where(video => !existingAwemeIds.Contains(video.AwemeId)) + .Where(v => !existingAwemeIdSet.Contains(v.AwemeId)) + .ToList(); + var videosToUpdate = videos + .Where(v => existingAwemeIdSet.Contains(v.AwemeId)) .ToList(); - // 4. 如果没有需要插入的新记录，直接返回成功 - if (!videosToInsert.Any()) - return true; + // 4. 事务包裹：确保插入/更新原子性 + var transaction = await _dyCollectVideoRepository.UseTranAsync(async () => + { + int insertedCount = 0; + int updatedCount = 0; - // 5. 批量插入过滤后的新记录 - var insertedCount = await _dyCollectVideoRepository.InsertRangeAsync(videosToInsert); + // 5. 批量插入新记录 + if (videosToInsert.Any()) + { + insertedCount = await _dyCollectVideoRepository.InsertRangeAsync(videosToInsert); + } - // 返回是否插入成功（至少插入一条） - return insertedCount > 0; + // 6. 批量更新已存在记录（核心逻辑） + if (videosToUpdate.Any()) + { + // 建立AwemeId与待更新数据的映射（O(1)匹配效率） + var updateMap = videosToUpdate.ToDictionary(v => v.AwemeId); + + // 遍历已存在实体，赋值需要更新的字段 + List updates= new List(); + foreach (var existingVideo in existingVideos) + { + if (updateMap.TryGetValue(existingVideo.AwemeId, out var updateData)) + { + existingVideo.VideoSavePath = updateData.VideoSavePath; + existingVideo.VideoCoverSavePath = updateData.VideoCoverSavePath; + } + } + // 批量更新数据库 + updatedCount = await _dyCollectVideoRepository.UpdateRangeAsync(existingVideos); + } + + }, ex => + { + Serilog.Log.Error(ex, "批量插入/更新抖音视频失败，AwemeIds：{AwemeIds}", string.Join(",", allAwemeIds)); + }); + return transaction; }