nanxun/douyin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

短剧、合集

Browse Source
This commit is contained in:
jianzhichu
2026-01-20 00:20:37 +08:00
parent 7c6ee0a8a6
commit 7175b3858a
25 changed files with 779 additions and 745 deletions
Download Patch File Download Diff File Expand all files Collapse all files
job/DouYinCollectSyncJob.cs
+11 -58
View File
@@ -9,14 +9,9 @@ namespace dy.net.job
{
public class DouyinCollectSyncJob : DouyinBasicSyncJob
{
public DouyinCollectSyncJob(
DouyinCookieService douyinCookieService,
DouyinHttpClientService douyinHttpClientService,
DouyinVideoService douyinVideoService,
DouyinCommonService douyinCommonService,DouyinFollowService douyinFollowService,DouyinMergeVideoService douyinMergeVideoService)
: base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService,douyinFollowService, douyinMergeVideoService) { }
public DouyinCollectSyncJob(DouyinCookieService douyinCookieService, DouyinHttpClientService douyinHttpClientService, DouyinVideoService douyinVideoService, DouyinCommonService douyinCommonService, DouyinFollowService douyinFollowService, DouyinMergeVideoService douyinMergeVideoService, DouyinCollectCateService douyinCollectCateService) : base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService, douyinCollectCateService)
{
}
protected override VideoTypeEnum VideoType => VideoTypeEnum.dy_collects;
protected override async Task BeforeProcessCookies()
@@ -24,30 +19,14 @@ namespace dy.net.job
var now = DateTime.Now;
if (now.Hour == 1 && now.Minute < 30)
{
//douyinCommonService.UpdateAllCookieSyncedToZero();
//顺手清理下日志文件
LogFileCleaner.CleanOldLogFiles(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "logs"), 1);
await Task.Delay(200);
}
}
protected override async Task<List<DouyinCookie>> GetValidCookies()
{
return await douyinCookieService.GetAllOpendAsync(x => !string.IsNullOrWhiteSpace(x.SavePath));
}
protected override bool IsCookieValid(DouyinCookie cookie)
protected override async Task<DouyinVideoInfoResponse> FetchVideoData(DouyinCookie cookie, string cursor, DouyinFollowed followed, DouyinCollectCate cate)
{
return !string.IsNullOrWhiteSpace(cookie.Cookies)&& !string.IsNullOrWhiteSpace(cookie.SavePath);
}
protected override async Task<DouyinVideoInfoResponse> FetchVideoData(DouyinCookie cookie, string cursor, DouyinFollowed followed,DouyinCollectItem collectItem=null)
{
if (cookie.UseCollectFolder && collectItem != null && !string.IsNullOrWhiteSpace(collectItem.CollectsId))
{
return await douyinHttpClientService.SyncCollectVideosByCollectId(cursor, count, cookie.Cookies, collectItem);
}
return await douyinHttpClientService.SyncCollectVideos(cursor, count, cookie.Cookies);
}
@@ -56,16 +35,6 @@ namespace dy.net.job
return data != null && data.HasMore == 1 && cookie.CollHasSyncd == 0;
}
protected override string GetNextCursor(DouyinVideoInfoResponse data)
{
return data?.Cursor ?? "0";
}
protected override string GetAuthorAvatarBasePath(DouyinCookie cookie)
{
return Path.Combine(cookie.SavePath, "author");
}
protected override async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount,DouyinFollowed followed)
{
if (syncCount > 0)
@@ -81,31 +50,15 @@ namespace dy.net.job
}
protected override VideoEntityDifferences GetVideoEntityDifferences(DouyinCookie cookie, Aweme item)
{
return new VideoEntityDifferences();
}
protected override string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed, DouyinCollectItem collectItem)
protected override string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed, DouyinCollectCate cate)
{
if(collectItem!=null&&cookie.UseCollectFolder &&!string.IsNullOrWhiteSpace(collectItem.CollectsId))
{
var safeCollectName = DouyinFileNameHelper.SanitizeLinuxFileName(collectItem.CollectsName, "", true);
var folderCollect = Path.Combine(cookie.SavePath, safeCollectName);
var safeVideoDesc = DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId, true);
var folder = Path.Combine(folderCollect, safeVideoDesc);
if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
return folder;
}
else
{
var (tag1, _, _) = GetVideoTags(item);
var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : DouyinFileNameHelper.SanitizeLinuxFileName(tag1, "", true);
var folder = Path.Combine(cookie.SavePath, safeTag1, $"{DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId, true)}");
if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
return folder;
}
var (tag1, _, _) = GetVideoTags(item);
var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : DouyinFileNameHelper.SanitizeLinuxFileName(tag1, "", true);
var folder = Path.Combine(cookie.SavePath, safeTag1, $"{DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId, true)}");
if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
return folder;
}
}
}
job/DouYinFavoritSyncJob.cs
+7 -27
View File
@@ -8,29 +8,19 @@ namespace dy.net.job
{
public class DouyinFavoritSyncJob : DouyinBasicSyncJob
{
public DouyinFavoritSyncJob(
DouyinCookieService douyinCookieService,
DouyinHttpClientService douyinHttpClientService,
DouyinVideoService douyinVideoService,
DouyinCommonService douyinCommonService,DouyinFollowService douyinFollowService,DouyinMergeVideoService douyinMergeVideoService)
: base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService) { }
public DouyinFavoritSyncJob(DouyinCookieService douyinCookieService, DouyinHttpClientService douyinHttpClientService, DouyinVideoService douyinVideoService, DouyinCommonService douyinCommonService, DouyinFollowService douyinFollowService, DouyinMergeVideoService douyinMergeVideoService, DouyinCollectCateService douyinCollectCateService) : base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService, douyinCollectCateService)
{
}
protected override VideoTypeEnum VideoType => VideoTypeEnum.dy_favorite;
protected override async Task<List<DouyinCookie>> GetValidCookies()
protected override async Task<List<DouyinCookie>> GetSyncCookies()
{
return await douyinCookieService.GetAllOpendAsync(x=> !string.IsNullOrWhiteSpace(x.FavSavePath));
return await douyinCookieService.GetOpendCookiesAsync(x=> !string.IsNullOrWhiteSpace(x.FavSavePath)&&!string.IsNullOrWhiteSpace(x.SecUserId));
}
protected override bool IsCookieValid(DouyinCookie cookie)
{
return !string.IsNullOrWhiteSpace(cookie.Cookies) && cookie.Cookies.Length >= 1000 &&
!string.IsNullOrWhiteSpace(cookie.FavSavePath) &&
!string.IsNullOrWhiteSpace(cookie.SecUserId) && cookie.SecUserId.Length >= 10;
}
protected override async Task<DouyinVideoInfoResponse> FetchVideoData(DouyinCookie cookie, string cursor,DouyinFollowed followed, DouyinCollectItem collectId)
protected override async Task<DouyinVideoInfoResponse> FetchVideoData(DouyinCookie cookie, string cursor,DouyinFollowed followed, DouyinCollectCate cate)
{
return await douyinHttpClientService.SyncFavoriteVideos(count, cursor, cookie.SecUserId, cookie.Cookies);
}
@@ -40,16 +30,6 @@ namespace dy.net.job
return data != null && data.HasMore == 1 && cookie.FavHasSyncd == 0;
}
protected override string GetNextCursor(DouyinVideoInfoResponse data)
{
return data?.MaxCursor ?? "0";
}
protected override string GetAuthorAvatarBasePath(DouyinCookie cookie)
{
return Path.Combine(cookie.FavSavePath, "author");
}
protected override async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount, DouyinFollowed followed)
{
@@ -65,7 +45,7 @@ namespace dy.net.job
}
}
protected override string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed, DouyinCollectItem collectItem)
protected override string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed,DouyinCollectCate cate)
{
var (tag1, _, _) = GetVideoTags(item);
var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : DouyinFileNameHelper.SanitizeLinuxFileName(tag1,"",true);
job/DouyinBasicSyncJob.cs
+180 -230
View File
@@ -24,17 +24,14 @@ namespace dy.net.job
/// 抖音Cookie服务，用于获取和管理用户Cookie
/// </summary>
protected readonly DouyinCookieService douyinCookieService;
/// <summary>
/// 抖音HTTP客户端服务，用于发送HTTP请求
/// </summary>
protected readonly DouyinHttpClientService douyinHttpClientService;
/// <summary>
/// 抖音视频服务，用于视频信息的数据库操作
/// </summary>
protected readonly DouyinVideoService douyinVideoService;
/// <summary>
/// 抖音通用服务，用于获取应用配置等
/// </summary>
@@ -43,20 +40,20 @@ namespace dy.net.job
/// 抖音关注列表
/// </summary>
private readonly DouyinFollowService douyinFollowService;
/// <summary>
/// 图文合成视频
/// </summary>
private readonly DouyinMergeVideoService douyinMergeVideoService;
/// <summary>
/// 收藏夹、短剧、合集
/// </summary>
private readonly DouyinCollectCateService douyinCollectCateService;
/// <summary>
/// 随机数生成器，用于生成随机延迟，模拟人类操作
/// </summary>
protected readonly Random _random = new Random();
/// <summary>
/// 每页请求的视频数量，可通过配置文件修改
/// 每页请求的视频数量.不可修改
/// </summary>
protected string count = "18";
@@ -79,12 +76,6 @@ namespace dy.net.job
#region 抽象属性
/// <summary>
/// 任务类型名称，用于日志记录和区分不同的同步任务
/// 子类必须实现此属性并返回具体的任务类型
/// </summary>
//protected abstract string VideoType { get; }
protected abstract VideoTypeEnum VideoType { get; }
#endregion
@@ -100,13 +91,15 @@ namespace dy.net.job
/// <param name="douyinCommonService">抖音通用服务</param>
/// <param name="douyinFollowService">抖音关注的</param>
/// <param name="douyinMergeVideoService">视频合成</param>
/// <param name="douyinCollectCateService"></param>
protected DouyinBasicSyncJob(
DouyinCookieService douyinCookieService,
DouyinHttpClientService douyinHttpClientService,
DouyinVideoService douyinVideoService,
DouyinCommonService douyinCommonService,
DouyinFollowService douyinFollowService,
DouyinMergeVideoService douyinMergeVideoService)
DouyinMergeVideoService douyinMergeVideoService,
DouyinCollectCateService douyinCollectCateService)
{
this.douyinCookieService = douyinCookieService ?? throw new ArgumentNullException(nameof(douyinCookieService));
this.douyinHttpClientService = douyinHttpClientService ?? throw new ArgumentNullException(nameof(douyinHttpClientService));
@@ -114,6 +107,7 @@ namespace dy.net.job
this.douyinCommonService = douyinCommonService ?? throw new ArgumentNullException(nameof(douyinCommonService));
this.douyinFollowService = douyinFollowService;
this.douyinMergeVideoService = douyinMergeVideoService;
this.douyinCollectCateService = douyinCollectCateService;
}
#endregion
@@ -135,22 +129,21 @@ namespace dy.net.job
var config = douyinCommonService.GetConfig();
if (config == null)
{
Log.Debug($"{VideoType}-未获取到系统配置，任务终止!!!");
Log.Debug($"{VideoType}-系统配置错误，任务终止!!!");
return;
}
// 在处理Cookie之前执行的预处理
await BeforeProcessCookies();
// 获取所有有效的Cookie
var cookies = await GetValidCookies();
var cookies = await GetSyncCookies();
if (cookies == null || !cookies.Any())
{
Log.Debug($"{VideoType}-无有效cookie或{VideoType}已关闭同步，任务终止!!!");
Log.Debug($"{VideoType}-Cookie无效或{VideoType}已关闭同步，任务终止!!!");
return;
}
Log.Debug($"{VideoType}-共发现{cookies.Count}个有效的cookie，同步即将开始...");
Log.Debug($"{VideoType}:共发现{cookies.Count}个有效Cookie，同步开始...");
// 遍历每个有效的Cookie，执行同步
foreach (var cookie in cookies)
@@ -174,16 +167,9 @@ namespace dy.net.job
/// 子类必须实现此方法，根据具体任务类型筛选有效的Cookie
/// </summary>
/// <returns>有效的Cookie列表</returns>
protected abstract Task<List<DouyinCookie>> GetValidCookies();
/// <summary>
/// 检查指定的Cookie是否有效
/// </summary>
/// <param name="cookie">要检查的Cookie</param>
/// <returns>如果Cookie有效，则为true；否则为false</returns>
protected virtual bool IsCookieValid(DouyinCookie cookie)
protected virtual async Task<List<DouyinCookie>> GetSyncCookies()
{
return !string.IsNullOrWhiteSpace(cookie.Cookies);
return await douyinCookieService.GetOpendCookiesAsync(x => !string.IsNullOrWhiteSpace(x.SavePath));
}
/// <summary>
@@ -192,14 +178,13 @@ namespace dy.net.job
/// </summary>
/// <param name="cookie">用户Cookie</param>
/// <param name="cursor">分页游标，用于获取下一页数据</param>
/// <param name="uperUid">关注的人</param>
/// <param name="collectId">收藏夹ID</param>
/// <param name="followed">关注的人</param>
/// <param name="cate">自定义收藏夹、合集、短剧</param>
/// <returns>视频信息对象，包含视频列表和分页信息</returns>
protected abstract Task<DouyinVideoInfoResponse> FetchVideoData(DouyinCookie cookie, string cursor, DouyinFollowed uperUid , DouyinCollectItem collectId);
protected abstract Task<DouyinVideoInfoResponse> FetchVideoData(DouyinCookie cookie, string cursor, DouyinFollowed followed, DouyinCollectCate cate);
/// <summary>
/// 判断是否应该继续同步下一页数据
/// 子类必须实现此方法，根据API返回的分页信息判断是否还有更多数据
/// </summary>
/// <param name="cookie">用户Cookie</param>
/// <param name="data">当前获取到的视频数据</param>
@@ -209,23 +194,29 @@ namespace dy.net.job
/// <summary>
/// 获取下一页数据的游标
/// 子类必须实现此方法，从当前视频数据中提取下一页的游标
/// </summary>
/// <param name="data">当前获取到的视频数据</param>
/// <returns>下一页数据的游标</returns>
protected abstract string GetNextCursor(DouyinVideoInfoResponse data);
private static string GetNextCursor(DouyinVideoInfoResponse data)
{
return data?.Cursor ?? (data?.MaxCursor ?? "0");
}
/// <summary>
/// 创建视频保存文件夹
/// 子类必须实现此方法，根据具体的存储策略创建文件夹
/// </summary>
/// <param name="cookie">用户Cookie</param>
/// <param name="item">视频信息</param>
/// <param name="followed">关注用户</param>
/// <param name="collectItem">收藏夹Item</param>
/// <param name="cate"></param>
/// <param name="config">应用配置</param>
/// <returns>创建的视频保存文件夹路径</returns>
protected abstract string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed,DouyinCollectItem collectItem);
protected virtual string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed,DouyinCollectCate cate)
{
var folder = Path.Combine(cookie.SavePath, DouyinFileNameHelper.SanitizeLinuxFileName(cate.Name, item.AwemeId, true));
if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
return folder;
}
/// <summary>
/// 获取视频文件名,默认就用id作文件名
@@ -246,21 +237,29 @@ namespace dy.net.job
/// </summary>
/// <param name="cookie">用户Cookie</param>
/// <returns>作者头像保存的基础路径</returns>
protected abstract string GetAuthorAvatarBasePath(DouyinCookie cookie);
//protected abstract string GetAuthorAvatarBasePath(DouyinCookie cookie);
/// <summary>
/// 处理同步完成后的操作
/// 子类必须实现此方法，可用于更新同步状态、发送通知等
/// </summary>
/// <param name="cookie">用户Cookie</param>
/// <param name="syncCount">本次同步成功的视频数量</param>
/// <param name="followed"></param>
/// <returns>一个表示异步操作的任务</returns>
protected abstract Task HandleSyncCompletion(DouyinCookie cookie, int syncCount, DouyinFollowed followed = null);
protected virtual async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount, DouyinFollowed followed = null)
{
if (syncCount > 0)
{
Serilog.Log.Debug($"{VideoType}-Cookie-[{cookie.UserName}],本次共同步成功{syncCount}条视频");
}
else
{
Serilog.Log.Debug($"{VideoType}-Cookie-[{cookie.UserName}],没有可以同步的新视频");
}
}
/// <summary>
/// 获取视频实体的差异信息
/// 子类必须实现此方法，用于区分不同类型的视频（如普通视频、图片合成视频等）
/// </summary>
/// <param name="cookie">用户Cookie</param>
/// <param name="item">视频信息</param>
@@ -294,98 +293,64 @@ namespace dy.net.job
{
try
{
// 检查Cookie是否有效
if (!IsCookieValid(cookie))
{
Log.Debug($"{VideoType}-Cookie[{cookie.UserName}]无效，任务终止!!!");
return;
}
Log.Debug($"{VideoType}- Cookie-[{cookie.UserName}]开始同步...");
//up主上传视频特殊处理
if (VideoType == VideoTypeEnum.dy_follows)
switch (VideoType)
{
int syncCount = 0; // 本次同步成功的视频数量
string cursor = "0"; // 初始游标
bool hasMore = true; // 是否还有更多数据
//查询关注列表开启了同步的关注
var follows = await douyinFollowService.GetSyncFollows(cookie.MyUserId);
var firstUp = follows?.Where(x => !string.IsNullOrWhiteSpace(x.SecUid)).FirstOrDefault();
if (firstUp == null)
{
return;
}
if (string.IsNullOrWhiteSpace(firstUp.SecUid))
{
Log.Debug($"{VideoType}-Cookie[{firstUp.UperName}]无效,没有sec_userid，任务终止!!!");
return;
}
foreach (var followed in follows)
{
cursor = "0"; // 初始游标
await GetViedos(cookie, config, syncCount, cursor, hasMore, followed);
hasMore = true;
// 处理同步完成后的操作
await HandleSyncCompletion(cookie, syncCount, followed);
}
}
else
{
// 仅在视频类型为抖音收藏时执行对应逻辑
if (VideoType == VideoTypeEnum.dy_collects)
{
// 统一声明变量，避免分散定义，提升可读性
int syncCount = 0; // 本次同步成功的视频数量
string cursor = "0"; // 初始游标
bool hasMore = true; // 是否还有更多数据
// 处理「使用收藏文件夹」和「不使用收藏文件夹」的统一逻辑
if (cookie.UseCollectFolder)
case VideoTypeEnum.dy_follows:
{
// 获取收藏文件夹列表
var folders = await douyinHttpClientService.SyncCollectFolderList(cookie.Cookies,"");
Log.Debug("获取收藏文件夹列表完成，共计{Count}个包含视频的收藏夹", folders?.CollectsList?.Count(x => x.TotalNumber > 0) ?? 0);
// 文件夹为空时，执行默认同步逻辑（与不使用文件夹逻辑一致）
if (folders?.CollectsList == null || !folders.CollectsList.Any(x => x.TotalNumber > 0))
int syncCount = 0; // 本次同步成功的视频数量
//查询关注列表开启了同步的关注
var follows = await douyinFollowService.GetSyncFollows(cookie.MyUserId);
if (follows != null && follows.Any())
{
(syncCount, cursor, hasMore) = await GetViedos(cookie, config, syncCount, cursor, hasMore);
await HandleSyncCompletion(cookie, syncCount);
}
// 文件夹不为空时，遍历每个文件夹同步视频
else
{
foreach (var item in folders.CollectsList.Where(x=>x.TotalNumber>0))
foreach (var followed in follows)
{
// 重置游标，每个文件夹独立从头同步
string folderCursor = "0";
await GetViedos(cookie, config, syncCount, folderCursor, hasMore, null, item);
string cursor = "0";
bool hasMore = true;
(syncCount, cursor, hasMore) = await GetAndSaveViedos(cookie, config, syncCount, cursor, hasMore, followed);
await HandleSyncCompletion(cookie, syncCount, followed);
}
}
}
break;
case VideoTypeEnum.dy_favorite:
case VideoTypeEnum.dy_collects:
{
int syncCount = 0;
string cursor = "0";
bool hasMore = true;
(syncCount, cursor, hasMore) = await GetAndSaveViedos(cookie, config, syncCount, cursor, hasMore);
await HandleSyncCompletion(cookie, syncCount);
}
break;
case VideoTypeEnum.dy_mix:
case VideoTypeEnum.dy_series:
case VideoTypeEnum.dy_custom_collect:
{
int syncCount = 0; // 本次同步成功的视频数量
var cates = await douyinCollectCateService.GetSyncCates(cookie.Id, VideoType);
if (cates != null && cates.Any())
{
foreach (var cate in cates)
{
string cursor = "0";
bool hasMore = true;
(syncCount, cursor, hasMore) = await GetAndSaveViedos(cookie, config, syncCount, cursor, hasMore, null);
await HandleSyncCompletion(cookie, syncCount, null);
}
}
}
// 不使用收藏文件夹，直接执行默认同步逻辑
else
{
(syncCount, cursor, hasMore) = await GetViedos(cookie, config, syncCount, cursor, hasMore);
await HandleSyncCompletion(cookie, syncCount);
}
}
// 非抖音收藏类型，执行统一的默认同步逻辑
else
{
// 提取重复变量，与上方逻辑保持一致命名
int syncCount = 0;
string cursor = "0";
bool hasMore = true;
(syncCount, cursor, hasMore) = await GetViedos(cookie, config, syncCount, cursor, hasMore);
await HandleSyncCompletion(cookie, syncCount);
}
break;
case VideoTypeEnum.ImageVideo:
default:
break;
}
}
catch (Exception ex)
{
@@ -393,30 +358,25 @@ namespace dy.net.job
}
}
private async Task<(int syncCount, string cursor, bool hasMore)> GetViedos(DouyinCookie cookie, AppConfig config, int syncCount, string cursor, bool hasMore, DouyinFollowed followed = null,DouyinCollectItem collectItem=null)
private async Task<(int syncCount, string cursor, bool hasMore)> GetAndSaveViedos(DouyinCookie cookie, AppConfig config, int syncCount, string cursor, bool hasMore, DouyinFollowed followed = null, DouyinCollectCate cate = null)
{
// 循环获取视频数据
while (hasMore)
{
// 获取视频数据
var data = await FetchVideoData(cookie, cursor, followed,collectItem);
if (data == null)
var data = await FetchVideoData(cookie, cursor, followed, cate);
if (data == null|| data.AwemeList == null || !data.AwemeList.Any())
{
Log.Debug($"{VideoType}-Cookie[{cookie.UserName}]同步数据失败!!!");
break;
}
// 判断是否还有更多数据
hasMore = ShouldContinueSync(cookie, data, followed);
// 获取下一页游标
cursor = GetNextCursor(data);
// 如果没有视频数据，退出循环
if (data.AwemeList == null || !data.AwemeList.Any())
break;
// 处理视频列表
var videos = await ProcessVideoList(cookie, data, config, followed, collectItem);
var videos = await ProcessVideoList(cookie, data, config, followed, cate);
// 保存视频信息到数据库
syncCount += await SaveVideos(videos);
@@ -426,7 +386,7 @@ namespace dy.net.job
break;
}
//随机等待
await Task.Delay(_random.Next(5, 10) * 1000);
await Task.Delay(_random.Next(2, 10) * 1000);
}
return (syncCount, cursor, hasMore);
@@ -474,9 +434,9 @@ namespace dy.net.job
/// <param name="data">视频信息对象</param>
/// <param name="config">应用配置</param>
/// <param name="followed">关注的</param>
/// <param name="collectItem">收藏夹Item</param>
/// <param name="cate">收藏夹、合集、短剧</param>
/// <returns>处理后的视频实体列表</returns>
protected async Task<List<DouyinVideo>> ProcessVideoList(DouyinCookie cookie, DouyinVideoInfoResponse data, AppConfig config, DouyinFollowed followed = null,DouyinCollectItem collectItem=null)
protected async Task<List<DouyinVideo>> ProcessVideoList(DouyinCookie cookie, DouyinVideoInfoResponse data, AppConfig config, DouyinFollowed followed = null, DouyinCollectCate cate = null)
{
var videos = new List<DouyinVideo>();
foreach (var item in data.AwemeList)
@@ -485,48 +445,54 @@ namespace dy.net.job
//{
// continue;
//}
//if (!item.Desc.Contains("小房车正式启程"))
//{
// continue;
//}
//判断视频是否是强制删除且不再下载的视频
var deleteVideo = await douyinCommonService.ExistDeleteVideo(item.AwemeId);
if (deleteVideo)
{
//Log.Debug($"{VideoType}-视频-{item.AwemeId}-[{item.Desc}]已被标记为强制删除，跳过下载");
continue;
}
//判断是否存在视频，是否根据去重规则进行去重处理。。
// 1. 查询数据库中是否已存在该视频（通过 AwemeId 唯一标识）
var exitVideo = await douyinVideoService.GetByAwemeId(item.AwemeId);
bool Goon = await AutoDistinct(config, item, cookie, exitVideo);
if (!Goon)
if (cate == null)
{
continue;
}
//如果存在，但是因为去重规则，选择的最高优先级变更了，需要删除原来的，重新下载。
if (exitVideo != null)
{
await douyinVideoService.DeleteById(exitVideo.Id);
}
var uper = await douyinFollowService.GetByUperId(item.AuthorUserId.ToString(), cookie.MyUserId);
if (uper != null)
{
if (uper.FullSync)
//判断视频是否是强制删除且不再下载的视频
var deleteVideo = await douyinCommonService.ExistDeleteVideo(item.AwemeId);
if (deleteVideo)
{
followed ??= uper;
//Log.Debug($"{VideoType}-视频-{item.AwemeId}-[{item.Desc}]已被标记为强制删除，跳过下载");
continue;
}
bool Goon = await AutoDistinct(config, exitVideo);
if (!Goon)
{
continue;
}
//如果存在，但是因为去重规则，选择的最高优先级变更了，需要删除原来的，重新下载。
if (exitVideo != null)
{
await douyinVideoService.DeleteById(exitVideo.Id);
}
var uper = await douyinFollowService.GetByUperId(item.AuthorUserId.ToString(), cookie.MyUserId);
if (uper != null)
{
if (uper.FullSync)
{
followed ??= uper;
}
}
}
else
{
if (exitVideo != null && exitVideo.ViedoType == cate.CateType &&File.Exists(exitVideo.VideoSavePath))
{
continue;
}
}
// 处理单个视频
var video = await ProcessSingleVideo(cookie, item, data, config, followed,collectItem);
var video = await ProcessSingleVideo(cookie, item, data, config, followed);
if (video != null)
videos.Add(video);
else
{
//处理多个视频-组合的图文视频--类似动图。
List<DouyinDynamicVideoDto> dynamicVideoUrls = new List<DouyinDynamicVideoDto>();
// 当需要下载动态视频时，获取其他URL
@@ -557,7 +523,7 @@ namespace dy.net.job
}
// 处理核心逻辑
if (config.DownDynamicVideo && dynamicVideoUrls.Count > 0)
if (dynamicVideoUrls.Count > 0)
{
// 处理动态视频
var dynamicVideo = await ProcessDynamicVideo(dynamicVideoUrls, cookie, item, data, config);
@@ -569,15 +535,15 @@ namespace dy.net.job
Log.Debug($"{VideoType}-动态视频[{item.Desc}]，下载成功 ,共{dynamicVideos?.Count}个视频...");
if (dynamicVideos != null && dynamicVideos.Count > 0)
{
var savePath = DouyinFileNameHelper.RemoveNumberSuffix(dynamicVideo.VideoSavePath);
//音频文件下载地址
var mp3Url= item?.Music?.PlayUrl?.UrlList?.FirstOrDefault();
var outPath = await douyinMergeVideoService.MergeMultipleVideosAsync(dynamicVideos,mp3Url, savePath,cookie.Cookies);
if(!string.IsNullOrWhiteSpace(outPath.mp4Path))
var savePath = DouyinFileNameHelper.RemoveNumberSuffix(dynamicVideo.VideoSavePath);
//音频文件下载地址
var mp3Url = item?.Music?.PlayUrl?.UrlList?.FirstOrDefault();
var outPath = await douyinMergeVideoService.MergeMultipleVideosAsync(dynamicVideos, mp3Url, savePath, cookie.Cookies);
if (!string.IsNullOrWhiteSpace(outPath.mp4Path))
{
if (File.Exists(outPath.mp4Path))
{
@@ -608,10 +574,10 @@ namespace dy.net.job
}
else
{
// 统一处理图文视频逻辑
// 处理图文视频逻辑
if (config.DownImageVideo || config.DownMp3 || config.DownImage)
{
var mergevideo = await ProcessImageSetAndMergeToVideo(cookie, item, data, config, followed, collectItem);
var mergevideo = await ProcessImageSetAndMergeToVideo(cookie, item, config, followed,cate);
if (mergevideo != null)
videos.Add(mergevideo);
}
@@ -621,7 +587,7 @@ namespace dy.net.job
return videos;
}
private async Task<bool> AutoDistinct(AppConfig config, Aweme item, DouyinCookie cookie, DouyinVideo exitVideo)
private async Task<bool> AutoDistinct(AppConfig config, DouyinVideo exitVideo)
{
// 去重，检查视频是否已存在（按优先级下载）
if (config.AutoDistinct)
@@ -806,9 +772,9 @@ namespace dy.net.job
/// <param name="data">视频信息对象</param>
/// <param name="config">应用配置</param>
/// <param name="followed">关注</param>
/// <param name="collectItem">收藏夹Item</param>
/// <param name="cate"></param>
/// <returns>处理后的视频实体，如果处理失败则为null</returns>
protected async Task<DouyinVideo> ProcessSingleVideo(DouyinCookie cookie, Aweme item, DouyinVideoInfoResponse data, AppConfig config, DouyinFollowed followed = null,DouyinCollectItem collectItem=null)
protected async Task<DouyinVideo> ProcessSingleVideo(DouyinCookie cookie, Aweme item, DouyinVideoInfoResponse data, AppConfig config, DouyinFollowed followed = null,DouyinCollectCate cate=null)
{
// 检查视频数据是否有效
if (!IsAwemeValid(item)) return null;
@@ -824,7 +790,7 @@ namespace dy.net.job
// 获取视频标签
var (tag1, tag2, tag3) = GetVideoTags(item);
// 创建保存文件夹
var saveFolder = CreateSaveFolder(cookie, item, config, followed,collectItem);
var saveFolder = CreateSaveFolder(cookie, item, config, followed, cate);
// 获取视频文件名
var fileName = GetVideoFileName(cookie, item, config);
// 拼接视频保存路径
@@ -860,24 +826,20 @@ namespace dy.net.job
// 下载视频封面
await DownVideoCover(item, saveFolder, cookie, config);
// 下载作者头像
var (avatarSavePath, avatarUrl) = await DownAuthorAvatar(cookie, item);
//var (avatarSavePath, avatarUrl) = await DownAuthorAvatar(cookie, item);
// 创建视频实体
return await CreateVideoEntity(config, cookie, item, v, savePath, saveFolder, tag1, tag2, tag3, avatarSavePath, avatarUrl, data);
return await CreateVideoEntity(config, cookie, item, v, savePath, saveFolder, tag1, tag2, tag3);
}
protected async Task<DouyinVideo> ProcessDynamicVideo(List<DouyinDynamicVideoDto> dynamicUrls, DouyinCookie cookie, Aweme item, DouyinVideoInfoResponse data, AppConfig config, DouyinFollowed followed = null,DouyinCollectItem collectItem=null)
protected async Task<DouyinVideo> ProcessDynamicVideo(List<DouyinDynamicVideoDto> dynamicUrls, DouyinCookie cookie, Aweme item, DouyinVideoInfoResponse data, AppConfig config, DouyinFollowed followed = null,DouyinCollectCate cate=null)
{
// 获取视频标签
var (tag1, tag2, tag3) = GetVideoTags(item);
// 创建保存文件夹
var saveFolder = CreateSaveFolder(cookie, item, config, followed,collectItem);
var saveFolder = CreateSaveFolder(cookie, item, config, followed, cate);
// 获取视频文件名
//var fileName = DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId) + ".mp4";
@@ -894,7 +856,7 @@ namespace dy.net.job
{
// 下载动态视频
var dynamicSavePath = savePath.Replace(".mp4", $"_00{i}.mp4");
DouyinDynamicVideoDto v = new DouyinDynamicVideoDto() { Height = dynamicUrl.Height, Width = dynamicUrl.Width,Path= dynamicSavePath };
DouyinDynamicVideoDto v = new DouyinDynamicVideoDto() { Height = dynamicUrl.Height, Width = dynamicUrl.Width, Path = dynamicSavePath };
i++;
// 如果文件已存在，跳过
@@ -932,19 +894,13 @@ namespace dy.net.job
{
Width = item?.Images?.FirstOrDefault()?.Width ?? 0,
Height = item?.Images?.FirstOrDefault()?.Height ?? 0,
DataSize = DouyinFileUtils.GetTotalFileSize(dynamicSavePaths.Select(x=>x.Path).ToList()) // 合成视频的文件大小
DataSize = DouyinFileUtils.GetTotalFileSize(dynamicSavePaths.Select(x => x.Path).ToList()) // 合成视频的文件大小
}
};
return await CreateVideoEntity(config, cookie, item, virtualBitRate, dynamicSavePaths.FirstOrDefault()?.Path, saveFolder, tag1, tag2, tag3, "", "", data, dynamicSavePaths);
return await CreateVideoEntity(config, cookie, item, virtualBitRate, dynamicSavePaths.FirstOrDefault()?.Path, saveFolder, tag1, tag2, tag3, dynamicSavePaths);
}
/// <summary>
/// 寻找其他视频地址下载...
/// </summary>
@@ -1010,8 +966,9 @@ namespace dy.net.job
/// <param name="data">视频信息对象</param>
/// <param name="config">应用配置</param>
/// <param name="followed">应用配置</param>
/// <param name="cate"></param>
/// <returns>合成后的视频实体，如果处理失败则为null</returns>
protected async Task<DouyinVideo> ProcessImageSetAndMergeToVideo(DouyinCookie cookie, Aweme item, DouyinVideoInfoResponse data, AppConfig config, DouyinFollowed followed,DouyinCollectItem collectItem)
protected async Task<DouyinVideo> ProcessImageSetAndMergeToVideo(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed,DouyinCollectCate cate)
{
try
{
@@ -1043,7 +1000,7 @@ namespace dy.net.job
}
else
{
fileNamefolder = CreateSaveFolder(cookie, item, config, followed,collectItem);
fileNamefolder = CreateSaveFolder(cookie, item, config, followed, cate);
}
if (!Directory.Exists(fileNamefolder)) Directory.CreateDirectory(fileNamefolder);
@@ -1113,7 +1070,7 @@ namespace dy.net.job
// 下载视频封面（使用第一张图片作为封面）
await DownVideoCover(imageUrls.FirstOrDefault(), fileNamefolder, cookie, item, config);
// 下载作者头像
var (avatarSavePath, avatarUrl) = await DownAuthorAvatar(cookie, item);
//var (avatarSavePath, avatarUrl) = await DownAuthorAvatar(cookie, item);
// 获取视频标签
var (tag1, tag2, tag3) = GetVideoTags(item);
@@ -1131,7 +1088,7 @@ namespace dy.net.job
// 创建视频实体
var videoEntity = await CreateVideoEntity(config,
cookie, item, virtualBitRate, savePath, fileNamefolder,
tag1, tag2, tag3, avatarSavePath, avatarUrl, data);
tag1, tag2, tag3);
// 特殊处理合成视频的字段
videoEntity.FileHash = string.Empty; // 合成视频没有原始文件哈希
@@ -1171,8 +1128,6 @@ namespace dy.net.job
}
}
/// <summary>
/// 下载视频封面
/// 从视频信息中提取封面URL并下载到指定文件夹
@@ -1196,25 +1151,25 @@ namespace dy.net.job
/// <param name="cookie">用户Cookie</param>
/// <param name="item">视频信息</param>
/// <returns>一个元组，包含头像保存路径和头像URL</returns>
protected async Task<(string savePath, string url)> DownAuthorAvatar(DouyinCookie cookie, Aweme item)
{
if (item.Author == null) return (null, null);
// 优先获取高清头像
var avatarUrl = item.Author.AvatarLarger?.UrlList?.FirstOrDefault() ?? item.Author.AvatarThumb?.UrlList?.FirstOrDefault();
if (string.IsNullOrWhiteSpace(avatarUrl)) return (null, null);
//protected async Task<(string savePath, string url)> DownAuthorAvatar(DouyinCookie cookie, Aweme item)
//{
// if (item.Author == null) return (null, null);
// // 优先获取高清头像
// var avatarUrl = item.Author.AvatarLarger?.UrlList?.FirstOrDefault() ?? item.Author.AvatarThumb?.UrlList?.FirstOrDefault();
// if (string.IsNullOrWhiteSpace(avatarUrl)) return (null, null);
// 拼接头像保存路径
var avatarSavePath = Path.Combine(GetAuthorAvatarBasePath(cookie), $"{item.Author.Uid}.jpg");
var avatarDir = Path.GetDirectoryName(avatarSavePath);
// 创建头像保存文件夹
if (!Directory.Exists(avatarDir)) Directory.CreateDirectory(avatarDir);
// 如果头像文件不存在，则下载
if (!File.Exists(avatarSavePath))
{
await douyinHttpClientService.DownloadAsync(avatarUrl, avatarSavePath, cookie.Cookies);
}
return (avatarSavePath, avatarUrl);
}
// // 拼接头像保存路径
// var avatarSavePath = Path.Combine(GetAuthorAvatarBasePath(cookie), $"{item.Author.Uid}.jpg");
// var avatarDir = Path.GetDirectoryName(avatarSavePath);
// // 创建头像保存文件夹
// if (!Directory.Exists(avatarDir)) Directory.CreateDirectory(avatarDir);
// // 如果头像文件不存在，则下载
// if (!File.Exists(avatarSavePath))
// {
// await douyinHttpClientService.DownloadAsync(avatarUrl, avatarSavePath, cookie.Cookies);
// }
// return (avatarSavePath, avatarUrl);
//}
#endregion
@@ -1226,7 +1181,7 @@ namespace dy.net.job
/// </summary>
/// <param name="item">视频信息</param>
/// <returns>如果视频数据有效，则为true；否则为false</returns>
private bool IsAwemeValid(Aweme item) => item != null && item.Video != null && item.Video.BitRate != null;
private static bool IsAwemeValid(Aweme item) => item != null && item.Video != null && item.Video.BitRate != null;
/// <summary>
/// 获取视频标签
@@ -1326,17 +1281,17 @@ namespace dy.net.job
/// <returns>创建的视频实体对象</returns>
private async Task<DouyinVideo> CreateVideoEntity(AppConfig config,
DouyinCookie cookie, Aweme item, VideoBitRate bitRate, string savePath, string saveFolder,
string tag1, string tag2, string tag3, string avatarSavePath, string avatarUrl, DouyinVideoInfoResponse data, List<DouyinDynamicVideoDto> dynamicVideos = null)
string tag1, string tag2, string tag3, List<DouyinDynamicVideoDto> dynamicVideos = null)
{
var diffs = GetVideoEntityDifferences(cookie, item);
var avatarUrl = item.Author.AvatarLarger?.UrlList?.FirstOrDefault() ?? item.Author.AvatarThumb?.UrlList?.FirstOrDefault();
var video = new DouyinVideo
{
ViedoType = VideoType,
AwemeId = item.AwemeId,
Author = item.Author?.Nickname,
AuthorId = item.Author?.Uid,
AuthorAvatar = avatarSavePath,
AuthorAvatar = "",
AuthorAvatarUrl = avatarUrl,
CreateTime = DateTimeUtil.Convert10BitTimestamp(item.CreateTime),
VideoTitle = string.IsNullOrWhiteSpace(item.Desc) ? $"{item.Author?.Nickname}-{item.CreateTime}" : item.Desc,
@@ -1377,11 +1332,6 @@ namespace dy.net.job
/// </summary>
public class VideoEntityDifferences
{
/// <summary>
/// 视频类型
/// </summary>
//public VideoTypeEnum VideoType { get; set; }
/// <summary>
/// 简化的视频标题
/// </summary>
job/DouyinCollectCatesSyncJob.cs
-124
View File
@@ -1,124 +0,0 @@
using dy.net.model.entity;
using dy.net.model.response;
using dy.net.service;
using Quartz;
using System.Collections.Generic;
namespace dy.net.job
{
[DisallowConcurrentExecution] // 禁止并发执行，确保同一时间只有一个实例在运行
public class DouyinCollectCatesSyncJob : IJob
{
/// <summary>
/// 抖音Cookie服务，用于获取和管理用户Cookie
/// </summary>
protected readonly DouyinCookieService _dyCookieService;
/// <summary>
/// 抖音HTTP客户端服务，用于发送HTTP请求
/// </summary>
protected readonly DouyinHttpClientService _douyinService;
/// <summary>
/// 抖音关注服务，用于管理关注数据
/// </summary>
protected readonly DouyinFollowService _followService;
protected readonly DouyinCommonService douyinCommonService;
public DouyinCollectCatesSyncJob(DouyinCookieService dyCookieService, DouyinHttpClientService douyinService, DouyinFollowService followService, DouyinCommonService douyinCommonService)
{
_dyCookieService = dyCookieService;
_douyinService = douyinService;
_followService = followService;
this.douyinCommonService = douyinCommonService;
}
public async Task Execute(IJobExecutionContext context)
{
var cookies = await _dyCookieService.GetAllOpendAsync();
if (cookies != null && cookies.Any())
{
AppConfig conf = douyinCommonService.GetConfig();
foreach (var ck in cookies)
{
if (string.IsNullOrWhiteSpace(ck.SecUserId))
{
Serilog.Log.Debug($"同步关注列表博主[{ck.UserName}]-未设置secuserid,跳过.");
continue;
}
string count = "20";
string offset = "0";
bool hasmore = true;
int total= 0;
List<FollowingsItem> follows = new List<FollowingsItem>();
while (hasmore)
{
try
{
var data = await _douyinService.SyncMyFollows(count, offset, ck.SecUserId, ck.Cookies, async err =>
{
if (err.StatusCode != 0)
{
Serilog.Log.Error($"同步关注列表博主[{ck.UserName}]关注列表时发生错误，错误信息：{err.StatusMsg}，状态码：{err.StatusCode}");
}
// 如果是未登录错误，则跳出循环
if (err.StatusCode == 8)
{
hasmore = false;
}
ck.StatusMsg = err.StatusCode == 8 ? "无效" : "正常";
ck.StatusCode = err.StatusCode;
await _dyCookieService.UpdateAsync(ck);
});
if (data != null)
{
// 绑定我的用户ID--之前没有这个字段
if (string.IsNullOrWhiteSpace(ck.MyUserId))
{
ck.MyUserId = data.MySelfUserId;
await _dyCookieService.UpdateAsync(ck);
}
total = data.Total;
hasmore = data.HasMore;
offset = data.Offset.ToString();
if (data.Followings != null && data.Followings.Count > 0)
{
follows.AddRange(data.Followings);
}
}
// 只有第一次启动时，才进行全部同步
if (!conf.IsFirstRunning)
{
hasmore = false;
}
}
catch (Exception ex)
{
Serilog.Log.Error($"关注列表同步失败,{ex.Message}");
hasmore = false;
break;
}
}
if (follows.Count > 0)
{
var (add, update, succ) = await _followService.Sync(follows, ck);
if (!succ) hasmore = false;
else
{
hasmore = add > 100|| update > 100;//一次最多100条数据
}
}
}
await douyinCommonService.SetConfigNotFirstRunning();
}
}
}
}
job/DouyinCollectCustomSyncJob.cs
+29
View File
@@ -0,0 +1,29 @@
using dy.net.model.dto;
using dy.net.model.entity;
using dy.net.model.response;
using dy.net.service;
using dy.net.utils;
using System;
namespace dy.net.job
{
public class DouyinCollectCustomSyncJob : DouyinBasicSyncJob
{
public DouyinCollectCustomSyncJob(DouyinCookieService douyinCookieService, DouyinHttpClientService douyinHttpClientService, DouyinVideoService douyinVideoService, DouyinCommonService douyinCommonService, DouyinFollowService douyinFollowService, DouyinMergeVideoService douyinMergeVideoService, DouyinCollectCateService douyinCollectCateService) : base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService, douyinCollectCateService)
{
}
protected override VideoTypeEnum VideoType => VideoTypeEnum.dy_custom_collect;
protected override async Task<DouyinVideoInfoResponse> FetchVideoData(DouyinCookie cookie, string cursor, DouyinFollowed followed, DouyinCollectCate cate)
{
return await douyinHttpClientService.SyncCollectVideos(cursor, count, cookie.Cookies);
}
protected override bool ShouldContinueSync(DouyinCookie cookie, DouyinVideoInfoResponse data, DouyinFollowed followed = null)
{
return data != null && data.HasMore == 1 && cookie.CollHasSyncd == 0;
}
}
}
job/DouyinFollowedUsersSyncJob.cs
-124
View File
@@ -1,124 +0,0 @@
using dy.net.model.entity;
using dy.net.model.response;
using dy.net.service;
using Quartz;
using System.Collections.Generic;
namespace dy.net.job
{
[DisallowConcurrentExecution] // 禁止并发执行，确保同一时间只有一个实例在运行
public class DouyinFollowedUsersSyncJob : IJob
{
/// <summary>
/// 抖音Cookie服务，用于获取和管理用户Cookie
/// </summary>
protected readonly DouyinCookieService _dyCookieService;
/// <summary>
/// 抖音HTTP客户端服务，用于发送HTTP请求
/// </summary>
protected readonly DouyinHttpClientService _douyinService;
/// <summary>
/// 抖音关注服务，用于管理关注数据
/// </summary>
protected readonly DouyinFollowService _followService;
protected readonly DouyinCommonService douyinCommonService;
public DouyinFollowedUsersSyncJob(DouyinCookieService dyCookieService, DouyinHttpClientService douyinService, DouyinFollowService followService, DouyinCommonService douyinCommonService)
{
_dyCookieService = dyCookieService;
_douyinService = douyinService;
_followService = followService;
this.douyinCommonService = douyinCommonService;
}
public async Task Execute(IJobExecutionContext context)
{
var cookies = await _dyCookieService.GetAllOpendAsync();
if (cookies != null && cookies.Any())
{
AppConfig conf = douyinCommonService.GetConfig();
foreach (var ck in cookies)
{
if (string.IsNullOrWhiteSpace(ck.SecUserId))
{
Serilog.Log.Debug($"同步关注列表博主[{ck.UserName}]-未设置secuserid,跳过.");
continue;
}
string count = "20";
string offset = "0";
bool hasmore = true;
int total= 0;
List<FollowingsItem> follows = new List<FollowingsItem>();
while (hasmore)
{
try
{
var data = await _douyinService.SyncMyFollows(count, offset, ck.SecUserId, ck.Cookies, async err =>
{
if (err.StatusCode != 0)
{
Serilog.Log.Error($"同步关注列表博主[{ck.UserName}]关注列表时发生错误，错误信息：{err.StatusMsg}，状态码：{err.StatusCode}");
}
// 如果是未登录错误，则跳出循环
if (err.StatusCode == 8)
{
hasmore = false;
}
ck.StatusMsg = err.StatusCode == 8 ? "无效" : "正常";
ck.StatusCode = err.StatusCode;
await _dyCookieService.UpdateAsync(ck);
});
if (data != null)
{
// 绑定我的用户ID--之前没有这个字段
if (string.IsNullOrWhiteSpace(ck.MyUserId))
{
ck.MyUserId = data.MySelfUserId;
await _dyCookieService.UpdateAsync(ck);
}
total = data.Total;
hasmore = data.HasMore;
offset = data.Offset.ToString();
if (data.Followings != null && data.Followings.Count > 0)
{
follows.AddRange(data.Followings);
}
}
// 只有第一次启动时，才进行全部同步
if (!conf.IsFirstRunning)
{
hasmore = false;
}
}
catch (Exception ex)
{
Serilog.Log.Error($"关注列表同步失败,{ex.Message}");
hasmore = false;
break;
}
}
if (follows.Count > 0)
{
var (add, update, succ) = await _followService.Sync(follows, ck);
if (!succ) hasmore = false;
else
{
hasmore = add > 100|| update > 100;//一次最多100条数据
}
}
}
await douyinCommonService.SetConfigNotFirstRunning();
}
}
}
}
job/DouyinFollowedViedoSyncJob.cs
+5 -20
View File
@@ -13,24 +13,18 @@ namespace dy.net.job
{
public class DouyinFollowedViedoSyncJob : DouyinBasicSyncJob
{
public DouyinFollowedViedoSyncJob(DouyinCookieService douyinCookieService, DouyinHttpClientService douyinHttpClientService, DouyinVideoService douyinVideoService, DouyinCommonService douyinCommonService, DouyinFollowService douyinFollowService, DouyinMergeVideoService douyinMergeVideoService) : base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService)
public DouyinFollowedViedoSyncJob(DouyinCookieService douyinCookieService, DouyinHttpClientService douyinHttpClientService, DouyinVideoService douyinVideoService, DouyinCommonService douyinCommonService, DouyinFollowService douyinFollowService, DouyinMergeVideoService douyinMergeVideoService, DouyinCollectCateService douyinCollectCateService) : base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService, douyinCollectCateService)
{
}
protected override VideoTypeEnum VideoType => VideoTypeEnum.dy_follows;
protected override async Task<List<DouyinCookie>> GetValidCookies()
protected override async Task<List<DouyinCookie>> GetSyncCookies()
{
return await douyinCookieService.GetAllOpendAsync(x => !string.IsNullOrWhiteSpace(x.UpSavePath));
return await douyinCookieService.GetOpendCookiesAsync(x => !string.IsNullOrWhiteSpace(x.UpSavePath));
}
protected override bool IsCookieValid(DouyinCookie cookie)
{
return !string.IsNullOrWhiteSpace(cookie.Cookies)&&!string.IsNullOrWhiteSpace(cookie.UpSavePath);
}
protected override async Task<DouyinVideoInfoResponse> FetchVideoData(DouyinCookie cookie, string cursor, DouyinFollowed followed, DouyinCollectItem collectId)
protected override async Task<DouyinVideoInfoResponse> FetchVideoData(DouyinCookie cookie, string cursor, DouyinFollowed followed,DouyinCollectCate cate)
{
return await douyinHttpClientService.SyncUpderPostVideos(count, cursor, followed.SecUid, cookie.Cookies);
}
@@ -40,10 +34,6 @@ namespace dy.net.job
return data != null && data.HasMore == 1 && cookie.UperSyncd == 0 && followed.FullSync;
}
protected override string GetNextCursor(DouyinVideoInfoResponse data)
{
return data?.MaxCursor ?? "0";
}
/// <summary>
/// 关注用户特殊处理文件夹存储路径，用户可自定义保存路径
@@ -53,7 +43,7 @@ namespace dy.net.job
/// <param name="followed"></param>
/// <param name="config"></param>
/// <returns></returns>
protected override string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed, DouyinCollectItem collectItem)
protected override string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed,DouyinCollectCate cate)
{
#region 默认使用UP主名称作为文件夹名称，若关注列表中有自定义保存路径则使用自定义路径
// 1. 优先获取有效的作者名称（遵循原有优先级：followed.UperName > item.Author.Nickname > 默认值）
@@ -170,11 +160,6 @@ namespace dy.net.job
}
}
protected override string GetAuthorAvatarBasePath(DouyinCookie cookie)
{
return Path.Combine(cookie.UpSavePath, "author");
}
protected override async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount, DouyinFollowed followed)
{
if (syncCount > 0)
job/DouyinFollowsAndCollnectsSyncJob.cs
+347
View File
@@ -0,0 +1,347 @@
using dy.net.model.dto;
using dy.net.model.entity;
using dy.net.model.response;
using dy.net.service;
using Quartz;
namespace dy.net.job
{
[DisallowConcurrentExecution] // 禁止并发执行，确保同一时间只有一个实例在运行
public class DouyinFollowsAndCollnectsSyncJob : IJob
{
/// <summary>
/// 抖音收藏夹服务
/// </summary>
protected readonly DouyinCollectCateService douyinCollectCateService;
/// <summary>
/// 抖音Cookie服务，用于获取和管理用户Cookie
/// </summary>
protected readonly DouyinCookieService _dyCookieService;
/// <summary>
/// 抖音HTTP客户端服务，用于发送HTTP请求
/// </summary>
protected readonly DouyinHttpClientService _douyinService;
/// <summary>
/// 抖音关注服务，用于管理关注数据
/// </summary>
protected readonly DouyinFollowService _followService;
protected readonly DouyinCommonService douyinCommonService;
public DouyinFollowsAndCollnectsSyncJob(DouyinCookieService dyCookieService, DouyinHttpClientService douyinService, DouyinFollowService followService, DouyinCommonService douyinCommonService, DouyinCollectCateService douyinCollectCateService)
{
_dyCookieService = dyCookieService;
_douyinService = douyinService;
_followService = followService;
this.douyinCommonService = douyinCommonService;
this.douyinCollectCateService = douyinCollectCateService;
}
// 第一步：定义常量和枚举（建议放在单独的常量类中，此处为内联演示）
private const string DEFAULT_FOLLOW_COUNT = "20";
private const int INVALID_COOKIE_STATUS_CODE = 8;
private const string LOG_TAG_COLLECT = "收藏夹同步";
private const string LOG_TAG_MIX = "合集列表同步";
private const string LOG_TAG_SERIES = "短剧列表同步";
private const string LOG_TAG_FOLLOW = "关注列表同步";
/// <summary>
/// 抖音数据同步任务执行方法（优化版）
/// </summary>
/// <param name="context">任务执行上下文</param>
public async Task Execute(IJobExecutionContext context)
{
// 1. 获取有效Cookie列表，提前做空值判断
var cookies = await _dyCookieService.GetOpendCookiesAsync();
if (cookies == null || !cookies.Any())
{
Serilog.Log.Debug("当前无可用的抖音Cookie，同步任务跳过");
return;
}
// 2. 获取应用配置
AppConfig conf = douyinCommonService.GetConfig();
// 3. 遍历每个Cookie，执行各类数据同步
foreach (var ck in cookies)
{
Serilog.Log.Information($"开始处理Cookie用户：[{ck.UserName}]（ID：{ck.Id}）");
// 3.1 同步自定义收藏夹（UseCollectFolder=true时）
if (ck.UseCollectFolder)
{
await SyncCollectDataAsync(
cookie: ck,
cateType: VideoTypeEnum.dy_custom_collect,
dataFetchFunc: async (offset) => await _douyinService.SyncCollectFolderList(ck.Cookies, offset),
entityConvertFunc: (collectItem) => new DouyinCollectCate
{
CookieId = ck.Id,
CateType = VideoTypeEnum.dy_custom_collect,
Name = collectItem.CollectsName,
Sync = false,
XId = collectItem.CollectsId
},
hasMoreFunc: (data) => data?.HasMore ?? false,
cursorFunc: (data) => data?.Cursor.ToString() ?? "0",
dataListFunc: (data) => data?.CollectsList,
logTag: LOG_TAG_COLLECT
);
}
// 3.2 同步收藏夹合集（DownMix=true时）
if (ck.DownMix)
{
await SyncCollectDataAsync(
cookie: ck,
cateType: VideoTypeEnum.dy_mix,
dataFetchFunc: async (offset) => await _douyinService.SyncMixList(ck.Cookies, offset),
entityConvertFunc: (mixItem) => new DouyinCollectCate
{
CookieId = ck.Id,
CateType = VideoTypeEnum.dy_mix,
Name = mixItem.MixName,
Sync = false,
XId = mixItem.MixId,
CoverUrl = mixItem.CoverUrl?.UrlList?.FirstOrDefault() // 空值防护：避免CoverUrl为null
},
hasMoreFunc: (data) => (data?.HasMore ?? 0) == 1,
cursorFunc: (data) => data?.Cursor.ToString() ?? "0",
dataListFunc: (data) => data?.MixInfos,
logTag: LOG_TAG_MIX
);
}
// 3.3 同步收藏夹短剧（DownSeries=true时）
if (ck.DownSeries)
{
await SyncCollectDataAsync(
cookie: ck,
cateType: VideoTypeEnum.dy_series,
dataFetchFunc: async (offset) => await _douyinService.SyncSeriesList(ck.Cookies, offset),
entityConvertFunc: (seriesItem) => new DouyinCollectCate
{
CookieId = ck.Id,
CateType = VideoTypeEnum.dy_series,
Name = seriesItem.SeriesName,
Sync = false,
XId = seriesItem.SeriesId,
CoverUrl = seriesItem.CoverImage?.ImageUrlList?.FirstOrDefault() // 空值防护：避免CoverImage为null
},
hasMoreFunc: (data) => (data?.HasMore ?? 0) == 1,
cursorFunc: (data) => data?.Cursor.ToString() ?? "0",
dataListFunc: (data) => data?.SeriesList,
logTag: LOG_TAG_SERIES
);
}
// 3.4 同步关注列表（单独处理，逻辑特殊不纳入通用方法）
await SyncFollowListAsync(ck, conf);
Serilog.Log.Debug($"完成Cookie用户：[{ck.UserName}]（ID：{ck.Id}）的所有收藏夹信息、合集信息、关注列表信息、短剧信息同步");
}
// 4. 标记首次运行完成（仅执行一次）
await douyinCommonService.SetConfigNotFirstRunning();
}
#region 私有通用辅助方法
/// <summary>
/// 通用收藏类数据同步方法（泛型封装，消除冗余）
/// </summary>
/// <typeparam name="TData">抖音返回的分页数据类型</typeparam>
/// <typeparam name="TItem">抖音返回的列表项类型</typeparam>
/// <param name="cookie">当前Cookie信息</param>
/// <param name="cateType">分类类型</param>
/// <param name="dataFetchFunc">分页数据获取委托</param>
/// <param name="entityConvertFunc">抖音项转换为DouyinCollectCate的委托</param>
/// <param name="hasMoreFunc">判断是否还有更多数据的委托</param>
/// <param name="cursorFunc">获取下一页游标值的委托</param>
/// <param name="dataListFunc">从分页数据中提取列表的委托</param>
/// <param name="logTag">日志标签</param>
private async Task SyncCollectDataAsync<TData, TItem>(
DouyinCookie cookie,
VideoTypeEnum cateType,
Func<string, Task<TData>> dataFetchFunc,
Func<TItem, DouyinCollectCate> entityConvertFunc,
Func<TData, bool> hasMoreFunc,
Func<TData, string> cursorFunc,
Func<TData, List<TItem>> dataListFunc,
string logTag)
{
string offset = "0";
bool hasMore = true;
List<DouyinCollectCate> collectDataList = new List<DouyinCollectCate>();
try
{
// 1. 分页获取抖音数据
while (hasMore)
{
var pageData = await dataFetchFunc(offset);
if (pageData == null)
{
//Serilog.Log.Warning($"[{cookie.UserName}] - {logTag}：获取分页数据为空，停止分页");
break;
}
// 2. 更新分页状态
hasMore = hasMoreFunc(pageData);
offset = cursorFunc(pageData);
// 3. 提取当前页列表数据并转换
var currentPageItems = dataListFunc(pageData);
if (currentPageItems != null && currentPageItems.Any())
{
var convertItems = currentPageItems.Select(entityConvertFunc).ToList();
collectDataList.AddRange(convertItems);
Serilog.Log.Debug($"[{cookie.UserName}] - {logTag}：获取到{currentPageItems.Count}条数据，累计{collectDataList.Count}条");
}
}
// 4. 调用Sync方法同步到数据库
if (collectDataList.Any())
{
var (add, update, delete, succ) = await douyinCollectCateService.Sync(
collectDataList,
cookie.Id,
cateType);
Serilog.Log.Debug($"[{cookie.UserName}] - {logTag}：同步完成，新增{add}条，更新{update}条，删除{delete}条，是否成功：{succ}");
}
else
{
Serilog.Log.Debug($"[{cookie.UserName}] - {logTag}：无有效数据需要同步");
}
}
catch (Exception ex)
{
Serilog.Log.Error(ex, $"[{cookie.UserName}] - {logTag}：分页同步失败，异常信息：{ex.Message}");
}
}
/// <summary>
/// 关注列表单独同步方法（逻辑特殊，单独封装）
/// </summary>
/// <param name="cookie">当前Cookie信息</param>
/// <param name="config">应用配置</param>
private async Task SyncFollowListAsync(DouyinCookie cookie, AppConfig config)
{
// 1. 校验SecUserId是否有效
if (string.IsNullOrWhiteSpace(cookie.SecUserId))
{
Serilog.Log.Debug($"[{cookie.UserName}] - {LOG_TAG_FOLLOW}：未设置SecUserId，跳过");
return;
}
string offset = "0";
bool hasMore = true;
int total = 0;
List<FollowingsItem> followList = new List<FollowingsItem>();
try
{
// 2. 分页获取关注列表数据
while (hasMore)
{
var (data, error) = await FetchFollowPageDataAsync(cookie, offset);
// 3. 处理错误信息
if (error != null && error.StatusCode != 0)
{
Serilog.Log.Error($"[{cookie.UserName}] - {LOG_TAG_FOLLOW}：发生错误，状态码：{error.StatusCode}，错误信息：{error.StatusMsg}");
// 无效Cookie（未登录）直接停止分页
if (error.StatusCode == INVALID_COOKIE_STATUS_CODE)
{
cookie.StatusMsg = "无效";
cookie.StatusCode = INVALID_COOKIE_STATUS_CODE;
await _dyCookieService.UpdateAsync(cookie);
break;
}
}
// 4. 处理有效数据
if (data != null)
{
// 5. 更新MyUserId（若未设置）
if (string.IsNullOrWhiteSpace(cookie.MyUserId))
{
cookie.MyUserId = data.MySelfUserId;
await _dyCookieService.UpdateAsync(cookie);
}
// 6. 更新分页状态和数据
total = data.Total;
hasMore = data.HasMore;
offset = data.Offset.ToString();
if (data.Followings != null && data.Followings.Any())
{
followList.AddRange(data.Followings);
Serilog.Log.Debug($"[{cookie.UserName}] - {LOG_TAG_FOLLOW}：获取到{data.Followings.Count}条关注数据，累计{followList.Count}条");
}
// 7. 非首次运行时，仅同步第一页数据
if (!config.IsFirstRunning)
{
hasMore = false;
//Serilog.Log.Debug($"[{cookie.UserName}] - {LOG_TAG_FOLLOW}：非首次运行，停止分页获取");
}
}
else
{
Serilog.Log.Warning($"[{cookie.UserName}] - {LOG_TAG_FOLLOW}：获取分页数据为空，停止分页");
break;
}
}
// 8. 同步关注列表到数据库
if (followList.Any())
{
var (add, update, succ) = await _followService.Sync(followList, cookie);
Serilog.Log.Information($"[{cookie.UserName}] - {LOG_TAG_FOLLOW}：同步完成，新增{add}条，更新{update}条，是否成功：{succ}，总关注数：{total}");
}
else
{
Serilog.Log.Debug($"[{cookie.UserName}] - {LOG_TAG_FOLLOW}：无有效关注数据需要同步");
}
}
catch (Exception ex)
{
Serilog.Log.Error(ex, $"[{cookie.UserName}] - {LOG_TAG_FOLLOW}：同步失败，异常信息：{ex.Message}");
}
}
/// <summary>
/// 分页获取关注列表数据（封装回调逻辑，简化代码）
/// </summary>
/// <param name="cookie">当前Cookie信息</param>
/// <param name="offset">分页偏移量</param>
private async Task<(DouyinFollowInfoResponse Data, FollowErrorDto Error)> FetchFollowPageDataAsync(DouyinCookie cookie, string offset)
{
DouyinFollowInfoResponse resultData = null;
FollowErrorDto resultError = null;
resultData = await _douyinService.SyncMyFollows(
DEFAULT_FOLLOW_COUNT,
offset,
cookie.SecUserId,
cookie.Cookies,
async (err) =>
{
resultError = err;
// 更新Cookie状态（异步不阻塞）
cookie.StatusMsg = err.StatusCode == INVALID_COOKIE_STATUS_CODE ? "无效" : "正常";
cookie.StatusCode = err.StatusCode;
await _dyCookieService.UpdateAsync(cookie);
});
// 若原方法是回调式无返回值，需调整封装逻辑
return (resultData, resultError);
}
#endregion
}
}
job/DouyinMixSyncJob.cs
+31
View File
@@ -0,0 +1,31 @@
using dy.net.model.dto;
using dy.net.model.entity;
using dy.net.model.response;
using dy.net.service;
using dy.net.utils;
namespace dy.net.job
{
public class DouyinMixSyncJob : DouyinBasicSyncJob
{
public DouyinMixSyncJob(DouyinCookieService douyinCookieService, DouyinHttpClientService douyinHttpClientService, DouyinVideoService douyinVideoService, DouyinCommonService douyinCommonService, DouyinFollowService douyinFollowService, DouyinMergeVideoService douyinMergeVideoService, DouyinCollectCateService douyinCollectCateService) : base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService, douyinCollectCateService)
{
}
protected override VideoTypeEnum VideoType => VideoTypeEnum.dy_favorite;
protected override async Task<DouyinVideoInfoResponse> FetchVideoData(DouyinCookie cookie, string cursor,DouyinFollowed followed,DouyinCollectCate cate)
{
return await douyinHttpClientService.SyncFavoriteVideos(count, cursor, cookie.SecUserId, cookie.Cookies);
}
protected override bool ShouldContinueSync(DouyinCookie cookie, DouyinVideoInfoResponse data, DouyinFollowed followed=null)
{
return data != null && data.HasMore == 1 && cookie.FavHasSyncd == 0;
}
}
}
job/DouyinSeriesSyncJob.cs
+29
View File
@@ -0,0 +1,29 @@
using dy.net.model.dto;
using dy.net.model.entity;
using dy.net.model.response;
using dy.net.service;
using dy.net.utils;
using System.Net;
namespace dy.net.job
{
public class DouyinSeriesSyncJob : DouyinBasicSyncJob
{
public DouyinSeriesSyncJob(DouyinCookieService douyinCookieService, DouyinHttpClientService douyinHttpClientService, DouyinVideoService douyinVideoService, DouyinCommonService douyinCommonService, DouyinFollowService douyinFollowService, DouyinMergeVideoService douyinMergeVideoService, DouyinCollectCateService douyinCollectCateService) : base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService, douyinCollectCateService)
{
}
protected override VideoTypeEnum VideoType => VideoTypeEnum.dy_series;
protected override async Task<DouyinVideoInfoResponse> FetchVideoData(DouyinCookie cookie, string cursor,DouyinFollowed followed, DouyinCollectCate cate)
{
return await douyinHttpClientService.SyncSeriesViedosByMSeriesId(cursor, count, cookie.Cookies,cate.XId);
}
protected override bool ShouldContinueSync(DouyinCookie cookie, DouyinVideoInfoResponse data, DouyinFollowed followed=null)
{
return data != null && data.HasMore == 1;
}
}
}