短剧、合集

This commit is contained in:
jianzhichu
2026-01-20 00:20:37 +08:00
parent 7c6ee0a8a6
commit 7175b3858a
25 changed files with 779 additions and 745 deletions
-13
View File
@@ -46,19 +46,6 @@ namespace dy.net.Controllers
pageSize = dto.PageSize pageSize = dto.PageSize
}); });
} }
/// <summary>
/// 重新同步-单次
/// </summary>
/// <returns></returns>
[HttpGet("sync")]
public async Task<IActionResult> SyncFollowList()
{
//后台异步
_douyinQuartzJobService.StartFollowJobOnceAsync();
await Task.Delay(1000);
return ApiResult.Success();
}
/// <summary> /// <summary>
/// 修改关注同步状态 /// 修改关注同步状态
+1
View File
@@ -52,6 +52,7 @@
"autoprefixer": "^10.4.7", "autoprefixer": "^10.4.7",
"babel-preset-es2015": "^6.24.1", "babel-preset-es2015": "^6.24.1",
"babel-register": "^6.26.0", "babel-register": "^6.26.0",
"baseline-browser-mapping": "^2.9.15",
"body-parser": "^1.20.1", "body-parser": "^1.20.1",
"crypto-js": "^4.2.0", "crypto-js": "^4.2.0",
"gh-pages": "^3.1.0", "gh-pages": "^3.1.0",
+11 -58
View File
@@ -9,14 +9,9 @@ namespace dy.net.job
{ {
public class DouyinCollectSyncJob : DouyinBasicSyncJob public class DouyinCollectSyncJob : DouyinBasicSyncJob
{ {
public DouyinCollectSyncJob( public DouyinCollectSyncJob(DouyinCookieService douyinCookieService, DouyinHttpClientService douyinHttpClientService, DouyinVideoService douyinVideoService, DouyinCommonService douyinCommonService, DouyinFollowService douyinFollowService, DouyinMergeVideoService douyinMergeVideoService, DouyinCollectCateService douyinCollectCateService) : base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService, douyinCollectCateService)
DouyinCookieService douyinCookieService, {
DouyinHttpClientService douyinHttpClientService, }
DouyinVideoService douyinVideoService,
DouyinCommonService douyinCommonService,DouyinFollowService douyinFollowService,DouyinMergeVideoService douyinMergeVideoService)
: base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService,douyinFollowService, douyinMergeVideoService) { }
protected override VideoTypeEnum VideoType => VideoTypeEnum.dy_collects; protected override VideoTypeEnum VideoType => VideoTypeEnum.dy_collects;
protected override async Task BeforeProcessCookies() protected override async Task BeforeProcessCookies()
@@ -24,30 +19,14 @@ namespace dy.net.job
var now = DateTime.Now; var now = DateTime.Now;
if (now.Hour == 1 && now.Minute < 30) if (now.Hour == 1 && now.Minute < 30)
{ {
//douyinCommonService.UpdateAllCookieSyncedToZero();
//顺手清理下日志文件
LogFileCleaner.CleanOldLogFiles(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "logs"), 1); LogFileCleaner.CleanOldLogFiles(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "logs"), 1);
await Task.Delay(200); await Task.Delay(200);
} }
} }
protected override async Task<List<DouyinCookie>> GetValidCookies()
{
return await douyinCookieService.GetAllOpendAsync(x => !string.IsNullOrWhiteSpace(x.SavePath));
}
protected override bool IsCookieValid(DouyinCookie cookie)
protected override async Task<DouyinVideoInfoResponse> FetchVideoData(DouyinCookie cookie, string cursor, DouyinFollowed followed, DouyinCollectCate cate)
{ {
return !string.IsNullOrWhiteSpace(cookie.Cookies)&& !string.IsNullOrWhiteSpace(cookie.SavePath);
}
protected override async Task<DouyinVideoInfoResponse> FetchVideoData(DouyinCookie cookie, string cursor, DouyinFollowed followed,DouyinCollectItem collectItem=null)
{
if (cookie.UseCollectFolder && collectItem != null && !string.IsNullOrWhiteSpace(collectItem.CollectsId))
{
return await douyinHttpClientService.SyncCollectVideosByCollectId(cursor, count, cookie.Cookies, collectItem);
}
return await douyinHttpClientService.SyncCollectVideos(cursor, count, cookie.Cookies); return await douyinHttpClientService.SyncCollectVideos(cursor, count, cookie.Cookies);
} }
@@ -56,16 +35,6 @@ namespace dy.net.job
return data != null && data.HasMore == 1 && cookie.CollHasSyncd == 0; return data != null && data.HasMore == 1 && cookie.CollHasSyncd == 0;
} }
protected override string GetNextCursor(DouyinVideoInfoResponse data)
{
return data?.Cursor ?? "0";
}
protected override string GetAuthorAvatarBasePath(DouyinCookie cookie)
{
return Path.Combine(cookie.SavePath, "author");
}
protected override async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount,DouyinFollowed followed) protected override async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount,DouyinFollowed followed)
{ {
if (syncCount > 0) if (syncCount > 0)
@@ -81,31 +50,15 @@ namespace dy.net.job
} }
protected override VideoEntityDifferences GetVideoEntityDifferences(DouyinCookie cookie, Aweme item)
{
return new VideoEntityDifferences();
}
protected override string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed, DouyinCollectItem collectItem) protected override string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed, DouyinCollectCate cate)
{ {
if(collectItem!=null&&cookie.UseCollectFolder &&!string.IsNullOrWhiteSpace(collectItem.CollectsId)) var (tag1, _, _) = GetVideoTags(item);
{ var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : DouyinFileNameHelper.SanitizeLinuxFileName(tag1, "", true);
var safeCollectName = DouyinFileNameHelper.SanitizeLinuxFileName(collectItem.CollectsName, "", true); var folder = Path.Combine(cookie.SavePath, safeTag1, $"{DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId, true)}");
var folderCollect = Path.Combine(cookie.SavePath, safeCollectName); if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
var safeVideoDesc = DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId, true); return folder;
var folder = Path.Combine(folderCollect, safeVideoDesc);
if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
return folder;
}
else
{
var (tag1, _, _) = GetVideoTags(item);
var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : DouyinFileNameHelper.SanitizeLinuxFileName(tag1, "", true);
var folder = Path.Combine(cookie.SavePath, safeTag1, $"{DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId, true)}");
if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
return folder;
}
} }
} }
} }
+7 -27
View File
@@ -8,29 +8,19 @@ namespace dy.net.job
{ {
public class DouyinFavoritSyncJob : DouyinBasicSyncJob public class DouyinFavoritSyncJob : DouyinBasicSyncJob
{ {
public DouyinFavoritSyncJob( public DouyinFavoritSyncJob(DouyinCookieService douyinCookieService, DouyinHttpClientService douyinHttpClientService, DouyinVideoService douyinVideoService, DouyinCommonService douyinCommonService, DouyinFollowService douyinFollowService, DouyinMergeVideoService douyinMergeVideoService, DouyinCollectCateService douyinCollectCateService) : base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService, douyinCollectCateService)
DouyinCookieService douyinCookieService, {
DouyinHttpClientService douyinHttpClientService, }
DouyinVideoService douyinVideoService,
DouyinCommonService douyinCommonService,DouyinFollowService douyinFollowService,DouyinMergeVideoService douyinMergeVideoService)
: base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService) { }
protected override VideoTypeEnum VideoType => VideoTypeEnum.dy_favorite; protected override VideoTypeEnum VideoType => VideoTypeEnum.dy_favorite;
protected override async Task<List<DouyinCookie>> GetValidCookies() protected override async Task<List<DouyinCookie>> GetSyncCookies()
{ {
return await douyinCookieService.GetAllOpendAsync(x=> !string.IsNullOrWhiteSpace(x.FavSavePath)); return await douyinCookieService.GetOpendCookiesAsync(x=> !string.IsNullOrWhiteSpace(x.FavSavePath)&&!string.IsNullOrWhiteSpace(x.SecUserId));
} }
protected override bool IsCookieValid(DouyinCookie cookie) protected override async Task<DouyinVideoInfoResponse> FetchVideoData(DouyinCookie cookie, string cursor,DouyinFollowed followed, DouyinCollectCate cate)
{
return !string.IsNullOrWhiteSpace(cookie.Cookies) && cookie.Cookies.Length >= 1000 &&
!string.IsNullOrWhiteSpace(cookie.FavSavePath) &&
!string.IsNullOrWhiteSpace(cookie.SecUserId) && cookie.SecUserId.Length >= 10;
}
protected override async Task<DouyinVideoInfoResponse> FetchVideoData(DouyinCookie cookie, string cursor,DouyinFollowed followed, DouyinCollectItem collectId)
{ {
return await douyinHttpClientService.SyncFavoriteVideos(count, cursor, cookie.SecUserId, cookie.Cookies); return await douyinHttpClientService.SyncFavoriteVideos(count, cursor, cookie.SecUserId, cookie.Cookies);
} }
@@ -40,16 +30,6 @@ namespace dy.net.job
return data != null && data.HasMore == 1 && cookie.FavHasSyncd == 0; return data != null && data.HasMore == 1 && cookie.FavHasSyncd == 0;
} }
protected override string GetNextCursor(DouyinVideoInfoResponse data)
{
return data?.MaxCursor ?? "0";
}
protected override string GetAuthorAvatarBasePath(DouyinCookie cookie)
{
return Path.Combine(cookie.FavSavePath, "author");
}
protected override async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount, DouyinFollowed followed) protected override async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount, DouyinFollowed followed)
{ {
@@ -65,7 +45,7 @@ namespace dy.net.job
} }
} }
protected override string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed, DouyinCollectItem collectItem) protected override string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed,DouyinCollectCate cate)
{ {
var (tag1, _, _) = GetVideoTags(item); var (tag1, _, _) = GetVideoTags(item);
var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : DouyinFileNameHelper.SanitizeLinuxFileName(tag1,"",true); var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : DouyinFileNameHelper.SanitizeLinuxFileName(tag1,"",true);
+180 -230
View File
@@ -24,17 +24,14 @@ namespace dy.net.job
/// 抖音Cookie服务,用于获取和管理用户Cookie /// 抖音Cookie服务,用于获取和管理用户Cookie
/// </summary> /// </summary>
protected readonly DouyinCookieService douyinCookieService; protected readonly DouyinCookieService douyinCookieService;
/// <summary> /// <summary>
/// 抖音HTTP客户端服务,用于发送HTTP请求 /// 抖音HTTP客户端服务,用于发送HTTP请求
/// </summary> /// </summary>
protected readonly DouyinHttpClientService douyinHttpClientService; protected readonly DouyinHttpClientService douyinHttpClientService;
/// <summary> /// <summary>
/// 抖音视频服务,用于视频信息的数据库操作 /// 抖音视频服务,用于视频信息的数据库操作
/// </summary> /// </summary>
protected readonly DouyinVideoService douyinVideoService; protected readonly DouyinVideoService douyinVideoService;
/// <summary> /// <summary>
/// 抖音通用服务,用于获取应用配置等 /// 抖音通用服务,用于获取应用配置等
/// </summary> /// </summary>
@@ -43,20 +40,20 @@ namespace dy.net.job
/// 抖音关注列表 /// 抖音关注列表
/// </summary> /// </summary>
private readonly DouyinFollowService douyinFollowService; private readonly DouyinFollowService douyinFollowService;
/// <summary> /// <summary>
/// 图文合成视频 /// 图文合成视频
/// </summary> /// </summary>
private readonly DouyinMergeVideoService douyinMergeVideoService; private readonly DouyinMergeVideoService douyinMergeVideoService;
/// <summary> /// <summary>
/// 收藏夹、短剧、合集
/// </summary>
private readonly DouyinCollectCateService douyinCollectCateService;
/// <summary>
/// 随机数生成器,用于生成随机延迟,模拟人类操作 /// 随机数生成器,用于生成随机延迟,模拟人类操作
/// </summary> /// </summary>
protected readonly Random _random = new Random(); protected readonly Random _random = new Random();
/// <summary> /// <summary>
/// 每页请求的视频数量,可通过配置文件修改 /// 每页请求的视频数量.不可修改
/// </summary> /// </summary>
protected string count = "18"; protected string count = "18";
@@ -79,12 +76,6 @@ namespace dy.net.job
#region #region
/// <summary>
/// 任务类型名称,用于日志记录和区分不同的同步任务
/// 子类必须实现此属性并返回具体的任务类型
/// </summary>
//protected abstract string VideoType { get; }
protected abstract VideoTypeEnum VideoType { get; } protected abstract VideoTypeEnum VideoType { get; }
#endregion #endregion
@@ -100,13 +91,15 @@ namespace dy.net.job
/// <param name="douyinCommonService">抖音通用服务</param> /// <param name="douyinCommonService">抖音通用服务</param>
/// <param name="douyinFollowService">抖音关注的</param> /// <param name="douyinFollowService">抖音关注的</param>
/// <param name="douyinMergeVideoService">视频合成</param> /// <param name="douyinMergeVideoService">视频合成</param>
/// <param name="douyinCollectCateService"></param>
protected DouyinBasicSyncJob( protected DouyinBasicSyncJob(
DouyinCookieService douyinCookieService, DouyinCookieService douyinCookieService,
DouyinHttpClientService douyinHttpClientService, DouyinHttpClientService douyinHttpClientService,
DouyinVideoService douyinVideoService, DouyinVideoService douyinVideoService,
DouyinCommonService douyinCommonService, DouyinCommonService douyinCommonService,
DouyinFollowService douyinFollowService, DouyinFollowService douyinFollowService,
DouyinMergeVideoService douyinMergeVideoService) DouyinMergeVideoService douyinMergeVideoService,
DouyinCollectCateService douyinCollectCateService)
{ {
this.douyinCookieService = douyinCookieService ?? throw new ArgumentNullException(nameof(douyinCookieService)); this.douyinCookieService = douyinCookieService ?? throw new ArgumentNullException(nameof(douyinCookieService));
this.douyinHttpClientService = douyinHttpClientService ?? throw new ArgumentNullException(nameof(douyinHttpClientService)); this.douyinHttpClientService = douyinHttpClientService ?? throw new ArgumentNullException(nameof(douyinHttpClientService));
@@ -114,6 +107,7 @@ namespace dy.net.job
this.douyinCommonService = douyinCommonService ?? throw new ArgumentNullException(nameof(douyinCommonService)); this.douyinCommonService = douyinCommonService ?? throw new ArgumentNullException(nameof(douyinCommonService));
this.douyinFollowService = douyinFollowService; this.douyinFollowService = douyinFollowService;
this.douyinMergeVideoService = douyinMergeVideoService; this.douyinMergeVideoService = douyinMergeVideoService;
this.douyinCollectCateService = douyinCollectCateService;
} }
#endregion #endregion
@@ -135,22 +129,21 @@ namespace dy.net.job
var config = douyinCommonService.GetConfig(); var config = douyinCommonService.GetConfig();
if (config == null) if (config == null)
{ {
Log.Debug($"{VideoType}-未获取到系统配置,任务终止!!!"); Log.Debug($"{VideoType}-系统配置错误,任务终止!!!");
return; return;
} }
// 在处理Cookie之前执行的预处理 // 在处理Cookie之前执行的预处理
await BeforeProcessCookies(); await BeforeProcessCookies();
// 获取所有有效的Cookie // 获取所有有效的Cookie
var cookies = await GetValidCookies(); var cookies = await GetSyncCookies();
if (cookies == null || !cookies.Any()) if (cookies == null || !cookies.Any())
{ {
Log.Debug($"{VideoType}-无有效cookie或{VideoType}已关闭同步,任务终止!!!"); Log.Debug($"{VideoType}-Cookie无效或{VideoType}已关闭同步,任务终止!!!");
return; return;
} }
Log.Debug($"{VideoType}-共发现{cookies.Count}个有效的cookie,同步即将开始..."); Log.Debug($"{VideoType}:共发现{cookies.Count}个有效Cookie,同步开始...");
// 遍历每个有效的Cookie,执行同步 // 遍历每个有效的Cookie,执行同步
foreach (var cookie in cookies) foreach (var cookie in cookies)
@@ -174,16 +167,9 @@ namespace dy.net.job
/// 子类必须实现此方法,根据具体任务类型筛选有效的Cookie /// 子类必须实现此方法,根据具体任务类型筛选有效的Cookie
/// </summary> /// </summary>
/// <returns>有效的Cookie列表</returns> /// <returns>有效的Cookie列表</returns>
protected abstract Task<List<DouyinCookie>> GetValidCookies(); protected virtual async Task<List<DouyinCookie>> GetSyncCookies()
/// <summary>
/// 检查指定的Cookie是否有效
/// </summary>
/// <param name="cookie">要检查的Cookie</param>
/// <returns>如果Cookie有效,则为true;否则为false</returns>
protected virtual bool IsCookieValid(DouyinCookie cookie)
{ {
return !string.IsNullOrWhiteSpace(cookie.Cookies); return await douyinCookieService.GetOpendCookiesAsync(x => !string.IsNullOrWhiteSpace(x.SavePath));
} }
/// <summary> /// <summary>
@@ -192,14 +178,13 @@ namespace dy.net.job
/// </summary> /// </summary>
/// <param name="cookie">用户Cookie</param> /// <param name="cookie">用户Cookie</param>
/// <param name="cursor">分页游标,用于获取下一页数据</param> /// <param name="cursor">分页游标,用于获取下一页数据</param>
/// <param name="uperUid">关注的人</param> /// <param name="followed">关注的人</param>
/// <param name="collectId">收藏夹ID</param> /// <param name="cate">自定义收藏夹、合集、短剧</param>
/// <returns>视频信息对象,包含视频列表和分页信息</returns> /// <returns>视频信息对象,包含视频列表和分页信息</returns>
protected abstract Task<DouyinVideoInfoResponse> FetchVideoData(DouyinCookie cookie, string cursor, DouyinFollowed uperUid , DouyinCollectItem collectId); protected abstract Task<DouyinVideoInfoResponse> FetchVideoData(DouyinCookie cookie, string cursor, DouyinFollowed followed, DouyinCollectCate cate);
/// <summary> /// <summary>
/// 判断是否应该继续同步下一页数据 /// 判断是否应该继续同步下一页数据
/// 子类必须实现此方法,根据API返回的分页信息判断是否还有更多数据
/// </summary> /// </summary>
/// <param name="cookie">用户Cookie</param> /// <param name="cookie">用户Cookie</param>
/// <param name="data">当前获取到的视频数据</param> /// <param name="data">当前获取到的视频数据</param>
@@ -209,23 +194,29 @@ namespace dy.net.job
/// <summary> /// <summary>
/// 获取下一页数据的游标 /// 获取下一页数据的游标
/// 子类必须实现此方法,从当前视频数据中提取下一页的游标
/// </summary> /// </summary>
/// <param name="data">当前获取到的视频数据</param> /// <param name="data">当前获取到的视频数据</param>
/// <returns>下一页数据的游标</returns> /// <returns>下一页数据的游标</returns>
protected abstract string GetNextCursor(DouyinVideoInfoResponse data); private static string GetNextCursor(DouyinVideoInfoResponse data)
{
return data?.Cursor ?? (data?.MaxCursor ?? "0");
}
/// <summary> /// <summary>
/// 创建视频保存文件夹 /// 创建视频保存文件夹
/// 子类必须实现此方法,根据具体的存储策略创建文件夹
/// </summary> /// </summary>
/// <param name="cookie">用户Cookie</param> /// <param name="cookie">用户Cookie</param>
/// <param name="item">视频信息</param> /// <param name="item">视频信息</param>
/// <param name="followed">关注用户</param> /// <param name="followed">关注用户</param>
/// <param name="collectItem">收藏夹Item</param> /// <param name="cate"></param>
/// <param name="config">应用配置</param> /// <param name="config">应用配置</param>
/// <returns>创建的视频保存文件夹路径</returns> /// <returns>创建的视频保存文件夹路径</returns>
protected abstract string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed,DouyinCollectItem collectItem); protected virtual string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed,DouyinCollectCate cate)
{
var folder = Path.Combine(cookie.SavePath, DouyinFileNameHelper.SanitizeLinuxFileName(cate.Name, item.AwemeId, true));
if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
return folder;
}
/// <summary> /// <summary>
/// 获取视频文件名,默认就用id作文件名 /// 获取视频文件名,默认就用id作文件名
@@ -246,21 +237,29 @@ namespace dy.net.job
/// </summary> /// </summary>
/// <param name="cookie">用户Cookie</param> /// <param name="cookie">用户Cookie</param>
/// <returns>作者头像保存的基础路径</returns> /// <returns>作者头像保存的基础路径</returns>
protected abstract string GetAuthorAvatarBasePath(DouyinCookie cookie); //protected abstract string GetAuthorAvatarBasePath(DouyinCookie cookie);
/// <summary> /// <summary>
/// 处理同步完成后的操作 /// 处理同步完成后的操作
/// 子类必须实现此方法,可用于更新同步状态、发送通知等
/// </summary> /// </summary>
/// <param name="cookie">用户Cookie</param> /// <param name="cookie">用户Cookie</param>
/// <param name="syncCount">本次同步成功的视频数量</param> /// <param name="syncCount">本次同步成功的视频数量</param>
/// <param name="followed"></param> /// <param name="followed"></param>
/// <returns>一个表示异步操作的任务</returns> /// <returns>一个表示异步操作的任务</returns>
protected abstract Task HandleSyncCompletion(DouyinCookie cookie, int syncCount, DouyinFollowed followed = null); protected virtual async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount, DouyinFollowed followed = null)
{
if (syncCount > 0)
{
Serilog.Log.Debug($"{VideoType}-Cookie-[{cookie.UserName}],本次共同步成功{syncCount}条视频");
}
else
{
Serilog.Log.Debug($"{VideoType}-Cookie-[{cookie.UserName}],没有可以同步的新视频");
}
}
/// <summary> /// <summary>
/// 获取视频实体的差异信息 /// 获取视频实体的差异信息
/// 子类必须实现此方法,用于区分不同类型的视频(如普通视频、图片合成视频等)
/// </summary> /// </summary>
/// <param name="cookie">用户Cookie</param> /// <param name="cookie">用户Cookie</param>
/// <param name="item">视频信息</param> /// <param name="item">视频信息</param>
@@ -294,98 +293,64 @@ namespace dy.net.job
{ {
try try
{ {
// 检查Cookie是否有效
if (!IsCookieValid(cookie))
{
Log.Debug($"{VideoType}-Cookie[{cookie.UserName}]无效,任务终止!!!");
return;
}
Log.Debug($"{VideoType}- Cookie-[{cookie.UserName}]开始同步..."); Log.Debug($"{VideoType}- Cookie-[{cookie.UserName}]开始同步...");
//up主上传视频特殊处理 switch (VideoType)
if (VideoType == VideoTypeEnum.dy_follows)
{ {
int syncCount = 0; // 本次同步成功的视频数量 case VideoTypeEnum.dy_follows:
string cursor = "0"; // 初始游标
bool hasMore = true; // 是否还有更多数据
//查询关注列表开启了同步的关注
var follows = await douyinFollowService.GetSyncFollows(cookie.MyUserId);
var firstUp = follows?.Where(x => !string.IsNullOrWhiteSpace(x.SecUid)).FirstOrDefault();
if (firstUp == null)
{
return;
}
if (string.IsNullOrWhiteSpace(firstUp.SecUid))
{
Log.Debug($"{VideoType}-Cookie[{firstUp.UperName}]无效,没有sec_userid,任务终止!!!");
return;
}
foreach (var followed in follows)
{
cursor = "0"; // 初始游标
await GetViedos(cookie, config, syncCount, cursor, hasMore, followed);
hasMore = true;
// 处理同步完成后的操作
await HandleSyncCompletion(cookie, syncCount, followed);
}
}
else
{
// 仅在视频类型为抖音收藏时执行对应逻辑
if (VideoType == VideoTypeEnum.dy_collects)
{
// 统一声明变量,避免分散定义,提升可读性
int syncCount = 0; // 本次同步成功的视频数量
string cursor = "0"; // 初始游标
bool hasMore = true; // 是否还有更多数据
// 处理「使用收藏文件夹」和「不使用收藏文件夹」的统一逻辑
if (cookie.UseCollectFolder)
{ {
// 获取收藏文件夹列表 int syncCount = 0; // 本次同步成功的视频数量
var folders = await douyinHttpClientService.SyncCollectFolderList(cookie.Cookies,""); //查询关注列表开启了同步的关注
Log.Debug("获取收藏文件夹列表完成,共计{Count}个包含视频的收藏夹", folders?.CollectsList?.Count(x => x.TotalNumber > 0) ?? 0); var follows = await douyinFollowService.GetSyncFollows(cookie.MyUserId);
// 文件夹为空时,执行默认同步逻辑(与不使用文件夹逻辑一致) if (follows != null && follows.Any())
if (folders?.CollectsList == null || !folders.CollectsList.Any(x => x.TotalNumber > 0))
{ {
(syncCount, cursor, hasMore) = await GetViedos(cookie, config, syncCount, cursor, hasMore); foreach (var followed in follows)
await HandleSyncCompletion(cookie, syncCount);
}
// 文件夹不为空时,遍历每个文件夹同步视频
else
{
foreach (var item in folders.CollectsList.Where(x=>x.TotalNumber>0))
{ {
// 重置游标,每个文件夹独立从头同步 string cursor = "0";
string folderCursor = "0"; bool hasMore = true;
await GetViedos(cookie, config, syncCount, folderCursor, hasMore, null, item); (syncCount, cursor, hasMore) = await GetAndSaveViedos(cookie, config, syncCount, cursor, hasMore, followed);
await HandleSyncCompletion(cookie, syncCount, followed);
}
}
}
break;
case VideoTypeEnum.dy_favorite:
case VideoTypeEnum.dy_collects:
{
int syncCount = 0;
string cursor = "0";
bool hasMore = true;
(syncCount, cursor, hasMore) = await GetAndSaveViedos(cookie, config, syncCount, cursor, hasMore);
await HandleSyncCompletion(cookie, syncCount);
}
break;
case VideoTypeEnum.dy_mix:
case VideoTypeEnum.dy_series:
case VideoTypeEnum.dy_custom_collect:
{
int syncCount = 0; // 本次同步成功的视频数量
var cates = await douyinCollectCateService.GetSyncCates(cookie.Id, VideoType);
if (cates != null && cates.Any())
{
foreach (var cate in cates)
{
string cursor = "0";
bool hasMore = true;
(syncCount, cursor, hasMore) = await GetAndSaveViedos(cookie, config, syncCount, cursor, hasMore, null);
await HandleSyncCompletion(cookie, syncCount, null); await HandleSyncCompletion(cookie, syncCount, null);
} }
} }
} }
// 不使用收藏文件夹,直接执行默认同步逻辑 break;
else case VideoTypeEnum.ImageVideo:
{ default:
(syncCount, cursor, hasMore) = await GetViedos(cookie, config, syncCount, cursor, hasMore); break;
await HandleSyncCompletion(cookie, syncCount);
}
}
// 非抖音收藏类型,执行统一的默认同步逻辑
else
{
// 提取重复变量,与上方逻辑保持一致命名
int syncCount = 0;
string cursor = "0";
bool hasMore = true;
(syncCount, cursor, hasMore) = await GetViedos(cookie, config, syncCount, cursor, hasMore);
await HandleSyncCompletion(cookie, syncCount);
}
} }
} }
catch (Exception ex) catch (Exception ex)
{ {
@@ -393,30 +358,25 @@ namespace dy.net.job
} }
} }
private async Task<(int syncCount, string cursor, bool hasMore)> GetViedos(DouyinCookie cookie, AppConfig config, int syncCount, string cursor, bool hasMore, DouyinFollowed followed = null,DouyinCollectItem collectItem=null) private async Task<(int syncCount, string cursor, bool hasMore)> GetAndSaveViedos(DouyinCookie cookie, AppConfig config, int syncCount, string cursor, bool hasMore, DouyinFollowed followed = null, DouyinCollectCate cate = null)
{ {
// 循环获取视频数据 // 循环获取视频数据
while (hasMore) while (hasMore)
{ {
// 获取视频数据 // 获取视频数据
var data = await FetchVideoData(cookie, cursor, followed,collectItem); var data = await FetchVideoData(cookie, cursor, followed, cate);
if (data == null) if (data == null|| data.AwemeList == null || !data.AwemeList.Any())
{ {
Log.Debug($"{VideoType}-Cookie[{cookie.UserName}]同步数据失败!!!");
break; break;
} }
// 判断是否还有更多数据 // 判断是否还有更多数据
hasMore = ShouldContinueSync(cookie, data, followed); hasMore = ShouldContinueSync(cookie, data, followed);
// 获取下一页游标 // 获取下一页游标
cursor = GetNextCursor(data); cursor = GetNextCursor(data);
// 如果没有视频数据,退出循环
if (data.AwemeList == null || !data.AwemeList.Any())
break;
// 处理视频列表 // 处理视频列表
var videos = await ProcessVideoList(cookie, data, config, followed, collectItem); var videos = await ProcessVideoList(cookie, data, config, followed, cate);
// 保存视频信息到数据库 // 保存视频信息到数据库
syncCount += await SaveVideos(videos); syncCount += await SaveVideos(videos);
@@ -426,7 +386,7 @@ namespace dy.net.job
break; break;
} }
//随机等待 //随机等待
await Task.Delay(_random.Next(5, 10) * 1000); await Task.Delay(_random.Next(2, 10) * 1000);
} }
return (syncCount, cursor, hasMore); return (syncCount, cursor, hasMore);
@@ -474,9 +434,9 @@ namespace dy.net.job
/// <param name="data">视频信息对象</param> /// <param name="data">视频信息对象</param>
/// <param name="config">应用配置</param> /// <param name="config">应用配置</param>
/// <param name="followed">关注的</param> /// <param name="followed">关注的</param>
/// <param name="collectItem">收藏夹Item</param> /// <param name="cate">收藏夹、合集、短剧</param>
/// <returns>处理后的视频实体列表</returns> /// <returns>处理后的视频实体列表</returns>
protected async Task<List<DouyinVideo>> ProcessVideoList(DouyinCookie cookie, DouyinVideoInfoResponse data, AppConfig config, DouyinFollowed followed = null,DouyinCollectItem collectItem=null) protected async Task<List<DouyinVideo>> ProcessVideoList(DouyinCookie cookie, DouyinVideoInfoResponse data, AppConfig config, DouyinFollowed followed = null, DouyinCollectCate cate = null)
{ {
var videos = new List<DouyinVideo>(); var videos = new List<DouyinVideo>();
foreach (var item in data.AwemeList) foreach (var item in data.AwemeList)
@@ -485,48 +445,54 @@ namespace dy.net.job
//{ //{
// continue; // continue;
//} //}
//if (!item.Desc.Contains("小房车正式启程"))
//{
// continue;
//}
//判断视频是否是强制删除且不再下载的视频
var deleteVideo = await douyinCommonService.ExistDeleteVideo(item.AwemeId);
if (deleteVideo)
{
//Log.Debug($"{VideoType}-视频-{item.AwemeId}-[{item.Desc}]已被标记为强制删除,跳过下载");
continue;
}
//判断是否存在视频,是否根据去重规则进行去重处理。。
// 1. 查询数据库中是否已存在该视频(通过 AwemeId 唯一标识) // 1. 查询数据库中是否已存在该视频(通过 AwemeId 唯一标识)
var exitVideo = await douyinVideoService.GetByAwemeId(item.AwemeId); var exitVideo = await douyinVideoService.GetByAwemeId(item.AwemeId);
bool Goon = await AutoDistinct(config, item, cookie, exitVideo); if (cate == null)
if (!Goon)
{ {
continue; //判断视频是否是强制删除且不再下载的视频
} var deleteVideo = await douyinCommonService.ExistDeleteVideo(item.AwemeId);
//如果存在,但是因为去重规则,选择的最高优先级变更了,需要删除原来的,重新下载。 if (deleteVideo)
if (exitVideo != null)
{
await douyinVideoService.DeleteById(exitVideo.Id);
}
var uper = await douyinFollowService.GetByUperId(item.AuthorUserId.ToString(), cookie.MyUserId);
if (uper != null)
{
if (uper.FullSync)
{ {
followed ??= uper; //Log.Debug($"{VideoType}-视频-{item.AwemeId}-[{item.Desc}]已被标记为强制删除,跳过下载");
continue;
}
bool Goon = await AutoDistinct(config, exitVideo);
if (!Goon)
{
continue;
}
//如果存在,但是因为去重规则,选择的最高优先级变更了,需要删除原来的,重新下载。
if (exitVideo != null)
{
await douyinVideoService.DeleteById(exitVideo.Id);
}
var uper = await douyinFollowService.GetByUperId(item.AuthorUserId.ToString(), cookie.MyUserId);
if (uper != null)
{
if (uper.FullSync)
{
followed ??= uper;
}
} }
} }
else
{
if (exitVideo != null && exitVideo.ViedoType == cate.CateType &&File.Exists(exitVideo.VideoSavePath))
{
continue;
}
}
// 处理单个视频 // 处理单个视频
var video = await ProcessSingleVideo(cookie, item, data, config, followed,collectItem); var video = await ProcessSingleVideo(cookie, item, data, config, followed);
if (video != null) if (video != null)
videos.Add(video); videos.Add(video);
else else
{ {
//处理多个视频-组合的图文视频--类似动图。 //处理多个视频-组合的图文视频--类似动图。
List<DouyinDynamicVideoDto> dynamicVideoUrls = new List<DouyinDynamicVideoDto>(); List<DouyinDynamicVideoDto> dynamicVideoUrls = new List<DouyinDynamicVideoDto>();
// 当需要下载动态视频时,获取其他URL // 当需要下载动态视频时,获取其他URL
@@ -557,7 +523,7 @@ namespace dy.net.job
} }
// 处理核心逻辑 // 处理核心逻辑
if (config.DownDynamicVideo && dynamicVideoUrls.Count > 0) if (dynamicVideoUrls.Count > 0)
{ {
// 处理动态视频 // 处理动态视频
var dynamicVideo = await ProcessDynamicVideo(dynamicVideoUrls, cookie, item, data, config); var dynamicVideo = await ProcessDynamicVideo(dynamicVideoUrls, cookie, item, data, config);
@@ -569,15 +535,15 @@ namespace dy.net.job
Log.Debug($"{VideoType}-动态视频[{item.Desc}],下载成功 ,共{dynamicVideos?.Count}个视频..."); Log.Debug($"{VideoType}-动态视频[{item.Desc}],下载成功 ,共{dynamicVideos?.Count}个视频...");
if (dynamicVideos != null && dynamicVideos.Count > 0) if (dynamicVideos != null && dynamicVideos.Count > 0)
{ {
var savePath = DouyinFileNameHelper.RemoveNumberSuffix(dynamicVideo.VideoSavePath);
//音频文件下载地址
var mp3Url= item?.Music?.PlayUrl?.UrlList?.FirstOrDefault();
var outPath = await douyinMergeVideoService.MergeMultipleVideosAsync(dynamicVideos,mp3Url, savePath,cookie.Cookies); var savePath = DouyinFileNameHelper.RemoveNumberSuffix(dynamicVideo.VideoSavePath);
if(!string.IsNullOrWhiteSpace(outPath.mp4Path))
//音频文件下载地址
var mp3Url = item?.Music?.PlayUrl?.UrlList?.FirstOrDefault();
var outPath = await douyinMergeVideoService.MergeMultipleVideosAsync(dynamicVideos, mp3Url, savePath, cookie.Cookies);
if (!string.IsNullOrWhiteSpace(outPath.mp4Path))
{ {
if (File.Exists(outPath.mp4Path)) if (File.Exists(outPath.mp4Path))
{ {
@@ -608,10 +574,10 @@ namespace dy.net.job
} }
else else
{ {
// 统一处理图文视频逻辑 // 处理图文视频逻辑
if (config.DownImageVideo || config.DownMp3 || config.DownImage) if (config.DownImageVideo || config.DownMp3 || config.DownImage)
{ {
var mergevideo = await ProcessImageSetAndMergeToVideo(cookie, item, data, config, followed, collectItem); var mergevideo = await ProcessImageSetAndMergeToVideo(cookie, item, config, followed,cate);
if (mergevideo != null) if (mergevideo != null)
videos.Add(mergevideo); videos.Add(mergevideo);
} }
@@ -621,7 +587,7 @@ namespace dy.net.job
return videos; return videos;
} }
private async Task<bool> AutoDistinct(AppConfig config, Aweme item, DouyinCookie cookie, DouyinVideo exitVideo) private async Task<bool> AutoDistinct(AppConfig config, DouyinVideo exitVideo)
{ {
// 去重,检查视频是否已存在(按优先级下载) // 去重,检查视频是否已存在(按优先级下载)
if (config.AutoDistinct) if (config.AutoDistinct)
@@ -806,9 +772,9 @@ namespace dy.net.job
/// <param name="data">视频信息对象</param> /// <param name="data">视频信息对象</param>
/// <param name="config">应用配置</param> /// <param name="config">应用配置</param>
/// <param name="followed">关注</param> /// <param name="followed">关注</param>
/// <param name="collectItem">收藏夹Item</param> /// <param name="cate"></param>
/// <returns>处理后的视频实体,如果处理失败则为null</returns> /// <returns>处理后的视频实体,如果处理失败则为null</returns>
protected async Task<DouyinVideo> ProcessSingleVideo(DouyinCookie cookie, Aweme item, DouyinVideoInfoResponse data, AppConfig config, DouyinFollowed followed = null,DouyinCollectItem collectItem=null) protected async Task<DouyinVideo> ProcessSingleVideo(DouyinCookie cookie, Aweme item, DouyinVideoInfoResponse data, AppConfig config, DouyinFollowed followed = null,DouyinCollectCate cate=null)
{ {
// 检查视频数据是否有效 // 检查视频数据是否有效
if (!IsAwemeValid(item)) return null; if (!IsAwemeValid(item)) return null;
@@ -824,7 +790,7 @@ namespace dy.net.job
// 获取视频标签 // 获取视频标签
var (tag1, tag2, tag3) = GetVideoTags(item); var (tag1, tag2, tag3) = GetVideoTags(item);
// 创建保存文件夹 // 创建保存文件夹
var saveFolder = CreateSaveFolder(cookie, item, config, followed,collectItem); var saveFolder = CreateSaveFolder(cookie, item, config, followed, cate);
// 获取视频文件名 // 获取视频文件名
var fileName = GetVideoFileName(cookie, item, config); var fileName = GetVideoFileName(cookie, item, config);
// 拼接视频保存路径 // 拼接视频保存路径
@@ -860,24 +826,20 @@ namespace dy.net.job
// 下载视频封面 // 下载视频封面
await DownVideoCover(item, saveFolder, cookie, config); await DownVideoCover(item, saveFolder, cookie, config);
// 下载作者头像 // 下载作者头像
var (avatarSavePath, avatarUrl) = await DownAuthorAvatar(cookie, item); //var (avatarSavePath, avatarUrl) = await DownAuthorAvatar(cookie, item);
// 创建视频实体 // 创建视频实体
return await CreateVideoEntity(config, cookie, item, v, savePath, saveFolder, tag1, tag2, tag3, avatarSavePath, avatarUrl, data); return await CreateVideoEntity(config, cookie, item, v, savePath, saveFolder, tag1, tag2, tag3);
} }
protected async Task<DouyinVideo> ProcessDynamicVideo(List<DouyinDynamicVideoDto> dynamicUrls, DouyinCookie cookie, Aweme item, DouyinVideoInfoResponse data, AppConfig config, DouyinFollowed followed = null,DouyinCollectCate cate=null)
protected async Task<DouyinVideo> ProcessDynamicVideo(List<DouyinDynamicVideoDto> dynamicUrls, DouyinCookie cookie, Aweme item, DouyinVideoInfoResponse data, AppConfig config, DouyinFollowed followed = null,DouyinCollectItem collectItem=null)
{ {
// 获取视频标签 // 获取视频标签
var (tag1, tag2, tag3) = GetVideoTags(item); var (tag1, tag2, tag3) = GetVideoTags(item);
// 创建保存文件夹 // 创建保存文件夹
var saveFolder = CreateSaveFolder(cookie, item, config, followed,collectItem); var saveFolder = CreateSaveFolder(cookie, item, config, followed, cate);
// 获取视频文件名 // 获取视频文件名
//var fileName = DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId) + ".mp4"; //var fileName = DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId) + ".mp4";
@@ -894,7 +856,7 @@ namespace dy.net.job
{ {
// 下载动态视频 // 下载动态视频
var dynamicSavePath = savePath.Replace(".mp4", $"_00{i}.mp4"); var dynamicSavePath = savePath.Replace(".mp4", $"_00{i}.mp4");
DouyinDynamicVideoDto v = new DouyinDynamicVideoDto() { Height = dynamicUrl.Height, Width = dynamicUrl.Width,Path= dynamicSavePath }; DouyinDynamicVideoDto v = new DouyinDynamicVideoDto() { Height = dynamicUrl.Height, Width = dynamicUrl.Width, Path = dynamicSavePath };
i++; i++;
// 如果文件已存在,跳过 // 如果文件已存在,跳过
@@ -932,19 +894,13 @@ namespace dy.net.job
{ {
Width = item?.Images?.FirstOrDefault()?.Width ?? 0, Width = item?.Images?.FirstOrDefault()?.Width ?? 0,
Height = item?.Images?.FirstOrDefault()?.Height ?? 0, Height = item?.Images?.FirstOrDefault()?.Height ?? 0,
DataSize = DouyinFileUtils.GetTotalFileSize(dynamicSavePaths.Select(x=>x.Path).ToList()) // 合成视频的文件大小 DataSize = DouyinFileUtils.GetTotalFileSize(dynamicSavePaths.Select(x => x.Path).ToList()) // 合成视频的文件大小
} }
}; };
return await CreateVideoEntity(config, cookie, item, virtualBitRate, dynamicSavePaths.FirstOrDefault()?.Path, saveFolder, tag1, tag2, tag3, "", "", data, dynamicSavePaths); return await CreateVideoEntity(config, cookie, item, virtualBitRate, dynamicSavePaths.FirstOrDefault()?.Path, saveFolder, tag1, tag2, tag3, dynamicSavePaths);
} }
/// <summary> /// <summary>
/// 寻找其他视频地址下载... /// 寻找其他视频地址下载...
/// </summary> /// </summary>
@@ -1010,8 +966,9 @@ namespace dy.net.job
/// <param name="data">视频信息对象</param> /// <param name="data">视频信息对象</param>
/// <param name="config">应用配置</param> /// <param name="config">应用配置</param>
/// <param name="followed">应用配置</param> /// <param name="followed">应用配置</param>
/// <param name="cate"></param>
/// <returns>合成后的视频实体,如果处理失败则为null</returns> /// <returns>合成后的视频实体,如果处理失败则为null</returns>
protected async Task<DouyinVideo> ProcessImageSetAndMergeToVideo(DouyinCookie cookie, Aweme item, DouyinVideoInfoResponse data, AppConfig config, DouyinFollowed followed,DouyinCollectItem collectItem) protected async Task<DouyinVideo> ProcessImageSetAndMergeToVideo(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed,DouyinCollectCate cate)
{ {
try try
{ {
@@ -1043,7 +1000,7 @@ namespace dy.net.job
} }
else else
{ {
fileNamefolder = CreateSaveFolder(cookie, item, config, followed,collectItem); fileNamefolder = CreateSaveFolder(cookie, item, config, followed, cate);
} }
if (!Directory.Exists(fileNamefolder)) Directory.CreateDirectory(fileNamefolder); if (!Directory.Exists(fileNamefolder)) Directory.CreateDirectory(fileNamefolder);
@@ -1113,7 +1070,7 @@ namespace dy.net.job
// 下载视频封面(使用第一张图片作为封面) // 下载视频封面(使用第一张图片作为封面)
await DownVideoCover(imageUrls.FirstOrDefault(), fileNamefolder, cookie, item, config); await DownVideoCover(imageUrls.FirstOrDefault(), fileNamefolder, cookie, item, config);
// 下载作者头像 // 下载作者头像
var (avatarSavePath, avatarUrl) = await DownAuthorAvatar(cookie, item); //var (avatarSavePath, avatarUrl) = await DownAuthorAvatar(cookie, item);
// 获取视频标签 // 获取视频标签
var (tag1, tag2, tag3) = GetVideoTags(item); var (tag1, tag2, tag3) = GetVideoTags(item);
@@ -1131,7 +1088,7 @@ namespace dy.net.job
// 创建视频实体 // 创建视频实体
var videoEntity = await CreateVideoEntity(config, var videoEntity = await CreateVideoEntity(config,
cookie, item, virtualBitRate, savePath, fileNamefolder, cookie, item, virtualBitRate, savePath, fileNamefolder,
tag1, tag2, tag3, avatarSavePath, avatarUrl, data); tag1, tag2, tag3);
// 特殊处理合成视频的字段 // 特殊处理合成视频的字段
videoEntity.FileHash = string.Empty; // 合成视频没有原始文件哈希 videoEntity.FileHash = string.Empty; // 合成视频没有原始文件哈希
@@ -1171,8 +1128,6 @@ namespace dy.net.job
} }
} }
/// <summary> /// <summary>
/// 下载视频封面 /// 下载视频封面
/// 从视频信息中提取封面URL并下载到指定文件夹 /// 从视频信息中提取封面URL并下载到指定文件夹
@@ -1196,25 +1151,25 @@ namespace dy.net.job
/// <param name="cookie">用户Cookie</param> /// <param name="cookie">用户Cookie</param>
/// <param name="item">视频信息</param> /// <param name="item">视频信息</param>
/// <returns>一个元组,包含头像保存路径和头像URL</returns> /// <returns>一个元组,包含头像保存路径和头像URL</returns>
protected async Task<(string savePath, string url)> DownAuthorAvatar(DouyinCookie cookie, Aweme item) //protected async Task<(string savePath, string url)> DownAuthorAvatar(DouyinCookie cookie, Aweme item)
{ //{
if (item.Author == null) return (null, null); // if (item.Author == null) return (null, null);
// 优先获取高清头像 // // 优先获取高清头像
var avatarUrl = item.Author.AvatarLarger?.UrlList?.FirstOrDefault() ?? item.Author.AvatarThumb?.UrlList?.FirstOrDefault(); // var avatarUrl = item.Author.AvatarLarger?.UrlList?.FirstOrDefault() ?? item.Author.AvatarThumb?.UrlList?.FirstOrDefault();
if (string.IsNullOrWhiteSpace(avatarUrl)) return (null, null); // if (string.IsNullOrWhiteSpace(avatarUrl)) return (null, null);
// 拼接头像保存路径 // // 拼接头像保存路径
var avatarSavePath = Path.Combine(GetAuthorAvatarBasePath(cookie), $"{item.Author.Uid}.jpg"); // var avatarSavePath = Path.Combine(GetAuthorAvatarBasePath(cookie), $"{item.Author.Uid}.jpg");
var avatarDir = Path.GetDirectoryName(avatarSavePath); // var avatarDir = Path.GetDirectoryName(avatarSavePath);
// 创建头像保存文件夹 // // 创建头像保存文件夹
if (!Directory.Exists(avatarDir)) Directory.CreateDirectory(avatarDir); // if (!Directory.Exists(avatarDir)) Directory.CreateDirectory(avatarDir);
// 如果头像文件不存在,则下载 // // 如果头像文件不存在,则下载
if (!File.Exists(avatarSavePath)) // if (!File.Exists(avatarSavePath))
{ // {
await douyinHttpClientService.DownloadAsync(avatarUrl, avatarSavePath, cookie.Cookies); // await douyinHttpClientService.DownloadAsync(avatarUrl, avatarSavePath, cookie.Cookies);
} // }
return (avatarSavePath, avatarUrl); // return (avatarSavePath, avatarUrl);
} //}
#endregion #endregion
@@ -1226,7 +1181,7 @@ namespace dy.net.job
/// </summary> /// </summary>
/// <param name="item">视频信息</param> /// <param name="item">视频信息</param>
/// <returns>如果视频数据有效,则为true;否则为false</returns> /// <returns>如果视频数据有效,则为true;否则为false</returns>
private bool IsAwemeValid(Aweme item) => item != null && item.Video != null && item.Video.BitRate != null; private static bool IsAwemeValid(Aweme item) => item != null && item.Video != null && item.Video.BitRate != null;
/// <summary> /// <summary>
/// 获取视频标签 /// 获取视频标签
@@ -1326,17 +1281,17 @@ namespace dy.net.job
/// <returns>创建的视频实体对象</returns> /// <returns>创建的视频实体对象</returns>
private async Task<DouyinVideo> CreateVideoEntity(AppConfig config, private async Task<DouyinVideo> CreateVideoEntity(AppConfig config,
DouyinCookie cookie, Aweme item, VideoBitRate bitRate, string savePath, string saveFolder, DouyinCookie cookie, Aweme item, VideoBitRate bitRate, string savePath, string saveFolder,
string tag1, string tag2, string tag3, string avatarSavePath, string avatarUrl, DouyinVideoInfoResponse data, List<DouyinDynamicVideoDto> dynamicVideos = null) string tag1, string tag2, string tag3, List<DouyinDynamicVideoDto> dynamicVideos = null)
{ {
var diffs = GetVideoEntityDifferences(cookie, item); var diffs = GetVideoEntityDifferences(cookie, item);
var avatarUrl = item.Author.AvatarLarger?.UrlList?.FirstOrDefault() ?? item.Author.AvatarThumb?.UrlList?.FirstOrDefault();
var video = new DouyinVideo var video = new DouyinVideo
{ {
ViedoType = VideoType, ViedoType = VideoType,
AwemeId = item.AwemeId, AwemeId = item.AwemeId,
Author = item.Author?.Nickname, Author = item.Author?.Nickname,
AuthorId = item.Author?.Uid, AuthorId = item.Author?.Uid,
AuthorAvatar = avatarSavePath, AuthorAvatar = "",
AuthorAvatarUrl = avatarUrl, AuthorAvatarUrl = avatarUrl,
CreateTime = DateTimeUtil.Convert10BitTimestamp(item.CreateTime), CreateTime = DateTimeUtil.Convert10BitTimestamp(item.CreateTime),
VideoTitle = string.IsNullOrWhiteSpace(item.Desc) ? $"{item.Author?.Nickname}-{item.CreateTime}" : item.Desc, VideoTitle = string.IsNullOrWhiteSpace(item.Desc) ? $"{item.Author?.Nickname}-{item.CreateTime}" : item.Desc,
@@ -1377,11 +1332,6 @@ namespace dy.net.job
/// </summary> /// </summary>
public class VideoEntityDifferences public class VideoEntityDifferences
{ {
/// <summary>
/// 视频类型
/// </summary>
//public VideoTypeEnum VideoType { get; set; }
/// <summary> /// <summary>
/// 简化的视频标题 /// 简化的视频标题
/// </summary> /// </summary>
-124
View File
@@ -1,124 +0,0 @@
using dy.net.model.entity;
using dy.net.model.response;
using dy.net.service;
using Quartz;
using System.Collections.Generic;
namespace dy.net.job
{
[DisallowConcurrentExecution] // 禁止并发执行,确保同一时间只有一个实例在运行
public class DouyinCollectCatesSyncJob : IJob
{
/// <summary>
/// 抖音Cookie服务,用于获取和管理用户Cookie
/// </summary>
protected readonly DouyinCookieService _dyCookieService;
/// <summary>
/// 抖音HTTP客户端服务,用于发送HTTP请求
/// </summary>
protected readonly DouyinHttpClientService _douyinService;
/// <summary>
/// 抖音关注服务,用于管理关注数据
/// </summary>
protected readonly DouyinFollowService _followService;
protected readonly DouyinCommonService douyinCommonService;
public DouyinCollectCatesSyncJob(DouyinCookieService dyCookieService, DouyinHttpClientService douyinService, DouyinFollowService followService, DouyinCommonService douyinCommonService)
{
_dyCookieService = dyCookieService;
_douyinService = douyinService;
_followService = followService;
this.douyinCommonService = douyinCommonService;
}
public async Task Execute(IJobExecutionContext context)
{
var cookies = await _dyCookieService.GetAllOpendAsync();
if (cookies != null && cookies.Any())
{
AppConfig conf = douyinCommonService.GetConfig();
foreach (var ck in cookies)
{
if (string.IsNullOrWhiteSpace(ck.SecUserId))
{
Serilog.Log.Debug($"同步关注列表博主[{ck.UserName}]-未设置secuserid,跳过.");
continue;
}
string count = "20";
string offset = "0";
bool hasmore = true;
int total= 0;
List<FollowingsItem> follows = new List<FollowingsItem>();
while (hasmore)
{
try
{
var data = await _douyinService.SyncMyFollows(count, offset, ck.SecUserId, ck.Cookies, async err =>
{
if (err.StatusCode != 0)
{
Serilog.Log.Error($"同步关注列表博主[{ck.UserName}]关注列表时发生错误,错误信息:{err.StatusMsg},状态码:{err.StatusCode}");
}
// 如果是未登录错误,则跳出循环
if (err.StatusCode == 8)
{
hasmore = false;
}
ck.StatusMsg = err.StatusCode == 8 ? "无效" : "正常";
ck.StatusCode = err.StatusCode;
await _dyCookieService.UpdateAsync(ck);
});
if (data != null)
{
// 绑定我的用户ID--之前没有这个字段
if (string.IsNullOrWhiteSpace(ck.MyUserId))
{
ck.MyUserId = data.MySelfUserId;
await _dyCookieService.UpdateAsync(ck);
}
total = data.Total;
hasmore = data.HasMore;
offset = data.Offset.ToString();
if (data.Followings != null && data.Followings.Count > 0)
{
follows.AddRange(data.Followings);
}
}
// 只有第一次启动时,才进行全部同步
if (!conf.IsFirstRunning)
{
hasmore = false;
}
}
catch (Exception ex)
{
Serilog.Log.Error($"关注列表同步失败,{ex.Message}");
hasmore = false;
break;
}
}
if (follows.Count > 0)
{
var (add, update, succ) = await _followService.Sync(follows, ck);
if (!succ) hasmore = false;
else
{
hasmore = add > 100|| update > 100;//一次最多100条数据
}
}
}
await douyinCommonService.SetConfigNotFirstRunning();
}
}
}
}
+29
View File
@@ -0,0 +1,29 @@
using dy.net.model.dto;
using dy.net.model.entity;
using dy.net.model.response;
using dy.net.service;
using dy.net.utils;
using System;
namespace dy.net.job
{
public class DouyinCollectCustomSyncJob : DouyinBasicSyncJob
{
public DouyinCollectCustomSyncJob(DouyinCookieService douyinCookieService, DouyinHttpClientService douyinHttpClientService, DouyinVideoService douyinVideoService, DouyinCommonService douyinCommonService, DouyinFollowService douyinFollowService, DouyinMergeVideoService douyinMergeVideoService, DouyinCollectCateService douyinCollectCateService) : base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService, douyinCollectCateService)
{
}
protected override VideoTypeEnum VideoType => VideoTypeEnum.dy_custom_collect;
protected override async Task<DouyinVideoInfoResponse> FetchVideoData(DouyinCookie cookie, string cursor, DouyinFollowed followed, DouyinCollectCate cate)
{
return await douyinHttpClientService.SyncCollectVideos(cursor, count, cookie.Cookies);
}
protected override bool ShouldContinueSync(DouyinCookie cookie, DouyinVideoInfoResponse data, DouyinFollowed followed = null)
{
return data != null && data.HasMore == 1 && cookie.CollHasSyncd == 0;
}
}
}
-124
View File
@@ -1,124 +0,0 @@
using dy.net.model.entity;
using dy.net.model.response;
using dy.net.service;
using Quartz;
using System.Collections.Generic;
namespace dy.net.job
{
[DisallowConcurrentExecution] // 禁止并发执行,确保同一时间只有一个实例在运行
public class DouyinFollowedUsersSyncJob : IJob
{
/// <summary>
/// 抖音Cookie服务,用于获取和管理用户Cookie
/// </summary>
protected readonly DouyinCookieService _dyCookieService;
/// <summary>
/// 抖音HTTP客户端服务,用于发送HTTP请求
/// </summary>
protected readonly DouyinHttpClientService _douyinService;
/// <summary>
/// 抖音关注服务,用于管理关注数据
/// </summary>
protected readonly DouyinFollowService _followService;
protected readonly DouyinCommonService douyinCommonService;
public DouyinFollowedUsersSyncJob(DouyinCookieService dyCookieService, DouyinHttpClientService douyinService, DouyinFollowService followService, DouyinCommonService douyinCommonService)
{
_dyCookieService = dyCookieService;
_douyinService = douyinService;
_followService = followService;
this.douyinCommonService = douyinCommonService;
}
public async Task Execute(IJobExecutionContext context)
{
var cookies = await _dyCookieService.GetAllOpendAsync();
if (cookies != null && cookies.Any())
{
AppConfig conf = douyinCommonService.GetConfig();
foreach (var ck in cookies)
{
if (string.IsNullOrWhiteSpace(ck.SecUserId))
{
Serilog.Log.Debug($"同步关注列表博主[{ck.UserName}]-未设置secuserid,跳过.");
continue;
}
string count = "20";
string offset = "0";
bool hasmore = true;
int total= 0;
List<FollowingsItem> follows = new List<FollowingsItem>();
while (hasmore)
{
try
{
var data = await _douyinService.SyncMyFollows(count, offset, ck.SecUserId, ck.Cookies, async err =>
{
if (err.StatusCode != 0)
{
Serilog.Log.Error($"同步关注列表博主[{ck.UserName}]关注列表时发生错误,错误信息:{err.StatusMsg},状态码:{err.StatusCode}");
}
// 如果是未登录错误,则跳出循环
if (err.StatusCode == 8)
{
hasmore = false;
}
ck.StatusMsg = err.StatusCode == 8 ? "无效" : "正常";
ck.StatusCode = err.StatusCode;
await _dyCookieService.UpdateAsync(ck);
});
if (data != null)
{
// 绑定我的用户ID--之前没有这个字段
if (string.IsNullOrWhiteSpace(ck.MyUserId))
{
ck.MyUserId = data.MySelfUserId;
await _dyCookieService.UpdateAsync(ck);
}
total = data.Total;
hasmore = data.HasMore;
offset = data.Offset.ToString();
if (data.Followings != null && data.Followings.Count > 0)
{
follows.AddRange(data.Followings);
}
}
// 只有第一次启动时,才进行全部同步
if (!conf.IsFirstRunning)
{
hasmore = false;
}
}
catch (Exception ex)
{
Serilog.Log.Error($"关注列表同步失败,{ex.Message}");
hasmore = false;
break;
}
}
if (follows.Count > 0)
{
var (add, update, succ) = await _followService.Sync(follows, ck);
if (!succ) hasmore = false;
else
{
hasmore = add > 100|| update > 100;//一次最多100条数据
}
}
}
await douyinCommonService.SetConfigNotFirstRunning();
}
}
}
}
+5 -20
View File
@@ -13,24 +13,18 @@ namespace dy.net.job
{ {
public class DouyinFollowedViedoSyncJob : DouyinBasicSyncJob public class DouyinFollowedViedoSyncJob : DouyinBasicSyncJob
{ {
public DouyinFollowedViedoSyncJob(DouyinCookieService douyinCookieService, DouyinHttpClientService douyinHttpClientService, DouyinVideoService douyinVideoService, DouyinCommonService douyinCommonService, DouyinFollowService douyinFollowService, DouyinMergeVideoService douyinMergeVideoService) : base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService) public DouyinFollowedViedoSyncJob(DouyinCookieService douyinCookieService, DouyinHttpClientService douyinHttpClientService, DouyinVideoService douyinVideoService, DouyinCommonService douyinCommonService, DouyinFollowService douyinFollowService, DouyinMergeVideoService douyinMergeVideoService, DouyinCollectCateService douyinCollectCateService) : base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService, douyinCollectCateService)
{ {
} }
protected override VideoTypeEnum VideoType => VideoTypeEnum.dy_follows; protected override VideoTypeEnum VideoType => VideoTypeEnum.dy_follows;
protected override async Task<List<DouyinCookie>> GetValidCookies() protected override async Task<List<DouyinCookie>> GetSyncCookies()
{ {
return await douyinCookieService.GetAllOpendAsync(x => !string.IsNullOrWhiteSpace(x.UpSavePath)); return await douyinCookieService.GetOpendCookiesAsync(x => !string.IsNullOrWhiteSpace(x.UpSavePath));
} }
protected override bool IsCookieValid(DouyinCookie cookie) protected override async Task<DouyinVideoInfoResponse> FetchVideoData(DouyinCookie cookie, string cursor, DouyinFollowed followed,DouyinCollectCate cate)
{
return !string.IsNullOrWhiteSpace(cookie.Cookies)&&!string.IsNullOrWhiteSpace(cookie.UpSavePath);
}
protected override async Task<DouyinVideoInfoResponse> FetchVideoData(DouyinCookie cookie, string cursor, DouyinFollowed followed, DouyinCollectItem collectId)
{ {
return await douyinHttpClientService.SyncUpderPostVideos(count, cursor, followed.SecUid, cookie.Cookies); return await douyinHttpClientService.SyncUpderPostVideos(count, cursor, followed.SecUid, cookie.Cookies);
} }
@@ -40,10 +34,6 @@ namespace dy.net.job
return data != null && data.HasMore == 1 && cookie.UperSyncd == 0 && followed.FullSync; return data != null && data.HasMore == 1 && cookie.UperSyncd == 0 && followed.FullSync;
} }
protected override string GetNextCursor(DouyinVideoInfoResponse data)
{
return data?.MaxCursor ?? "0";
}
/// <summary> /// <summary>
/// 关注用户特殊处理文件夹存储路径,用户可自定义保存路径 /// 关注用户特殊处理文件夹存储路径,用户可自定义保存路径
@@ -53,7 +43,7 @@ namespace dy.net.job
/// <param name="followed"></param> /// <param name="followed"></param>
/// <param name="config"></param> /// <param name="config"></param>
/// <returns></returns> /// <returns></returns>
protected override string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed, DouyinCollectItem collectItem) protected override string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed,DouyinCollectCate cate)
{ {
#region 使UP主名称作为文件夹名称使 #region 使UP主名称作为文件夹名称使
// 1. 优先获取有效的作者名称(遵循原有优先级:followed.UperName > item.Author.Nickname > 默认值) // 1. 优先获取有效的作者名称(遵循原有优先级:followed.UperName > item.Author.Nickname > 默认值)
@@ -170,11 +160,6 @@ namespace dy.net.job
} }
} }
protected override string GetAuthorAvatarBasePath(DouyinCookie cookie)
{
return Path.Combine(cookie.UpSavePath, "author");
}
protected override async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount, DouyinFollowed followed) protected override async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount, DouyinFollowed followed)
{ {
if (syncCount > 0) if (syncCount > 0)
+347
View File
@@ -0,0 +1,347 @@
using dy.net.model.dto;
using dy.net.model.entity;
using dy.net.model.response;
using dy.net.service;
using Quartz;
namespace dy.net.job
{
[DisallowConcurrentExecution] // 禁止并发执行,确保同一时间只有一个实例在运行
public class DouyinFollowsAndCollnectsSyncJob : IJob
{
/// <summary>
/// 抖音收藏夹服务
/// </summary>
protected readonly DouyinCollectCateService douyinCollectCateService;
/// <summary>
/// 抖音Cookie服务,用于获取和管理用户Cookie
/// </summary>
protected readonly DouyinCookieService _dyCookieService;
/// <summary>
/// 抖音HTTP客户端服务,用于发送HTTP请求
/// </summary>
protected readonly DouyinHttpClientService _douyinService;
/// <summary>
/// 抖音关注服务,用于管理关注数据
/// </summary>
protected readonly DouyinFollowService _followService;
protected readonly DouyinCommonService douyinCommonService;
public DouyinFollowsAndCollnectsSyncJob(DouyinCookieService dyCookieService, DouyinHttpClientService douyinService, DouyinFollowService followService, DouyinCommonService douyinCommonService, DouyinCollectCateService douyinCollectCateService)
{
_dyCookieService = dyCookieService;
_douyinService = douyinService;
_followService = followService;
this.douyinCommonService = douyinCommonService;
this.douyinCollectCateService = douyinCollectCateService;
}
// 第一步:定义常量和枚举(建议放在单独的常量类中,此处为内联演示)
private const string DEFAULT_FOLLOW_COUNT = "20";
private const int INVALID_COOKIE_STATUS_CODE = 8;
private const string LOG_TAG_COLLECT = "收藏夹同步";
private const string LOG_TAG_MIX = "合集列表同步";
private const string LOG_TAG_SERIES = "短剧列表同步";
private const string LOG_TAG_FOLLOW = "关注列表同步";
/// <summary>
/// 抖音数据同步任务执行方法(优化版)
/// </summary>
/// <param name="context">任务执行上下文</param>
public async Task Execute(IJobExecutionContext context)
{
// 1. 获取有效Cookie列表,提前做空值判断
var cookies = await _dyCookieService.GetOpendCookiesAsync();
if (cookies == null || !cookies.Any())
{
Serilog.Log.Debug("当前无可用的抖音Cookie,同步任务跳过");
return;
}
// 2. 获取应用配置
AppConfig conf = douyinCommonService.GetConfig();
// 3. 遍历每个Cookie,执行各类数据同步
foreach (var ck in cookies)
{
Serilog.Log.Information($"开始处理Cookie用户:[{ck.UserName}]ID{ck.Id}");
// 3.1 同步自定义收藏夹(UseCollectFolder=true时)
if (ck.UseCollectFolder)
{
await SyncCollectDataAsync(
cookie: ck,
cateType: VideoTypeEnum.dy_custom_collect,
dataFetchFunc: async (offset) => await _douyinService.SyncCollectFolderList(ck.Cookies, offset),
entityConvertFunc: (collectItem) => new DouyinCollectCate
{
CookieId = ck.Id,
CateType = VideoTypeEnum.dy_custom_collect,
Name = collectItem.CollectsName,
Sync = false,
XId = collectItem.CollectsId
},
hasMoreFunc: (data) => data?.HasMore ?? false,
cursorFunc: (data) => data?.Cursor.ToString() ?? "0",
dataListFunc: (data) => data?.CollectsList,
logTag: LOG_TAG_COLLECT
);
}
// 3.2 同步收藏夹合集(DownMix=true时)
if (ck.DownMix)
{
await SyncCollectDataAsync(
cookie: ck,
cateType: VideoTypeEnum.dy_mix,
dataFetchFunc: async (offset) => await _douyinService.SyncMixList(ck.Cookies, offset),
entityConvertFunc: (mixItem) => new DouyinCollectCate
{
CookieId = ck.Id,
CateType = VideoTypeEnum.dy_mix,
Name = mixItem.MixName,
Sync = false,
XId = mixItem.MixId,
CoverUrl = mixItem.CoverUrl?.UrlList?.FirstOrDefault() // 空值防护:避免CoverUrl为null
},
hasMoreFunc: (data) => (data?.HasMore ?? 0) == 1,
cursorFunc: (data) => data?.Cursor.ToString() ?? "0",
dataListFunc: (data) => data?.MixInfos,
logTag: LOG_TAG_MIX
);
}
// 3.3 同步收藏夹短剧(DownSeries=true时)
if (ck.DownSeries)
{
await SyncCollectDataAsync(
cookie: ck,
cateType: VideoTypeEnum.dy_series,
dataFetchFunc: async (offset) => await _douyinService.SyncSeriesList(ck.Cookies, offset),
entityConvertFunc: (seriesItem) => new DouyinCollectCate
{
CookieId = ck.Id,
CateType = VideoTypeEnum.dy_series,
Name = seriesItem.SeriesName,
Sync = false,
XId = seriesItem.SeriesId,
CoverUrl = seriesItem.CoverImage?.ImageUrlList?.FirstOrDefault() // 空值防护:避免CoverImage为null
},
hasMoreFunc: (data) => (data?.HasMore ?? 0) == 1,
cursorFunc: (data) => data?.Cursor.ToString() ?? "0",
dataListFunc: (data) => data?.SeriesList,
logTag: LOG_TAG_SERIES
);
}
// 3.4 同步关注列表(单独处理,逻辑特殊不纳入通用方法)
await SyncFollowListAsync(ck, conf);
Serilog.Log.Debug($"完成Cookie用户:[{ck.UserName}]ID{ck.Id})的所有收藏夹信息、合集信息、关注列表信息、短剧信息同步");
}
// 4. 标记首次运行完成(仅执行一次)
await douyinCommonService.SetConfigNotFirstRunning();
}
#region
/// <summary>
/// 通用收藏类数据同步方法(泛型封装,消除冗余)
/// </summary>
/// <typeparam name="TData">抖音返回的分页数据类型</typeparam>
/// <typeparam name="TItem">抖音返回的列表项类型</typeparam>
/// <param name="cookie">当前Cookie信息</param>
/// <param name="cateType">分类类型</param>
/// <param name="dataFetchFunc">分页数据获取委托</param>
/// <param name="entityConvertFunc">抖音项转换为DouyinCollectCate的委托</param>
/// <param name="hasMoreFunc">判断是否还有更多数据的委托</param>
/// <param name="cursorFunc">获取下一页游标值的委托</param>
/// <param name="dataListFunc">从分页数据中提取列表的委托</param>
/// <param name="logTag">日志标签</param>
private async Task SyncCollectDataAsync<TData, TItem>(
DouyinCookie cookie,
VideoTypeEnum cateType,
Func<string, Task<TData>> dataFetchFunc,
Func<TItem, DouyinCollectCate> entityConvertFunc,
Func<TData, bool> hasMoreFunc,
Func<TData, string> cursorFunc,
Func<TData, List<TItem>> dataListFunc,
string logTag)
{
string offset = "0";
bool hasMore = true;
List<DouyinCollectCate> collectDataList = new List<DouyinCollectCate>();
try
{
// 1. 分页获取抖音数据
while (hasMore)
{
var pageData = await dataFetchFunc(offset);
if (pageData == null)
{
//Serilog.Log.Warning($"[{cookie.UserName}] - {logTag}:获取分页数据为空,停止分页");
break;
}
// 2. 更新分页状态
hasMore = hasMoreFunc(pageData);
offset = cursorFunc(pageData);
// 3. 提取当前页列表数据并转换
var currentPageItems = dataListFunc(pageData);
if (currentPageItems != null && currentPageItems.Any())
{
var convertItems = currentPageItems.Select(entityConvertFunc).ToList();
collectDataList.AddRange(convertItems);
Serilog.Log.Debug($"[{cookie.UserName}] - {logTag}:获取到{currentPageItems.Count}条数据,累计{collectDataList.Count}条");
}
}
// 4. 调用Sync方法同步到数据库
if (collectDataList.Any())
{
var (add, update, delete, succ) = await douyinCollectCateService.Sync(
collectDataList,
cookie.Id,
cateType);
Serilog.Log.Debug($"[{cookie.UserName}] - {logTag}:同步完成,新增{add}条,更新{update}条,删除{delete}条,是否成功:{succ}");
}
else
{
Serilog.Log.Debug($"[{cookie.UserName}] - {logTag}:无有效数据需要同步");
}
}
catch (Exception ex)
{
Serilog.Log.Error(ex, $"[{cookie.UserName}] - {logTag}:分页同步失败,异常信息:{ex.Message}");
}
}
/// <summary>
/// 关注列表单独同步方法(逻辑特殊,单独封装)
/// </summary>
/// <param name="cookie">当前Cookie信息</param>
/// <param name="config">应用配置</param>
private async Task SyncFollowListAsync(DouyinCookie cookie, AppConfig config)
{
// 1. 校验SecUserId是否有效
if (string.IsNullOrWhiteSpace(cookie.SecUserId))
{
Serilog.Log.Debug($"[{cookie.UserName}] - {LOG_TAG_FOLLOW}:未设置SecUserId,跳过");
return;
}
string offset = "0";
bool hasMore = true;
int total = 0;
List<FollowingsItem> followList = new List<FollowingsItem>();
try
{
// 2. 分页获取关注列表数据
while (hasMore)
{
var (data, error) = await FetchFollowPageDataAsync(cookie, offset);
// 3. 处理错误信息
if (error != null && error.StatusCode != 0)
{
Serilog.Log.Error($"[{cookie.UserName}] - {LOG_TAG_FOLLOW}:发生错误,状态码:{error.StatusCode},错误信息:{error.StatusMsg}");
// 无效Cookie(未登录)直接停止分页
if (error.StatusCode == INVALID_COOKIE_STATUS_CODE)
{
cookie.StatusMsg = "无效";
cookie.StatusCode = INVALID_COOKIE_STATUS_CODE;
await _dyCookieService.UpdateAsync(cookie);
break;
}
}
// 4. 处理有效数据
if (data != null)
{
// 5. 更新MyUserId(若未设置)
if (string.IsNullOrWhiteSpace(cookie.MyUserId))
{
cookie.MyUserId = data.MySelfUserId;
await _dyCookieService.UpdateAsync(cookie);
}
// 6. 更新分页状态和数据
total = data.Total;
hasMore = data.HasMore;
offset = data.Offset.ToString();
if (data.Followings != null && data.Followings.Any())
{
followList.AddRange(data.Followings);
Serilog.Log.Debug($"[{cookie.UserName}] - {LOG_TAG_FOLLOW}:获取到{data.Followings.Count}条关注数据,累计{followList.Count}条");
}
// 7. 非首次运行时,仅同步第一页数据
if (!config.IsFirstRunning)
{
hasMore = false;
//Serilog.Log.Debug($"[{cookie.UserName}] - {LOG_TAG_FOLLOW}:非首次运行,停止分页获取");
}
}
else
{
Serilog.Log.Warning($"[{cookie.UserName}] - {LOG_TAG_FOLLOW}:获取分页数据为空,停止分页");
break;
}
}
// 8. 同步关注列表到数据库
if (followList.Any())
{
var (add, update, succ) = await _followService.Sync(followList, cookie);
Serilog.Log.Information($"[{cookie.UserName}] - {LOG_TAG_FOLLOW}:同步完成,新增{add}条,更新{update}条,是否成功:{succ},总关注数:{total}");
}
else
{
Serilog.Log.Debug($"[{cookie.UserName}] - {LOG_TAG_FOLLOW}:无有效关注数据需要同步");
}
}
catch (Exception ex)
{
Serilog.Log.Error(ex, $"[{cookie.UserName}] - {LOG_TAG_FOLLOW}:同步失败,异常信息:{ex.Message}");
}
}
/// <summary>
/// 分页获取关注列表数据(封装回调逻辑,简化代码)
/// </summary>
/// <param name="cookie">当前Cookie信息</param>
/// <param name="offset">分页偏移量</param>
private async Task<(DouyinFollowInfoResponse Data, FollowErrorDto Error)> FetchFollowPageDataAsync(DouyinCookie cookie, string offset)
{
DouyinFollowInfoResponse resultData = null;
FollowErrorDto resultError = null;
resultData = await _douyinService.SyncMyFollows(
DEFAULT_FOLLOW_COUNT,
offset,
cookie.SecUserId,
cookie.Cookies,
async (err) =>
{
resultError = err;
// 更新Cookie状态(异步不阻塞)
cookie.StatusMsg = err.StatusCode == INVALID_COOKIE_STATUS_CODE ? "无效" : "正常";
cookie.StatusCode = err.StatusCode;
await _dyCookieService.UpdateAsync(cookie);
});
// 若原方法是回调式无返回值,需调整封装逻辑
return (resultData, resultError);
}
#endregion
}
}
+31
View File
@@ -0,0 +1,31 @@
using dy.net.model.dto;
using dy.net.model.entity;
using dy.net.model.response;
using dy.net.service;
using dy.net.utils;
namespace dy.net.job
{
public class DouyinMixSyncJob : DouyinBasicSyncJob
{
public DouyinMixSyncJob(DouyinCookieService douyinCookieService, DouyinHttpClientService douyinHttpClientService, DouyinVideoService douyinVideoService, DouyinCommonService douyinCommonService, DouyinFollowService douyinFollowService, DouyinMergeVideoService douyinMergeVideoService, DouyinCollectCateService douyinCollectCateService) : base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService, douyinCollectCateService)
{
}
protected override VideoTypeEnum VideoType => VideoTypeEnum.dy_favorite;
protected override async Task<DouyinVideoInfoResponse> FetchVideoData(DouyinCookie cookie, string cursor,DouyinFollowed followed,DouyinCollectCate cate)
{
return await douyinHttpClientService.SyncFavoriteVideos(count, cursor, cookie.SecUserId, cookie.Cookies);
}
protected override bool ShouldContinueSync(DouyinCookie cookie, DouyinVideoInfoResponse data, DouyinFollowed followed=null)
{
return data != null && data.HasMore == 1 && cookie.FavHasSyncd == 0;
}
}
}
+29
View File
@@ -0,0 +1,29 @@
using dy.net.model.dto;
using dy.net.model.entity;
using dy.net.model.response;
using dy.net.service;
using dy.net.utils;
using System.Net;
namespace dy.net.job
{
public class DouyinSeriesSyncJob : DouyinBasicSyncJob
{
public DouyinSeriesSyncJob(DouyinCookieService douyinCookieService, DouyinHttpClientService douyinHttpClientService, DouyinVideoService douyinVideoService, DouyinCommonService douyinCommonService, DouyinFollowService douyinFollowService, DouyinMergeVideoService douyinMergeVideoService, DouyinCollectCateService douyinCollectCateService) : base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService, douyinCollectCateService)
{
}
protected override VideoTypeEnum VideoType => VideoTypeEnum.dy_series;
protected override async Task<DouyinVideoInfoResponse> FetchVideoData(DouyinCookie cookie, string cursor,DouyinFollowed followed, DouyinCollectCate cate)
{
return await douyinHttpClientService.SyncSeriesViedosByMSeriesId(cursor, count, cookie.Cookies,cate.XId);
}
protected override bool ShouldContinueSync(DouyinCookie cookie, DouyinVideoInfoResponse data, DouyinFollowed followed=null)
{
return data != null && data.HasMore == 1;
}
}
}
+1 -1
View File
@@ -59,6 +59,6 @@ namespace dy.net.model.dto
{ {
public string cookieId { get; set; } public string cookieId { get; set; }
public int cateType { get; set; } public VideoTypeEnum cateType { get; set; }
} }
} }
+18 -1
View File
@@ -23,7 +23,24 @@ namespace dy.net.model.dto
/// 图文视频,无用了,但是不能删... /// 图文视频,无用了,但是不能删...
/// </summary> /// </summary>
[Description("图片视频")] [Description("图片视频")]
ImageVideo = 4 ImageVideo = 4,
/// <summary>
/// 自定义收藏夹
/// </summary>
[Description("收藏短剧")]
dy_custom_collect = 5,
/// <summary>
/// 收藏合集
/// </summary>
[Description("收藏合集")]
dy_mix = 6,
/// <summary>
/// 收藏短剧
/// </summary>
[Description("收藏短剧")]
dy_series = 7,
} }
} }
+9 -4
View File
@@ -1,4 +1,5 @@
using SqlSugar; using dy.net.model.dto;
using SqlSugar;
namespace dy.net.model.entity namespace dy.net.model.entity
{ {
@@ -26,7 +27,7 @@ namespace dy.net.model.entity
[SugarColumn(Length = 500, IsNullable = false)] [SugarColumn(Length = 500, IsNullable = false)]
public string Name { get; set; } public string Name { get; set; }
/// <summary> /// <summary>
/// 收藏夹Id /// 收藏夹Id、合集Id、短剧Id
/// </summary> /// </summary>
[SugarColumn(Length =60,IsNullable =false)] [SugarColumn(Length =60,IsNullable =false)]
public string XId { get; set; } public string XId { get; set; }
@@ -45,12 +46,16 @@ namespace dy.net.model.entity
public bool Sync { get; set; } public bool Sync { get; set; }
/// <summary> /// <summary>
/// 收藏夹类型(0:自定义收藏夹,1:合集,2:短剧) /// 收藏夹类型(5:自定义收藏夹,6:合集,7:短剧)
/// </summary> /// </summary>
public int CateType { get; set; } public VideoTypeEnum CateType { get; set; }
public DateTime CreateTime { get; set; } public DateTime CreateTime { get; set; }
public DateTime UpdateTime { get; set; } public DateTime UpdateTime { get; set; }
/// <summary>
/// 是否已完结
/// </summary>
public bool IsEnd { get; set; }
} }
} }
+6
View File
@@ -159,5 +159,11 @@ namespace dy.net.model.entity
/// 是否仅下载图片或仅下载音频 /// 是否仅下载图片或仅下载音频
/// </summary> /// </summary>
public bool OnlyImgOrOnlyMp3 { get; set; } public bool OnlyImgOrOnlyMp3 { get; set; }
/// <summary>
/// 自定义收藏夹、合集、短剧 绑定的Id
/// </summary>
[SugarColumn(Length =200,IsNullable =true)]
public string CateId { get; set; }
} }
} }
+6 -6
View File
@@ -23,7 +23,7 @@ namespace dy.net.model.response
//public object ShortDramaSingleInfo { get; set; } // 原始值为null,兼容扩展 //public object ShortDramaSingleInfo { get; set; } // 原始值为null,兼容扩展
[JsonProperty("series_infos")] [JsonProperty("series_infos")]
public List<DouyinShortDramaInfo> ShortDramaList { get; set; } public List<DouyinSeriesInfo> SeriesList { get; set; }
[JsonProperty("status_code")] [JsonProperty("status_code")]
public int ResponseStatusCode { get; set; } public int ResponseStatusCode { get; set; }
@@ -57,7 +57,7 @@ namespace dy.net.model.response
/// <summary> /// <summary>
/// 抖音短剧核心信息 /// 抖音短剧核心信息
/// </summary> /// </summary>
public class DouyinShortDramaInfo public class DouyinSeriesInfo
{ {
[JsonProperty("actors")] [JsonProperty("actors")]
public List<object> Actors { get; set; } public List<object> Actors { get; set; }
@@ -78,7 +78,7 @@ namespace dy.net.model.response
//public DouyinImageResource DarkModeIcon { get; set; } //public DouyinImageResource DarkModeIcon { get; set; }
[JsonProperty("desc")] [JsonProperty("desc")]
public string DramaDescription { get; set; } public string SeriesDescription { get; set; }
//[JsonProperty("directors")] //[JsonProperty("directors")]
//public List<object> Directors { get; set; } //public List<object> Directors { get; set; }
@@ -139,7 +139,7 @@ namespace dy.net.model.response
/// 短剧ID /// 短剧ID
/// </summary> /// </summary>
[JsonProperty("series_id")] [JsonProperty("series_id")]
public string DramaId { get; set; } public string SeriesId { get; set; }
//[JsonProperty("series_interactive")] //[JsonProperty("series_interactive")]
//public DouyinDramaInteractiveConfig InteractiveConfig { get; set; } //public DouyinDramaInteractiveConfig InteractiveConfig { get; set; }
@@ -147,13 +147,13 @@ namespace dy.net.model.response
/// 短剧名称 /// 短剧名称
/// </summary> /// </summary>
[JsonProperty("series_name")] [JsonProperty("series_name")]
public string DramaName { get; set; } public string SeriesName { get; set; }
//[JsonProperty("series_paid_type_list")] //[JsonProperty("series_paid_type_list")]
//public object PaidTypeList { get; set; } // 原始值为null,兼容扩展 //public object PaidTypeList { get; set; } // 原始值为null,兼容扩展
[JsonProperty("series_price")] [JsonProperty("series_price")]
public int DramaPrice { get; set; } public int SeriesPrice { get; set; }
//[JsonProperty("series_rank_info")] //[JsonProperty("series_rank_info")]
//public DouyinDramaRankInfo RankInfo { get; set; } //public DouyinDramaRankInfo RankInfo { get; set; }
+42 -38
View File
@@ -63,7 +63,7 @@ namespace dy.net.repository
return await Db.Updateable(cates).ExecuteCommandAsync() > 0; return await Db.Updateable(cates).ExecuteCommandAsync() > 0;
} }
internal async Task<(int add, int update, int delete, bool succ)> Sync(List<DouyinCollectCate> cates, string ckId,int cateType) internal async Task<(int add, int update, int delete, bool succ)> Sync(List<DouyinCollectCate> cates, string ckId, VideoTypeEnum cateType)
{ {
// 初始化返回结果 // 初始化返回结果
int addCount = 0; int addCount = 0;
@@ -83,56 +83,60 @@ namespace dy.net.repository
cates = new List<DouyinCollectCate>(); // 空列表则执行删除所有操作 cates = new List<DouyinCollectCate>(); // 空列表则执行删除所有操作
} }
// 2. 开启SqlSugar事务 // 2. 开启SqlSugar事务
await Db.Ado.BeginTranAsync(); await Db.Ado.BeginTranAsync();
// 3. 查询数据库中当前用户的所有分类 // 3. 查询数据库中当前用户的所有分类,并转为Dictionary优化查找效率(核心优化1)
var dbCates = await Db.Queryable<DouyinCollectCate>() var dbCates = await Db.Queryable<DouyinCollectCate>()
.Where(c => c.CookieId == ckId && c.CateType == cateType) .Where(c => c.CookieId == ckId && c.CateType == cateType)
.ToListAsync(); .ToListAsync();
// 以XId为Key构建字典,避免循环内FirstOrDefault的低效查找
var dbCateDict = dbCates.ToDictionary(c => c.XId, c => c);
// 4. 处理新增和【批量】更新逻辑
var xIds = cates.Select(c => c.XId).ToList();
// 定义批量更新的待处理集合(核心优化2)
var toUpdateCates = new List<DouyinCollectCate>();
// 4. 处理新增和更新逻辑
var cateIds = cates.Select(c => c.Id).ToList();
foreach (var cate in cates) foreach (var cate in cates)
{ {
// 用Dictionary查找,效率更高
cate.UpdateTime = DateTime.Now; // 更新时间戳 if (dbCateDict.TryGetValue(cate.XId, out var existCate))
// 检查是否已存在
if (string.IsNullOrWhiteSpace(cate.Id))
{ {
cate.CreateTime = DateTime.Now;
cate.Id = IdGener.GetLong().ToString();
// 新增
await Db.Insertable(cate).ExecuteCommandAsync();
addCount++;
}
else
{
var existCate = dbCates.FirstOrDefault(x => x.Id == cate.Id);
if (existCate == null)
{
Serilog.Log.Error($" cate {cate.Id} id 不存在");
continue;
}
// 对比字段是否有变化(根据实际业务字段调整) // 对比字段是否有变化(根据实际业务字段调整)
bool isChanged = !string.Equals(existCate.Name, cate.Name, StringComparison.Ordinal); bool isChanged = !string.Equals(existCate.Name, cate.Name, StringComparison.Ordinal);
if (isChanged) if (isChanged)
{ {
// 更新(只更新有变化的字段,或全量更新 // 补充更新信息,加入批量更新集合
await Db.Updateable(cate) cate.UpdateTime = DateTime.Now;
.IgnoreColumns(ignoreAllNullColumns: true) // 忽略空值字段 // 确保更新时的Id正确(从数据库已有记录中获取)
.Where(c => c.Id == cate.Id) cate.Id = existCate.Id;
.ExecuteCommandAsync(); toUpdateCates.Add(cate);
updateCount++;
} }
} }
else
{
// 新增逻辑保持不变
cate.CreateTime = DateTime.Now;
cate.Id = IdGener.GetLong().ToString();
await Db.Insertable(cate).ExecuteCommandAsync();
addCount++;
}
} }
// 5. 处理删除逻辑:数据库中有但传入列表中没有的分类 // 5. 执行批量更新(核心优化3:批量提交,减少数据库IO)
var deleteCates = dbCates.Where(c => !cateIds.Contains(c.Id)).ToList(); if (toUpdateCates.Any())
{
updateCount = await Db.Updateable(toUpdateCates)
.IgnoreColumns(x => new { x.Sync, x.CreateTime, x.CookieId })
.IgnoreNullColumns() // 忽略空值字段
.Where(c => c.Id == c.Id) // 批量更新默认按主键匹配,此处显式指定(也可省略,SqlSugar自动识别主键)
.ExecuteCommandAsync();
}
// 6. 处理删除逻辑:数据库中有但传入列表中没有的分类
var deleteCates = dbCates.Where(c => !xIds.Contains(c.XId)).ToList();
if (deleteCates.Any()) if (deleteCates.Any())
{ {
var deleteIds = deleteCates.Select(c => c.Id).ToList(); var deleteIds = deleteCates.Select(c => c.Id).ToList();
@@ -141,20 +145,20 @@ namespace dy.net.repository
.ExecuteCommandAsync(); .ExecuteCommandAsync();
} }
// 6. 提交事务 // 7. 提交事务
await Db.Ado.CommitTranAsync(); await Db.Ado.CommitTranAsync();
isSuccess = true; isSuccess = true;
} }
catch (Exception ex) catch (Exception ex)
{ {
// 异常时回滚事务 // 异常时回滚事务(补充:避免Db.Ado未初始化事务导致的空引用)
await Db.Ado.RollbackTranAsync(); await Db.Ado?.RollbackTranAsync();
isSuccess = false; isSuccess = false;
// throw; Serilog.Log.Error($"同步收藏夹:类型-{cateType}异常,{ex.StackTrace}");
Serilog.Log.Error($"同步收藏夹:类型-{cateType}异常,{ex.StackTrace}" );
} }
// 7. 返回最终结果 //Serilog.Log.Debug($"本次收藏夹{cateType}同步结果,新增:{addCount},更新:{updateCount},删除:{deleteCount}");
// 8. 返回最终结果
return (addCount, updateCount, deleteCount, isSuccess); return (addCount, updateCount, deleteCount, isSuccess);
} }
} }
+2 -1
View File
@@ -16,7 +16,8 @@ namespace dy.net.repository
{ {
// 1. 初始化查询:先加固定条件 Status == 1 // 1. 初始化查询:先加固定条件 Status == 1
var query = Db.Queryable<DouyinCookie>() var query = Db.Queryable<DouyinCookie>()
.Where(x => x.Status == 1); // 固定条件(必选) .Where(x => x.Status == 1)
.Where(x=>!string.IsNullOrWhiteSpace(x.Cookies)); // 固定条件(必选)
// 2. 若传入自定义条件,叠加 Where(自动 AND 组合) // 2. 若传入自定义条件,叠加 Where(自动 AND 组合)
if (whereExpression != null) if (whereExpression != null)
+1 -1
View File
@@ -84,7 +84,7 @@ namespace dy.net.repository
public async Task<List<DouyinFollowed>> GetSyncFollows(string userId) public async Task<List<DouyinFollowed>> GetSyncFollows(string userId)
{ {
return await this.Db.Queryable<DouyinFollowed>() return await this.Db.Queryable<DouyinFollowed>()
.Where(x => x.OpenSync == true).Where(x => x.mySelfId == userId) .Where(x => x.OpenSync == true).Where(x => x.mySelfId == userId).Where(x=>!string.IsNullOrWhiteSpace(x.SecUid))
.ToListAsync(); .ToListAsync();
} }
+4 -2
View File
@@ -24,6 +24,8 @@ namespace dy.net.service
{ {
return await douyinCollectCateRepository.InsertAsync(cate); return await douyinCollectCateRepository.InsertAsync(cate);
} }
/// <summary> /// <summary>
@@ -43,7 +45,7 @@ namespace dy.net.service
/// <param name="ckId"></param> /// <param name="ckId"></param>
/// <param name="cateType"></param> /// <param name="cateType"></param>
/// <returns></returns> /// <returns></returns>
public async Task<(int add, int update,int delete, bool succ)> Sync(List<DouyinCollectCate> cates, string ckId,int cateType) public async Task<(int add, int update,int delete, bool succ)> Sync(List<DouyinCollectCate> cates, string ckId,VideoTypeEnum cateType)
{ {
return await douyinCollectCateRepository.Sync(cates, ckId, cateType); return await douyinCollectCateRepository.Sync(cates, ckId, cateType);
} }
@@ -66,7 +68,7 @@ namespace dy.net.service
/// 获取需要同步的合集、短剧、收藏夹 /// 获取需要同步的合集、短剧、收藏夹
/// </summary> /// </summary>
/// <returns></returns> /// <returns></returns>
public async Task<List<DouyinCollectCate>> GetSyncCates(string cookieId,int cateType) public async Task<List<DouyinCollectCate>> GetSyncCates(string cookieId, VideoTypeEnum cateType)
{ {
return await douyinCollectCateRepository.GetListAsync(x => x.CookieId == cookieId && x.CateType == cateType); return await douyinCollectCateRepository.GetListAsync(x => x.CookieId == cookieId && x.CateType == cateType);
} }
-69
View File
@@ -108,75 +108,6 @@ namespace dy.net.service
} }
/// <summary>
/// 初始化一些数据,兼容旧版数据结构。
/// </summary>
public void UpdateCollectViedoType()
{
//更新视频类型字段-兼容老版本
string sql = @"UPDATE dy_collect_video
SET ViedoType = CASE
WHEN ViedoType = '0' THEN 0
WHEN ViedoType = '1' THEN 1
WHEN ViedoType = '2' THEN 2
WHEN ViedoType = '3' THEN 3
WHEN ViedoType = '4' THEN 4
ELSE NULL
END;";
sqlSugarClient.Ado.ExecuteCommand(sql);
//更新关注表的IsNoFollowed字段为空的数据为0--兼容老版本-新加的字段
string followUpdateSql = @"Update dy_follow SET IsNoFollowed=0 WHERE IsNoFollowed is NULL";
sqlSugarClient.Ado.ExecuteCommand(followUpdateSql);
//更新图片视频的合并状态
string updateIsMergeVideoSql = @"UPDATE dy_collect_video SET IsMergeVideo = 1 WHERE IsMergeVideo IS NULL and ViedoType='4';";
sqlSugarClient.Ado.ExecuteCommand(updateIsMergeVideoSql);
//更新非图片视频的合并状态
string updateNoIsMergeVideoSql = @"UPDATE dy_collect_video SET IsMergeVideo = 0 WHERE IsMergeVideo IS NULL and ViedoType<>'4';";
sqlSugarClient.Ado.ExecuteCommand(updateNoIsMergeVideoSql);
//强制开启去重
sqlSugarClient.Updateable<AppConfig>().SetColumns(x => new AppConfig { AutoDistinct = true }).Where(x => !string.IsNullOrWhiteSpace(x.Id)).ExecuteCommand();
//重新根据保存路径更新图片视频的类型为喜欢,收藏或关注
var collectViedos = sqlSugarClient.Queryable<DouyinVideo>().Where(x => x.ViedoType == VideoTypeEnum.ImageVideo).ToList();
if (collectViedos != null && collectViedos.Any())
{
var cookies = sqlSugarClient.Queryable<DouyinCookie>().ToList();
collectViedos.ForEach(x =>
{
var ck = cookies.FirstOrDefault(c => c.Id == x.CookieId);
if (ck != null)
{
var savePath = x.VideoSavePath;
if (savePath != null)
{
if (savePath.StartsWith(ck.SavePath))
{
x.ViedoType = VideoTypeEnum.dy_collects;
}
else if (savePath.StartsWith(ck.UpSavePath))
{
x.ViedoType = VideoTypeEnum.dy_follows;
}
else if (savePath.StartsWith(ck.FavSavePath))
{
x.ViedoType = VideoTypeEnum.dy_favorite;
}
else
{
x.ViedoType = VideoTypeEnum.dy_collects;
}
}
}
});
//只更新图片视频的类型
sqlSugarClient.Updateable(collectViedos).UpdateColumns(x => new DouyinVideo { ViedoType = x.ViedoType }).IgnoreColumns(true).ExecuteCommand();
}
}
/// <summary> /// <summary>
/// 重置所有Cookie的同步状态为0 /// 重置所有Cookie的同步状态为0
/// </summary> /// </summary>
+1 -1
View File
@@ -18,7 +18,7 @@ namespace dy.net.service
_cookieRepository = cookieRepository; _cookieRepository = cookieRepository;
} }
public Task<List<DouyinCookie>> GetAllOpendAsync(Expression<Func<DouyinCookie, bool>> whereExpression = null) public Task<List<DouyinCookie>> GetOpendCookiesAsync(Expression<Func<DouyinCookie, bool>> whereExpression = null)
{ {
return _cookieRepository.GetAllCookies(whereExpression); return _cookieRepository.GetAllCookies(whereExpression);
} }
+47 -23
View File
@@ -110,7 +110,10 @@ namespace dy.net.service
if (respose.IsSuccessStatusCode) if (respose.IsSuccessStatusCode)
{ {
var data = await respose.Content.ReadAsStringAsync(); var data = await respose.Content.ReadAsStringAsync();
return JsonConvert.DeserializeObject<DouyinVideoInfoResponse>(data); var model= JsonConvert.DeserializeObject<DouyinVideoInfoResponse>(data);
if (model == null)
Serilog.Log.Error($"SyncCollectVideos fail: {data}");
return model;
} }
else else
{ {
@@ -160,6 +163,8 @@ namespace dy.net.service
{ {
var data = await respose.Content.ReadAsStringAsync(); var data = await respose.Content.ReadAsStringAsync();
var model = JsonConvert.DeserializeObject<DouyinCollectListResponse>(data); var model = JsonConvert.DeserializeObject<DouyinCollectListResponse>(data);
if (model == null)
Serilog.Log.Error($"SyncCollectFolderList fail: {data}");
return model; return model;
} }
else else
@@ -181,15 +186,15 @@ namespace dy.net.service
/// <param name="cursor"></param> /// <param name="cursor"></param>
/// <param name="count"></param> /// <param name="count"></param>
/// <param name="cookie"></param> /// <param name="cookie"></param>
/// <param name="collect"></param> /// <param name="collectsId"></param>
/// <returns></returns> /// <returns></returns>
/// <exception cref="ArgumentException"></exception> /// <exception cref="ArgumentException"></exception>
public async Task<DouyinVideoInfoResponse> SyncCollectVideosByCollectId(string cursor, string count, string cookie, DouyinCollectItem collect) public async Task<DouyinVideoInfoResponse> SyncCollectVideosByCollectId(string cursor, string count, string cookie, string collectsId)
{ {
#region #region
if (collect is null || string.IsNullOrWhiteSpace(collect.CollectsId)) if (string.IsNullOrWhiteSpace(collectsId))
{ {
throw new ArgumentException($"“{nameof(collect)}”不能为 null 或空。", nameof(collect)); throw new ArgumentException($"“{nameof(collectsId)}”不能为 null 或空。", nameof(collectsId));
} }
if (string.IsNullOrWhiteSpace(cursor)) if (string.IsNullOrWhiteSpace(cursor))
{ {
@@ -217,7 +222,7 @@ namespace dy.net.service
{ {
requestParameters["cursor"] = cursor; requestParameters["cursor"] = cursor;
requestParameters["count"] = count; requestParameters["count"] = count;
requestParameters["collects_id"] = collect.CollectsId; requestParameters["collects_id"] = collectsId;
} }
var respose = await GetHttpResponseMessage(HttpMethod.Post, requestUrl, requestParameters, refererValue, cookie); var respose = await GetHttpResponseMessage(HttpMethod.Post, requestUrl, requestParameters, refererValue, cookie);
@@ -225,7 +230,10 @@ namespace dy.net.service
if (respose.IsSuccessStatusCode) if (respose.IsSuccessStatusCode)
{ {
var data = await respose.Content.ReadAsStringAsync(); var data = await respose.Content.ReadAsStringAsync();
return JsonConvert.DeserializeObject<DouyinVideoInfoResponse>(data); var model= JsonConvert.DeserializeObject<DouyinVideoInfoResponse>(data);
if (model == null)
Serilog.Log.Error($"SyncCollectVideosByCollectId fail: {data}");
return model;
} }
else else
{ {
@@ -266,7 +274,7 @@ namespace dy.net.service
var requestParameters = DouyinRequestParamManager.DouyinMixListParams; var requestParameters = DouyinRequestParamManager.DouyinMixListParams;
{ {
// 添加动态参数 // 添加动态参数
requestParameters["count"] = "20"; requestParameters["count"] = "100";
requestParameters["cursor"] = cursor; //页码 requestParameters["cursor"] = cursor; //页码
} }
@@ -275,6 +283,8 @@ namespace dy.net.service
{ {
var data = await respose.Content.ReadAsStringAsync(); var data = await respose.Content.ReadAsStringAsync();
var model = JsonConvert.DeserializeObject<DouyinMixListResponse>(data); var model = JsonConvert.DeserializeObject<DouyinMixListResponse>(data);
if (model == null)
Serilog.Log.Error($"SyncMixList fail: {data}");
return model; return model;
} }
else else
@@ -300,12 +310,12 @@ namespace dy.net.service
/// <param name="mix"></param> /// <param name="mix"></param>
/// <returns></returns> /// <returns></returns>
/// <exception cref="ArgumentException"></exception> /// <exception cref="ArgumentException"></exception>
public async Task<DouyinVideoInfoResponse> SyncMixViedosByMixId(string cursor, string count, string cookie, DouyinMixInfo mix) public async Task<DouyinVideoInfoResponse> SyncMixViedosByMixId(string cursor, string count, string cookie, string mixId)
{ {
#region #region
if (mix is null ||string.IsNullOrWhiteSpace(mix.MixId)) if (string.IsNullOrWhiteSpace(mixId))
{ {
throw new ArgumentException($"“{nameof(mix)}”不能为 null 或空。", nameof(mix)); throw new ArgumentException($"“{nameof(mixId)}”不能为 null 或空。", nameof(mixId));
} }
if (string.IsNullOrWhiteSpace(cursor)) if (string.IsNullOrWhiteSpace(cursor))
{ {
@@ -333,7 +343,7 @@ namespace dy.net.service
{ {
requestParameters["cursor"] = cursor; requestParameters["cursor"] = cursor;
requestParameters["count"] = count; requestParameters["count"] = count;
requestParameters["mix_id"] = mix.MixId; requestParameters["mix_id"] = mixId;
} }
var respose = await GetHttpResponseMessage(HttpMethod.Post, requestUrl, requestParameters, refererValue, cookie); var respose = await GetHttpResponseMessage(HttpMethod.Post, requestUrl, requestParameters, refererValue, cookie);
@@ -341,7 +351,10 @@ namespace dy.net.service
if (respose.IsSuccessStatusCode) if (respose.IsSuccessStatusCode)
{ {
var data = await respose.Content.ReadAsStringAsync(); var data = await respose.Content.ReadAsStringAsync();
return JsonConvert.DeserializeObject<DouyinVideoInfoResponse>(data); var model = JsonConvert.DeserializeObject<DouyinVideoInfoResponse>(data);
if (model == null)
Serilog.Log.Error($"SyncMixViedosByMixId fail: {data}");
return model;
} }
else else
{ {
@@ -365,7 +378,7 @@ namespace dy.net.service
/// <param name="cookie"></param> /// <param name="cookie"></param>
/// <param name="cursor"></param> /// <param name="cursor"></param>
/// <returns></returns> /// <returns></returns>
public async Task<DouyinSeriesListResponse> SyncShortList(string cookie, string cursor) public async Task<DouyinSeriesListResponse> SyncSeriesList(string cookie, string cursor)
{ {
if (string.IsNullOrWhiteSpace(cookie)) if (string.IsNullOrWhiteSpace(cookie))
{ {
@@ -381,7 +394,7 @@ namespace dy.net.service
var requestParameters = DouyinRequestParamManager.DouyinSeriesListParams; var requestParameters = DouyinRequestParamManager.DouyinSeriesListParams;
{ {
// 添加动态参数 // 添加动态参数
requestParameters["count"] = "20"; requestParameters["count"] = "100";
requestParameters["cursor"] = cursor; //页码 requestParameters["cursor"] = cursor; //页码
} }
@@ -390,6 +403,8 @@ namespace dy.net.service
{ {
var data = await respose.Content.ReadAsStringAsync(); var data = await respose.Content.ReadAsStringAsync();
var model = JsonConvert.DeserializeObject<DouyinSeriesListResponse>(data); var model = JsonConvert.DeserializeObject<DouyinSeriesListResponse>(data);
if (model == null)
Serilog.Log.Error($"SyncShortList fail: {data}");
return model; return model;
} }
else else
@@ -412,15 +427,15 @@ namespace dy.net.service
/// <param name="cursor"></param> /// <param name="cursor"></param>
/// <param name="count"></param> /// <param name="count"></param>
/// <param name="cookie"></param> /// <param name="cookie"></param>
/// <param name="series"></param> /// <param name="seriesId"></param>
/// <returns></returns> /// <returns></returns>
/// <exception cref="ArgumentException"></exception> /// <exception cref="ArgumentException"></exception>
public async Task<DouyinVideoInfoResponse> SyncSeriesViedosByMSeriesId(string cursor, string count, string cookie, DouyinShortDramaInfo series) public async Task<DouyinVideoInfoResponse> SyncSeriesViedosByMSeriesId(string cursor, string count, string cookie, string seriesId)
{ {
#region #region
if (series is null || string.IsNullOrWhiteSpace(series.DramaId)) if (string.IsNullOrWhiteSpace(seriesId))
{ {
throw new ArgumentException($"“{nameof(series)}”不能为 null 或空。", nameof(series)); throw new ArgumentException($"“{nameof(seriesId)}”不能为 null 或空。", nameof(seriesId));
} }
if (string.IsNullOrWhiteSpace(cursor)) if (string.IsNullOrWhiteSpace(cursor))
{ {
@@ -448,7 +463,7 @@ namespace dy.net.service
{ {
requestParameters["cursor"] = cursor; requestParameters["cursor"] = cursor;
requestParameters["count"] = count; requestParameters["count"] = count;
requestParameters["series_id"] = series.DramaId; requestParameters["series_id"] = seriesId;
} }
var respose = await GetHttpResponseMessage(HttpMethod.Post, requestUrl, requestParameters, refererValue, cookie); var respose = await GetHttpResponseMessage(HttpMethod.Post, requestUrl, requestParameters, refererValue, cookie);
@@ -456,7 +471,10 @@ namespace dy.net.service
if (respose.IsSuccessStatusCode) if (respose.IsSuccessStatusCode)
{ {
var data = await respose.Content.ReadAsStringAsync(); var data = await respose.Content.ReadAsStringAsync();
return JsonConvert.DeserializeObject<DouyinVideoInfoResponse>(data); var model= JsonConvert.DeserializeObject<DouyinVideoInfoResponse>(data);
if (model == null)
Serilog.Log.Error($"SyncSeriesViedosByMSeriesId fail: {data}");
return model;
} }
else else
{ {
@@ -525,7 +543,10 @@ namespace dy.net.service
if (respose.IsSuccessStatusCode) if (respose.IsSuccessStatusCode)
{ {
var data = await respose.Content.ReadAsStringAsync(); var data = await respose.Content.ReadAsStringAsync();
return JsonConvert.DeserializeObject<DouyinVideoInfoResponse>(data); var model= JsonConvert.DeserializeObject<DouyinVideoInfoResponse>(data);
if (model == null)
Serilog.Log.Error($"SyncFavoriteVideos fail: {data}");
return model;
} }
else else
{ {
@@ -594,7 +615,10 @@ namespace dy.net.service
if (respose.IsSuccessStatusCode) if (respose.IsSuccessStatusCode)
{ {
var data = await respose.Content.ReadAsStringAsync(); var data = await respose.Content.ReadAsStringAsync();
return JsonConvert.DeserializeObject<DouyinVideoInfoResponse>(data); var model= JsonConvert.DeserializeObject<DouyinVideoInfoResponse>(data);
if (model == null)
Serilog.Log.Error($"SyncUpderPostVideos fail: {data}");
return model;
} }
else else
{ {
+2 -2
View File
@@ -57,7 +57,7 @@ namespace dy.net.service
{ {
"follow_user", "follow_user",
new JobConfig( new JobConfig(
typeof(DouyinFollowedUsersSyncJob), typeof(DouyinFollowsAndCollnectsSyncJob),
"dy.job.key.follow_user", "dy.job.key.follow_user",
"dy.trigger.key.follow_user", "dy.trigger.key.follow_user",
"抖音关注同步任务") "抖音关注同步任务")
@@ -65,7 +65,7 @@ namespace dy.net.service
{ {
"follow_user_once", "follow_user_once",
new JobConfig( new JobConfig(
typeof(DouyinFollowedUsersSyncJob), typeof(DouyinFollowsAndCollnectsSyncJob),
"dy.job.key.follow_user_once", "dy.job.key.follow_user_once",
"dy.trigger.key.follow_user_once", "dy.trigger.key.follow_user_once",
"抖音关注同步任务(单次执行)") "抖音关注同步任务(单次执行)")