1、去重

2、分享
3、视频播放
4、批量删除,重新下载
5、自定义标题(仅博主视频)
6、授权页面去掉博主添加,新增关注列表
7、图文视频合成优化
8、其他优化
This commit is contained in:
jianzhichu
2025-11-30 00:42:25 +08:00
parent 63b51d211e
commit e6e9ff3cc3
94 changed files with 8155 additions and 1544 deletions
+28 -35
View File
@@ -6,47 +6,45 @@ using System;
namespace dy.net.job
{
public class DouyinCollectSyncJob : DouyinBaseSyncJob
public class DouyinCollectSyncJob : DouyinBasicSyncJob
{
public DouyinCollectSyncJob(
DouyinCookieService dyCookieService,
DouyinHttpClientService dyHttpClientService,
DouyinVideoService dyCollectVideoService,
DouyinCommonService commonService,IServiceProvider serviceProvider,IWebHostEnvironment webHostEnvironment)
: base(dyCookieService, dyHttpClientService, dyCollectVideoService, commonService, serviceProvider,webHostEnvironment) { }
DouyinCookieService douyinCookieService,
DouyinHttpClientService douyinHttpClientService,
DouyinVideoService douyinVideoService,
DouyinCommonService douyinCommonService,DouyinFollowService douyinFollowService,DouyinMergeVideoService douyinMergeVideoService)
: base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService,douyinFollowService, douyinMergeVideoService) { }
protected override string JobType => "collect";
protected override string JobType => SystemStaticUtil.DY_COLLECTS;
protected override async Task BeforeProcessCookies()
{
var now = DateTime.Now;
if (now.Hour == 1 && now.Minute < 30)
{
_commonService.UpdateAllCookieSyncedToZero();
douyinCommonService.UpdateAllCookieSyncedToZero();
//顺手清理下日志文件
LogFileCleaner.CleanOldLogFiles(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "logs"), 1);
await Task.Delay(200);
}
}
protected override async Task<List<DouyinUserCookie>> GetValidCookies()
protected override async Task<List<DouyinCookie>> GetValidCookies()
{
var cookies = await _dyCookieService.GetAllCookies();
return cookies.Where(c => !string.IsNullOrWhiteSpace(c.SavePath)).ToList();
return await douyinCookieService.GetAllOpendAsync(x => !string.IsNullOrWhiteSpace(x.SavePath));
}
protected override bool IsCookieValid(DouyinUserCookie cookie)
protected override bool IsCookieValid(DouyinCookie cookie)
{
return !string.IsNullOrWhiteSpace(cookie.Cookies) && cookie.Cookies.Length >= 1000 &&
!string.IsNullOrWhiteSpace(cookie.SavePath);
return !string.IsNullOrWhiteSpace(cookie.Cookies)&& !string.IsNullOrWhiteSpace(cookie.SavePath);
}
protected override async Task<DouyinVideoInfo> FetchVideoData(DouyinUserCookie cookie, string cursor)
protected override async Task<DouyinVideoInfo> FetchVideoData(DouyinCookie cookie, string cursor,string uperUid)
{
return await _douyinService.SyncCollectVideos(cursor, count, cookie.Cookies);
return await douyinHttpClientService.SyncCollectVideos(cursor, count, cookie.Cookies);
}
protected override bool ShouldContinueSync(DouyinUserCookie cookie, DouyinVideoInfo data)
protected override bool ShouldContinueSync(DouyinCookie cookie, DouyinVideoInfo data, DouyinFollowed followed = null)
{
return data != null && data.HasMore == 1 && cookie.CollHasSyncd == 0;
}
@@ -56,32 +54,18 @@ namespace dy.net.job
return data?.Cursor ?? "0";
}
protected override string CreateSaveFolder(DouyinUserCookie cookie, Aweme item, string tag1, string tag2, AppConfig config)
{
var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : TikTokFileNameHelper.SanitizePath(tag1);
var folder = Path.Combine(cookie.SavePath, safeTag1, $"{TikTokFileNameHelper.SanitizePath(item.Desc)}@{item.AwemeId}");
if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
return folder;
}
protected override string GetVideoFileName(DouyinUserCookie cookie, Aweme item)
{
var bitRate=item.Video.BitRate.FirstOrDefault();
return $"{item.AwemeId}.{bitRate.Format}";
}
protected override string GetAuthorAvatarBasePath(DouyinUserCookie cookie)
protected override string GetAuthorAvatarBasePath(DouyinCookie cookie)
{
return Path.Combine(cookie.SavePath, "author");
}
protected override async Task HandleSyncCompletion(DouyinUserCookie cookie, int syncCount)
protected override async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount)
{
if (syncCount > 0)
{
Serilog.Log.Debug($"{JobType}-Cookie-[{cookie.UserName}],本次同步成功{syncCount}条视频");
cookie.CollHasSyncd = 1;
await _dyCookieService.UpdateAsync(cookie);
await douyinCookieService.UpdateAsync(cookie);
}
else
{
@@ -90,12 +74,21 @@ namespace dy.net.job
}
protected override VideoEntityDifferences GetVideoEntityDifferences(DouyinUserCookie cookie, Aweme item)
protected override VideoEntityDifferences GetVideoEntityDifferences(DouyinCookie cookie, Aweme item)
{
return new VideoEntityDifferences
{
VideoType = VideoTypeEnum.Collect,
};
}
protected override string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed)
{
var (tag1, _, _) = GetVideoTags(item);
var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : DouyinFileNameHelper.SanitizePath(tag1);
var folder = Path.Combine(cookie.SavePath, safeTag1, $"{DouyinFileNameHelper.SanitizePath(item.Desc)}@{item.AwemeId}");
if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
return folder;
}
}
}
+26 -31
View File
@@ -8,35 +8,34 @@ using System.Threading.Tasks;
namespace dy.net.job
{
public class DouyinFavoritSyncJob : DouyinBaseSyncJob
public class DouyinFavoritSyncJob : DouyinBasicSyncJob
{
public DouyinFavoritSyncJob(
DouyinCookieService dyCookieService,
DouyinHttpClientService dyHttpClientService,
DouyinVideoService dyCollectVideoService,
DouyinCommonService commonService, IServiceProvider serviceProvider, IWebHostEnvironment webHostEnvironment)
: base(dyCookieService, dyHttpClientService, dyCollectVideoService, commonService, serviceProvider, webHostEnvironment) { }
DouyinCookieService douyinCookieService,
DouyinHttpClientService douyinHttpClientService,
DouyinVideoService douyinVideoService,
DouyinCommonService douyinCommonService,DouyinFollowService douyinFollowService,DouyinMergeVideoService douyinMergeVideoService)
: base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService) { }
protected override string JobType => "favorite";
protected override async Task<List<DouyinUserCookie>> GetValidCookies()
protected override string JobType => SystemStaticUtil.DY_FAVORITES;
protected override async Task<List<DouyinCookie>> GetValidCookies()
{
var cookies = await _dyCookieService.GetAllCookies();
return cookies.Where(c => !string.IsNullOrWhiteSpace(c.FavSavePath)).ToList();
return await douyinCookieService.GetAllOpendAsync(x=> !string.IsNullOrWhiteSpace(x.FavSavePath));
}
protected override bool IsCookieValid(DouyinUserCookie cookie)
protected override bool IsCookieValid(DouyinCookie cookie)
{
return !string.IsNullOrWhiteSpace(cookie.Cookies) && cookie.Cookies.Length >= 1000 &&
!string.IsNullOrWhiteSpace(cookie.FavSavePath) &&
!string.IsNullOrWhiteSpace(cookie.SecUserId) && cookie.SecUserId.Length >= 10;
}
protected override async Task<DouyinVideoInfo> FetchVideoData(DouyinUserCookie cookie, string cursor)
protected override async Task<DouyinVideoInfo> FetchVideoData(DouyinCookie cookie, string cursor,string uperUid)
{
return await _douyinService.SyncFavoriteVideos(count, cursor, cookie.SecUserId, cookie.Cookies);
return await douyinHttpClientService.SyncFavoriteVideos(count, cursor, cookie.SecUserId, cookie.Cookies);
}
protected override bool ShouldContinueSync(DouyinUserCookie cookie, DouyinVideoInfo data)
protected override bool ShouldContinueSync(DouyinCookie cookie, DouyinVideoInfo data, DouyinFollowed followed=null)
{
return data != null && data.HasMore == 1 && cookie.FavHasSyncd == 0;
}
@@ -46,32 +45,19 @@ namespace dy.net.job
return data?.MaxCursor ?? "0";
}
protected override string CreateSaveFolder(DouyinUserCookie cookie, Aweme item, string tag1, string tag2, AppConfig config)
{
var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : TikTokFileNameHelper.SanitizePath(tag1);
var folder = Path.Combine(cookie.FavSavePath, safeTag1, $"{TikTokFileNameHelper.SanitizePath(item.Desc)}@{item.AwemeId}");
if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
return folder;
}
protected override string GetVideoFileName(DouyinUserCookie cookie, Aweme item)
{
var bitRate = item.Video.BitRate.FirstOrDefault();
return $"{item.AwemeId}.{bitRate.Format}";
}
protected override string GetAuthorAvatarBasePath(DouyinUserCookie cookie)
protected override string GetAuthorAvatarBasePath(DouyinCookie cookie)
{
return Path.Combine(cookie.FavSavePath, "author");
}
protected override async Task HandleSyncCompletion(DouyinUserCookie cookie, int syncCount)
protected override async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount)
{
if (syncCount > 0)
{
Serilog.Log.Debug($"{JobType}-Cookie-[{cookie.UserName}],本次同步成功{syncCount}条视频");
cookie.FavHasSyncd = 1;
await _dyCookieService.UpdateAsync(cookie);
await douyinCookieService.UpdateAsync(cookie);
}
else
{
@@ -79,12 +65,21 @@ namespace dy.net.job
}
}
protected override VideoEntityDifferences GetVideoEntityDifferences(DouyinUserCookie cookie, Aweme item)
protected override VideoEntityDifferences GetVideoEntityDifferences(DouyinCookie cookie, Aweme item)
{
return new VideoEntityDifferences
{
VideoType = VideoTypeEnum.Favorite
};
}
protected override string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed)
{
var (tag1, _, _) = GetVideoTags(item);
var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : DouyinFileNameHelper.SanitizePath(tag1);
var folder = Path.Combine(cookie.FavSavePath, safeTag1, $"{DouyinFileNameHelper.SanitizePath(item.Desc)}@{item.AwemeId}");
if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
return folder;
}
}
}
-149
View File
@@ -1,149 +0,0 @@
using dy.net.dto;
using dy.net.model;
using dy.net.service;
using dy.net.utils;
using Newtonsoft.Json;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
namespace dy.net.job
{
public class DouyinUperPostSyncJob : DouyinBaseSyncJob
{
public DouyinUperPostSyncJob(
DouyinCookieService dyCookieService,
DouyinHttpClientService dyHttpClientService,
DouyinVideoService dyCollectVideoService,
DouyinCommonService commonService, IServiceProvider serviceProvider, IWebHostEnvironment webHostEnvironment)
: base(dyCookieService, dyHttpClientService, dyCollectVideoService, commonService, serviceProvider, webHostEnvironment) { }
protected override string JobType => "dyuploder";
protected override async Task<List<DouyinUserCookie>> GetValidCookies()
{
var cookies = await _dyCookieService.GetAllCookies();
return cookies.Where(x => !string.IsNullOrWhiteSpace(x.UpSecUserIds) && !string.IsNullOrWhiteSpace(x.UpSavePath)).ToList();
}
protected override bool IsCookieValid(DouyinUserCookie cookie)
{
return !string.IsNullOrWhiteSpace(cookie.Cookies) && !string.IsNullOrWhiteSpace(cookie.UpSavePath) && !string.IsNullOrWhiteSpace(cookie.UpSecUserIds);
}
protected override async Task<DouyinVideoInfo> FetchVideoData(DouyinUserCookie cookie, string cursor)
{
// 简化处理:假设只同步第一个UP主
var ups = JsonConvert.DeserializeObject<List<DouyinUpSecUserIdDto>>(cookie.UpSecUserIds);
var firstUpId = ups?.FirstOrDefault()?.uid;
if (string.IsNullOrEmpty(firstUpId)) return null;
return await _douyinService.SyncUpderPostVideos(count, cursor, firstUpId, cookie.Cookies);
}
protected override bool ShouldContinueSync(DouyinUserCookie cookie, DouyinVideoInfo data)
{
return data != null && data.HasMore == 1 && cookie.UperSyncd == 0;
}
protected override string GetNextCursor(DouyinVideoInfo data)
{
return data?.MaxCursor ?? "0";
}
protected override string CreateSaveFolder(DouyinUserCookie cookie, Aweme item, string tag1, string tag2,AppConfig config)
{
// UP主视频通常按作者名创建文件夹
var authorName = string.IsNullOrWhiteSpace(item.Author?.Nickname) ? "UnknownAuthor" : TikTokFileNameHelper.SanitizePath(item.Author.Nickname);
var folder = Path.Combine(cookie.UpSavePath, authorName);
if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
if(config.UperSaveTogether)
{
return folder;
}
else
{
var sampleName = TikTokFileNameHelper.GenerateFileName(item.Desc, item.AwemeId);
var (existingName, _) = _douyinVideoService.GetUperLastViedoFileName(item.Author.Uid, sampleName).Result;
var fileNameFolder = string.IsNullOrWhiteSpace(existingName) ? sampleName : existingName;
return Path.Combine(folder, fileNameFolder);
}
}
protected override string GetVideoFileName(DouyinUserCookie cookie, Aweme item)
{
var bitRate = item.Video.BitRate.FirstOrDefault();
var config = _commonService.GetConfig();
var fileName = string.Empty;
if (config?.UperUseViedoTitle ?? false)
{
var sampleName = TikTokFileNameHelper.GenerateFileName(item.Desc, item.AwemeId);
var (existingName, _) = _douyinVideoService.GetUperLastViedoFileName(item.Author.Uid, sampleName).Result;
fileName= string.IsNullOrWhiteSpace(existingName) ? $"{sampleName}.{bitRate.Format}" : $"{existingName}.{bitRate.Format}";
}
else
{
fileName = $"{item.AwemeId}.{bitRate.Format}";
}
return fileName;
}
/// <summary>
///
/// </summary>
/// <param name="cookie"></param>
/// <param name="item"></param>
/// <param name="config"></param>
/// <param name="imageType"></param>
/// <returns></returns>
protected override string GetNfoFileName(DouyinUserCookie cookie, Aweme item, AppConfig config, string imageType)
{
if (config.UperSaveTogether)
{
var videoFileName = GetVideoFileName(cookie, item);
return $"{Path.GetFileNameWithoutExtension(videoFileName)}{imageType}";
}
else
{
return base.GetNfoFileName(cookie,item,config,imageType);
}
}
protected override string GetAuthorAvatarBasePath(DouyinUserCookie cookie)
{
return Path.Combine(cookie.UpSavePath, "author");
}
protected override async Task HandleSyncCompletion(DouyinUserCookie cookie, int syncCount)
{
if (syncCount > 0)
{
Serilog.Log.Debug($"{JobType}-Cookie-[{cookie.UserName}],本次同步成功{syncCount}条视频");
cookie.UperSyncd = 1;
await _dyCookieService.UpdateAsync(cookie);
}
else
{
Serilog.Log.Debug($"{JobType}-Cookie-[{cookie.UserName}],本次没有查询到新的视频");
}
}
protected override VideoEntityDifferences GetVideoEntityDifferences(DouyinUserCookie cookie, Aweme item)
{
var config = _commonService.GetConfig();
string simplifiedTitle = string.Empty;
if (config?.UperUseViedoTitle ?? false)
{
simplifiedTitle = TikTokFileNameHelper.GenerateFileName(item.Desc, item.AwemeId);
}
return new VideoEntityDifferences
{
VideoType = VideoTypeEnum.UperPost,
VideoTitleSimplify = simplifiedTitle
};
}
}
}
@@ -1,5 +1,4 @@
using ClockSnowFlake;
using dy.image;
using dy.net.dto;
using dy.net.model;
using dy.net.service;
@@ -22,44 +21,44 @@ namespace dy.net.job
/// 提供了通用的同步逻辑,如Cookie处理、视频下载、数据存储等
/// </summary>
[DisallowConcurrentExecution] // 禁止并发执行,确保同一时间只有一个实例在运行
public abstract class DouyinBaseSyncJob : IJob
public abstract class DouyinBasicSyncJob : IJob
{
#region
/// <summary>
/// 抖音Cookie服务,用于获取和管理用户Cookie
/// </summary>
protected readonly DouyinCookieService _dyCookieService;
protected readonly DouyinCookieService douyinCookieService;
/// <summary>
/// 抖音HTTP客户端服务,用于发送HTTP请求
/// </summary>
protected readonly DouyinHttpClientService _douyinService;
protected readonly DouyinHttpClientService douyinHttpClientService;
/// <summary>
/// 抖音视频服务,用于视频信息的数据库操作
/// </summary>
protected readonly DouyinVideoService _douyinVideoService;
protected readonly DouyinVideoService douyinVideoService;
/// <summary>
/// 抖音通用服务,用于获取应用配置等
/// </summary>
protected readonly DouyinCommonService _commonService;
protected readonly DouyinCommonService douyinCommonService;
/// <summary>
/// 抖音关注列表
/// </summary>
private readonly DouyinFollowService douyinFollowService;
/// <summary>
/// 图文合成视频
/// </summary>
private readonly DouyinMergeVideoService douyinMergeVideoService;
/// <summary>
/// 随机数生成器,用于生成随机延迟,模拟人类操作
/// </summary>
protected readonly Random _random = new Random();
/// <summary>
/// 服务提供器,用于获取其他服务实例
/// </summary>
protected readonly IServiceProvider _serviceProvider;
/// <summary>
/// Web主机环境,用于获取应用程序路径等信息
/// </summary>
protected readonly IWebHostEnvironment _environment;
/// <summary>
/// 每页请求的视频数量,可通过配置文件修改
@@ -90,28 +89,28 @@ namespace dy.net.job
#region
/// <summary>
/// 初始化 <see cref="DouyinBaseSyncJob"/> 类的新实例
/// 初始化 <see cref="DouyinBasicSyncJob"/> 类的新实例
/// </summary>
/// <param name="dyCookieService">抖音Cookie服务</param>
/// <param name="dyHttpClientService">抖音HTTP客户端服务</param>
/// <param name="dyCollectVideoService">抖音视频服务</param>
/// <param name="commonService">抖音通用服务</param>
/// <param name="serviceProvider">服务提供器</param>
/// <param name="webHostEnvironment">Web主机环境</param>
protected DouyinBaseSyncJob(
DouyinCookieService dyCookieService,
DouyinHttpClientService dyHttpClientService,
DouyinVideoService dyCollectVideoService,
DouyinCommonService commonService,
IServiceProvider serviceProvider,
IWebHostEnvironment webHostEnvironment)
/// <param name="douyinCookieService">抖音Cookie服务</param>
/// <param name="douyinHttpClientService">抖音HTTP客户端服务</param>
/// <param name="douyinVideoService">抖音视频服务</param>
/// <param name="douyinCommonService">抖音通用服务</param>
/// <param name="douyinFollowService">抖音关注的</param>
/// <param name="douyinMergeVideoService">视频合成</param>
protected DouyinBasicSyncJob(
DouyinCookieService douyinCookieService,
DouyinHttpClientService douyinHttpClientService,
DouyinVideoService douyinVideoService,
DouyinCommonService douyinCommonService,
DouyinFollowService douyinFollowService,
DouyinMergeVideoService douyinMergeVideoService)
{
_dyCookieService = dyCookieService ?? throw new ArgumentNullException(nameof(dyCookieService));
_douyinService = dyHttpClientService ?? throw new ArgumentNullException(nameof(dyHttpClientService));
_douyinVideoService = dyCollectVideoService ?? throw new ArgumentNullException(nameof(dyCollectVideoService));
_commonService = commonService ?? throw new ArgumentNullException(nameof(commonService));
_serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
_environment = webHostEnvironment ?? throw new ArgumentNullException(nameof(webHostEnvironment));
this.douyinCookieService = douyinCookieService ?? throw new ArgumentNullException(nameof(douyinCookieService));
this.douyinHttpClientService = douyinHttpClientService ?? throw new ArgumentNullException(nameof(douyinHttpClientService));
this.douyinVideoService = douyinVideoService ?? throw new ArgumentNullException(nameof(douyinVideoService));
this.douyinCommonService = douyinCommonService ?? throw new ArgumentNullException(nameof(douyinCommonService));
this.douyinFollowService = douyinFollowService;
this.douyinMergeVideoService = douyinMergeVideoService;
}
#endregion
@@ -127,15 +126,13 @@ namespace dy.net.job
public async Task Execute(IJobExecutionContext context)
{
// 1. 获取应用配置
var config = _commonService.GetConfig();
var config = douyinCommonService.GetConfig();
if (config == null)
{
Log.Debug($"{JobType}-未获取到系统配置,任务终止!!!");
return;
}
//将配置项打印日志
//config.PrintAsTable();
// 2. 从配置中获取每页请求数量
if (config.BatchCount > 0)
@@ -169,8 +166,7 @@ namespace dy.net.job
#region
/// <summary>
/// 在处理Cookie之前执行的预处理操作
/// 子类可以重写此方法来实现特定的预处理逻辑
/// 在处理Cookie之前执行的预处理操作-AOP
/// </summary>
/// <returns>一个表示异步操作的任务</returns>
protected virtual Task BeforeProcessCookies() => Task.CompletedTask;
@@ -180,15 +176,26 @@ namespace dy.net.job
/// 子类必须实现此方法,根据具体任务类型筛选有效的Cookie
/// </summary>
/// <returns>有效的Cookie列表</returns>
protected abstract Task<List<DouyinUserCookie>> GetValidCookies();
protected abstract Task<List<DouyinCookie>> GetValidCookies();
/// <summary>
/// 获取关注列表
/// </summary>
/// <returns></returns>
protected virtual Task<List<DouyinFollowed>> GetFollows()
{
return Task.FromResult(new List<DouyinFollowed>());
}
/// <summary>
/// 检查指定的Cookie是否有效
/// 子类必须实现此方法,提供具体的Cookie有效性检查逻辑
/// </summary>
/// <param name="cookie">要检查的Cookie</param>
/// <returns>如果Cookie有效,则为true;否则为false</returns>
protected abstract bool IsCookieValid(DouyinUserCookie cookie);
protected virtual bool IsCookieValid(DouyinCookie cookie)
{
return !string.IsNullOrWhiteSpace(cookie.Cookies);
}
/// <summary>
/// 根据Cookie和游标获取视频数据
@@ -196,8 +203,9 @@ namespace dy.net.job
/// </summary>
/// <param name="cookie">用户Cookie</param>
/// <param name="cursor">分页游标,用于获取下一页数据</param>
/// <param name="uperUid">关注的人</param>
/// <returns>视频信息对象,包含视频列表和分页信息</returns>
protected abstract Task<DouyinVideoInfo> FetchVideoData(DouyinUserCookie cookie, string cursor);
protected abstract Task<DouyinVideoInfo> FetchVideoData(DouyinCookie cookie, string cursor, string uperUid = "");
/// <summary>
/// 判断是否应该继续同步下一页数据
@@ -205,8 +213,9 @@ namespace dy.net.job
/// </summary>
/// <param name="cookie">用户Cookie</param>
/// <param name="data">当前获取到的视频数据</param>
/// <param name="followed">关注博主</param>
/// <returns>如果应该继续同步,则为true;否则为false</returns>
protected abstract bool ShouldContinueSync(DouyinUserCookie cookie, DouyinVideoInfo data);
protected abstract bool ShouldContinueSync(DouyinCookie cookie, DouyinVideoInfo data, DouyinFollowed followed);
/// <summary>
/// 获取下一页数据的游标
@@ -222,28 +231,31 @@ namespace dy.net.job
/// </summary>
/// <param name="cookie">用户Cookie</param>
/// <param name="item">视频信息</param>
/// <param name="tag1">视频标签1</param>
/// <param name="tag2">视频标签2</param>
/// <param name="followed">关注用户</param>
/// <param name="config">应用配置</param>
/// <returns>创建的视频保存文件夹路径</returns>
protected abstract string CreateSaveFolder(DouyinUserCookie cookie, Aweme item, string tag1, string tag2, AppConfig config);
protected abstract string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed);
/// <summary>
/// 获取视频文件名
/// 子类必须实现此方法,根据具体的命名规则生成文件名
/// 获取视频文件名,默认就用id作文件名
/// </summary>
/// <param name="cookie">用户Cookie</param>
/// <param name="item">视频信息</param>
/// <param name="config">配置信息</param>
/// <returns>生成的视频文件名</returns>
protected abstract string GetVideoFileName(DouyinUserCookie cookie, Aweme item);
protected virtual string GetVideoFileName(DouyinCookie cookie, Aweme item, AppConfig config)
{
if (item.Video != null && item.Video.BitRate != null)
return $"{item.AwemeId}.{item.Video.BitRate.FirstOrDefault().Format}";
return $"{item.AwemeId}.mp4";
}
/// <summary>
/// 获取作者头像保存的基础路径
/// 子类必须实现此方法,指定头像的存储位置
/// </summary>
/// <param name="cookie">用户Cookie</param>
/// <returns>作者头像保存的基础路径</returns>
protected abstract string GetAuthorAvatarBasePath(DouyinUserCookie cookie);
protected abstract string GetAuthorAvatarBasePath(DouyinCookie cookie);
/// <summary>
/// 处理同步完成后的操作
@@ -252,7 +264,7 @@ namespace dy.net.job
/// <param name="cookie">用户Cookie</param>
/// <param name="syncCount">本次同步成功的视频数量</param>
/// <returns>一个表示异步操作的任务</returns>
protected abstract Task HandleSyncCompletion(DouyinUserCookie cookie, int syncCount);
protected abstract Task HandleSyncCompletion(DouyinCookie cookie, int syncCount);
/// <summary>
/// 获取视频实体的差异信息
@@ -261,7 +273,7 @@ namespace dy.net.job
/// <param name="cookie">用户Cookie</param>
/// <param name="item">视频信息</param>
/// <returns>视频实体的差异信息,包含视频类型和简化标题</returns>
protected abstract VideoEntityDifferences GetVideoEntityDifferences(DouyinUserCookie cookie, Aweme item);
protected abstract VideoEntityDifferences GetVideoEntityDifferences(DouyinCookie cookie, Aweme item);
/// <summary>
/// 获取NFO文件中的图片(如海报)文件名
@@ -272,7 +284,7 @@ namespace dy.net.job
/// <param name="config">应用配置</param>
/// <param name="fileName">原文件名称(如poster.jpg</param>
/// <returns>封面图片的文件名</returns>
protected virtual string GetNfoFileName(DouyinUserCookie cookie, Aweme item, AppConfig config, string fileName)
protected virtual string GetNfoFileName(DouyinCookie cookie, Aweme item, AppConfig config, string fileName)
{
return fileName;
}
@@ -284,7 +296,7 @@ namespace dy.net.job
/// <param name="cookie">用户Cookie</param>
/// <param name="config">应用配置</param>
/// <returns>一个表示异步操作的任务</returns>
protected async Task ProcessSyncUserCookie(DouyinUserCookie cookie, AppConfig config)
protected async Task ProcessSyncUserCookie(DouyinCookie cookie, AppConfig config)
{
try
{
@@ -294,44 +306,51 @@ namespace dy.net.job
Log.Debug($"{JobType}-Cookie[{cookie.UserName}]无效,任务终止!!!");
return;
}
Log.Debug($"{JobType}- Cookie-[{cookie.UserName}]开始同步...");
int syncCount = 0; // 本次同步成功的视频数量
string cursor = "0"; // 初始游标
bool hasMore = true; // 是否还有更多数据
// 循环获取视频数据
while (hasMore)
//up主上传视频特殊处理
if (JobType == SystemStaticUtil.DY_FOLLOWEDS)
{
// 获取视频数
var data = await FetchVideoData(cookie, cursor);
if (data == null)
int syncCount = 0; // 本次同步成功的视频数
string cursor = "0"; // 初始游标
bool hasMore = true; // 是否还有更多数据
var follows = await douyinFollowService.GetSyncFollows(cookie.MyUserId);
//var ups = JsonConvert.DeserializeObject<List<DouyinUpSecUserIdDto>>(cookie.UpSecUserIds);
var firstUp = follows?.Where(x => !string.IsNullOrWhiteSpace(x.SecUid)).FirstOrDefault();
if (firstUp == null)
{
Log.Debug($"{JobType}-Cookie[{cookie.UserName}]读取数据失败!!!");
break;
return;
}
if (string.IsNullOrWhiteSpace(firstUp.SecUid))
{
Log.Debug($"{JobType}-Cookie[{firstUp.UperName}]无效,没有sec_userid,任务终止!!!");
return;
}
// 判断是否还有更多数据
hasMore = ShouldContinueSync(cookie, data);
// 获取下一页游标
cursor = GetNextCursor(data);
// 如果没有视频数据,退出循环
if (data.AwemeList == null || !data.AwemeList.Any())
break;
foreach (var item in follows)
{
cursor = "0"; // 初始游标
await GetViedos(cookie, config, syncCount, cursor, hasMore, item);
hasMore = true;
// 处理同步完成后的操作
await HandleSyncCompletion(cookie, syncCount);
}
}
else
{
int syncCount = 0; // 本次同步成功的视频数量
string cursor = "0"; // 初始游标
bool hasMore = true; // 是否还有更多数据
(syncCount, cursor, hasMore) = await GetViedos(cookie, config, syncCount, cursor, hasMore);
// 处理视频列表
var videos = await ProcessVideoList(cookie, data, config);
// 保存视频信息到数据库
syncCount += await SaveVideos(videos);
// 随机延迟,模拟人类操作,避免请求过快
await Task.Delay(_random.Next(5, 10) * 1000);
// 处理同步完成后的操作
await HandleSyncCompletion(cookie, syncCount);
}
// 处理同步完成后的操作
await HandleSyncCompletion(cookie, syncCount);
}
catch (Exception ex)
{
@@ -339,6 +358,40 @@ namespace dy.net.job
}
}
private async Task<(int syncCount, string cursor, bool hasMore)> GetViedos(DouyinCookie cookie, AppConfig config, int syncCount, string cursor, bool hasMore, DouyinFollowed followed = null)
{
// 循环获取视频数据
while (hasMore)
{
// 获取视频数据
var data = await FetchVideoData(cookie, cursor, followed == null ? "" : followed?.SecUid);
if (data == null)
{
Log.Debug($"{JobType}-Cookie[{cookie.UserName}]读取数据失败!!!");
break;
}
// 判断是否还有更多数据
hasMore = ShouldContinueSync(cookie, data, followed);
// 获取下一页游标
cursor = GetNextCursor(data);
// 如果没有视频数据,退出循环
if (data.AwemeList == null || !data.AwemeList.Any())
break;
// 处理视频列表
var videos = await ProcessVideoList(cookie, data, config, followed);
// 保存视频信息到数据库
syncCount += await SaveVideos(videos);
// 随机延迟,模拟人类操作,避免请求过快
await Task.Delay(_random.Next(5, 10) * 1000);
}
return (syncCount, cursor, hasMore);
}
/// <summary>
/// 处理视频列表
/// 遍历视频列表,分别处理每个视频和图片集
@@ -346,23 +399,45 @@ namespace dy.net.job
/// <param name="cookie">用户Cookie</param>
/// <param name="data">视频信息对象</param>
/// <param name="config">应用配置</param>
/// <param name="followed">关注的</param>
/// <returns>处理后的视频实体列表</returns>
protected async Task<List<DouyinVideo>> ProcessVideoList(DouyinUserCookie cookie, DouyinVideoInfo data, AppConfig config)
protected async Task<List<DouyinVideo>> ProcessVideoList(DouyinCookie cookie, DouyinVideoInfo data, AppConfig config, DouyinFollowed followed = null)
{
var videos = new List<DouyinVideo>();
foreach (var item in data.AwemeList)
{
//去重,检查视频是否已存在
if (config.AutoDistinct)
{
var exitVideo = await douyinVideoService.GetByAwemeId(item.AwemeId);
if (exitVideo != null)
{
if (File.Exists(exitVideo.VideoSavePath))
{
continue;// 已存在则跳过
}
}
}
var uper = await douyinFollowService.GetByUperId(item.AuthorUserId.ToString(), cookie.MyUserId);
if (uper != null)
{
if (uper.FullSync)
{
followed ??= uper;
}
}
// 处理单个视频
var video = await ProcessSingleVideo(cookie, item, data, config);
var video = await ProcessSingleVideo(cookie, item, data, config, followed);
if (video != null)
videos.Add(video);
// 如果配置了下载图片视频,则处理图片集并合成视频
if (_downImageVideo)
{
if(config.DownImageVideo||config.DownMp3||config.DownImage)
if (config.DownImageVideo || config.DownMp3 || config.DownImage)
{
var mergevideo = await ProcessImageSetAndMergeToVideo(cookie, item, data, config);
var mergevideo = await ProcessImageSetAndMergeToVideo(cookie, item, data, config, followed);
if (mergevideo != null)
videos.Add(mergevideo);
}
@@ -379,8 +454,9 @@ namespace dy.net.job
/// <param name="item">视频信息</param>
/// <param name="data">视频信息对象</param>
/// <param name="config">应用配置</param>
/// <param name="followed">关注</param>
/// <returns>处理后的视频实体,如果处理失败则为null</returns>
protected async Task<DouyinVideo> ProcessSingleVideo(DouyinUserCookie cookie, Aweme item, DouyinVideoInfo data, AppConfig config)
protected async Task<DouyinVideo> ProcessSingleVideo(DouyinCookie cookie, Aweme item, DouyinVideoInfo data, AppConfig config, DouyinFollowed followed = null)
{
// 检查视频数据是否有效
if (!IsAwemeValid(item)) return null;
@@ -396,27 +472,27 @@ namespace dy.net.job
// 获取视频标签
var (tag1, tag2, tag3) = GetVideoTags(item);
// 创建保存文件夹
var saveFolder = CreateSaveFolder(cookie, item, tag1, tag2, config);
var saveFolder = CreateSaveFolder(cookie, item, config, followed);
// 获取视频文件名
var fileName = GetVideoFileName(cookie, item);
var fileName = GetVideoFileName(cookie, item, config);
// 拼接视频保存路径
var savePath = Path.Combine(saveFolder, fileName);
// 如果文件已存在,跳过
if (File.Exists(savePath)) return null;
Log.Debug($"{JobType}-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]开始下载...");
Log.Debug($"{JobType}-视频[{DouyinFileNameHelper.SanitizePath(item.Desc)}]开始下载...");
// 随机延迟,模拟人类操作
await Task.Delay(_random.Next(1, 4) * 1000);
// 下载视频
if (!await _douyinService.DownloadAsync(videoUrl, savePath, cookie.Cookies))
if (!await douyinHttpClientService.DownloadAsync(videoUrl, savePath, cookie.Cookies))
{
Log.Error($"{JobType}-{item?.Author?.Nickname??""}-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]下载失败!!!");
Log.Error($"{JobType}-{item?.Author?.Nickname ?? ""}-视频[{DouyinFileNameHelper.SanitizePath(item.Desc)}]下载失败!!!");
return null;
}
else
{
Log.Debug($"{JobType}-{item?.Author?.Nickname ?? ""}-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]下载完成.");
Log.Debug($"{JobType}-{item?.Author?.Nickname ?? ""}-视频[{DouyinFileNameHelper.SanitizePath(item.Desc)}]下载完成.");
}
// 下载视频封面
@@ -424,7 +500,7 @@ namespace dy.net.job
// 下载作者头像
var (avatarSavePath, avatarUrl) = await DownAuthorAvatar(cookie, item);
// 生成NFO文件
await GenerateNfoFile(saveFolder, item, avatarSavePath, avatarUrl, cookie, config);
await GenerateNfoFile(saveFolder, item, avatarUrl, cookie, config);
// 创建视频实体
return CreateVideoEntity(cookie, item, v, savePath, saveFolder, tag1, tag2, tag3, avatarSavePath, avatarUrl, data);
}
@@ -437,8 +513,9 @@ namespace dy.net.job
/// <param name="item">视频信息(包含图片集)</param>
/// <param name="data">视频信息对象</param>
/// <param name="config">应用配置</param>
/// <param name="followed">应用配置</param>
/// <returns>合成后的视频实体,如果处理失败则为null</returns>
protected async Task<DouyinVideo> ProcessImageSetAndMergeToVideo(DouyinUserCookie cookie, Aweme item, DouyinVideoInfo data, AppConfig config)
protected async Task<DouyinVideo> ProcessImageSetAndMergeToVideo(DouyinCookie cookie, Aweme item, DouyinVideoInfo data, AppConfig config, DouyinFollowed followed)
{
try
{
@@ -455,26 +532,28 @@ namespace dy.net.job
return null;
}
// 检查图片保存路径是否配置
if (string.IsNullOrWhiteSpace(cookie.ImgSavePath))
{
Log.Error($"{JobType}-图文视频同步-没有配置图片存储路径,任务终止!!!");
return null;
}
// 获取图片合成视频服务
var imageService = _serviceProvider.GetService<ImageMergeToVideoService>();
if (imageService == null)
{
Log.Error($"{JobType} -图文视频同步异常,请联系作者!!!");
return null;
}
// 创建图片保存文件夹
var fileNamefolder = Path.Combine(cookie.ImgSavePath, TikTokFileNameHelper.GenerateFileName(item.Desc, item.AwemeId));
var fileNamefolder = string.Empty;
if (config.ImageViedoSaveAlone)
{
// 检查图片保存路径是否配置
if (string.IsNullOrWhiteSpace(cookie.ImgSavePath))
{
Log.Error($"{JobType}-图文视频同步-没有配置图片存储路径,任务终止!!!");
return null;
}
fileNamefolder = Path.Combine(cookie.ImgSavePath, DouyinFileNameHelper.GenerateFileName(item.Desc, item.AwemeId));
}
else
{
fileNamefolder = CreateSaveFolder(cookie, item, config, followed);
}
if (!Directory.Exists(fileNamefolder)) Directory.CreateDirectory(fileNamefolder);
var fileName = GetVideoFileName(cookie, item, config);
// 合成视频的保存路径
var savePath = Path.Combine(fileNamefolder, $"{item.AwemeId}.mp4");
var savePath = Path.Combine(fileNamefolder, fileName);
// 如果文件已存在,返回null
if (File.Exists(savePath)) return null;
@@ -482,6 +561,10 @@ namespace dy.net.job
// 获取音乐URL
var mp3Url = item.Music?.PlayUrl?.UrlList?.FirstOrDefault();
var firstImage = item.Images.FirstOrDefault();
int height = firstImage.Height;
int width = firstImage.Width;
// 准备合成视频的请求参数
var reqParams = new MediaMergeRequest
{
@@ -490,15 +573,15 @@ namespace dy.net.job
VideoFps = 30, // 视频帧率
AudioUrls = string.IsNullOrWhiteSpace(mp3Url) ? new List<string>() : new List<string> { mp3Url }, // 音频URL列表
ImageUrls = imageUrls, // 图片URL列表
VideoWidth = 1080, // 视频宽度
VideoHeight = 1920, // 视频高度
VideoWidth = width > 0 ? width : 1080, // 视频宽度
VideoHeight = height > 0 ? height : 1920, // 视频高度
};
// 执行图片合成视频操作
var mergeResult = await imageService.MergeToVideo(AppContext.BaseDirectory, reqParams, savePath, fileNamefolder,config.DownImageVideo,config.DownImage,config.DownMp3);
var mergeResult = await douyinMergeVideoService.MergeToVideo(cookie.Cookies, AppContext.BaseDirectory, reqParams, savePath, fileNamefolder, config.DownImageVideo, config.DownImage, config.DownMp3);
if (!mergeResult)
{
Log.Error($"{JobType}-图文视频-[{TikTokFileNameHelper.SanitizePath(item.Desc)}]合成失败!!!");
Log.Error($"{JobType}-图文视频-[{DouyinFileNameHelper.SanitizePath(item.Desc)}]合成失败!!!");
return null;
}
@@ -507,7 +590,7 @@ namespace dy.net.job
// 检查合成后的视频文件是否有效
if (!File.Exists(savePath) || new FileInfo(savePath).Length <= 0)
{
Log.Error($"{JobType}-图文视频-[{TikTokFileNameHelper.SanitizePath(item.Desc)}]合成失败!!!");
Log.Error($"{JobType}-图文视频-[{DouyinFileNameHelper.SanitizePath(item.Desc)}]合成失败!!!");
// 清理无效的文件和文件夹
if (Directory.Exists(fileNamefolder))
{
@@ -522,12 +605,12 @@ namespace dy.net.job
{
}
// 下载视频封面(使用第一张图片作为封面)
await DownVideoCover(imageUrls.FirstOrDefault(), fileNamefolder, cookie, item, config);
// 下载视频封面(使用第一张图片作为封面)
await DownVideoCover(imageUrls.FirstOrDefault(), fileNamefolder, cookie, item, config);
// 下载作者头像
var (avatarSavePath, avatarUrl) = await DownAuthorAvatar(cookie, item);
// 生成NFO文件
await GenerateNfoFile(fileNamefolder, item, avatarSavePath, avatarUrl, cookie, config);
await GenerateNfoFile(fileNamefolder, item, avatarUrl, cookie, config);
// 获取视频标签
var (tag1, tag2, tag3) = GetVideoTags(item);
@@ -572,7 +655,26 @@ namespace dy.net.job
if (!videos.Any()) return 0;
try
{
await _douyinVideoService.batchInsert(videos);
await douyinVideoService.batchInsert(videos);
var redowns = await douyinCommonService.GetAllRedown();
if (redowns != null && redowns.Any())
{
//找出viedos和redowns重复的
var duplicateVideos = videos.Where(v => redowns.Any(r => r.ViedoId == v.AwemeId)).ToList();
if (duplicateVideos != null && duplicateVideos.Any())
{
var duplicateVideosIds = duplicateVideos.Select(x => x.AwemeId).ToList();
var downeds = redowns.Where(x => duplicateVideosIds.Contains(x.ViedoId))?.ToList();
foreach (var item in downeds)
{
item.Status = 1;
var v = videos.FirstOrDefault(x => x.AwemeId == item.ViedoId);
Log.Debug($"{JobType}-重新下载成功:-{v.VideoTitle}");
}
await douyinCommonService.UpdateRedownStatus(downeds);
}
}
return videos.Count;
}
catch (Exception ex)
@@ -591,12 +693,11 @@ namespace dy.net.job
/// <param name="saveFolder">NFO文件的保存文件夹</param>
/// <param name="item">视频信息</param>
/// <param name="avatarSavePath">作者头像保存路径</param>
/// <param name="avatarUrl">作者头像URL</param>
/// <param name="cookie"></param>
/// <param name="config"></param>
/// <returns>一个表示异步操作的任务</returns>
protected async Task GenerateNfoFile(string saveFolder, Aweme item,
string avatarSavePath, string avatarUrl, DouyinUserCookie cookie, AppConfig config)
string avatarSavePath, DouyinCookie cookie, AppConfig config)
{
// 异步生成NFO文件,避免阻塞主线程
await Task.Run(() =>
@@ -634,7 +735,7 @@ namespace dy.net.job
/// <param name="cookie">用户Cookie</param>
/// <param name="config">应用配置</param>
/// <returns>一个表示异步操作的任务</returns>
protected async Task DownVideoCover(Aweme item, string saveFolder, DouyinUserCookie cookie, AppConfig config)
protected async Task DownVideoCover(Aweme item, string saveFolder, DouyinCookie cookie, AppConfig config)
{
var coverUrl = item.Video.Cover.UrlList?.FirstOrDefault();
if (string.IsNullOrWhiteSpace(coverUrl)) return;
@@ -648,7 +749,7 @@ namespace dy.net.job
/// <param name="cookie">用户Cookie</param>
/// <param name="item">视频信息</param>
/// <returns>一个元组,包含头像保存路径和头像URL</returns>
protected async Task<(string savePath, string url)> DownAuthorAvatar(DouyinUserCookie cookie, Aweme item)
protected async Task<(string savePath, string url)> DownAuthorAvatar(DouyinCookie cookie, Aweme item)
{
if (item.Author == null) return (null, null);
// 优先获取高清头像
@@ -663,7 +764,7 @@ namespace dy.net.job
// 如果头像文件不存在,则下载
if (!File.Exists(avatarSavePath))
{
await _douyinService.DownloadAsync(avatarUrl, avatarSavePath, cookie.Cookies);
await douyinHttpClientService.DownloadAsync(avatarUrl, avatarSavePath, cookie.Cookies);
}
return (avatarSavePath, avatarUrl);
}
@@ -686,7 +787,7 @@ namespace dy.net.job
/// </summary>
/// <param name="item">视频信息</param>
/// <returns>一个元组,包含三个级别的视频标签</returns>
private (string tag1, string tag2, string tag3) GetVideoTags(Aweme item)
protected (string tag1, string tag2, string tag3) GetVideoTags(Aweme item)
{
var tags = item.VideoTags;
return (
@@ -758,7 +859,7 @@ namespace dy.net.job
/// <param name="item">视频信息</param>
/// <param name="config">应用配置</param>
/// <returns>一个表示异步操作的任务</returns>
private async Task DownVideoCover(string coverUrl, string saveFolder, DouyinUserCookie cookie, Aweme item, AppConfig config)
private async Task DownVideoCover(string coverUrl, string saveFolder, DouyinCookie cookie, Aweme item, AppConfig config)
{
if (string.IsNullOrWhiteSpace(coverUrl)) return;
// 获取封面图片文件名
@@ -768,14 +869,7 @@ namespace dy.net.job
// 如果封面文件不存在,则下载
if (!File.Exists(coverSavePath))
{
var downRes = await _douyinService.DownloadAsync(coverUrl, coverSavePath, cookie.Cookies);
//if (downRes)
//{
// // 复制封面图片为fanart.jpg
// var fanartImgName = GetNfoFileName(cookie, item, config, "fanart.jpg");
// var copyPath = Path.Combine(saveFolder, fanartImgName);
// File.Copy(coverSavePath, copyPath, true);
//}
var downRes = await douyinHttpClientService.DownloadAsync(coverUrl, coverSavePath, cookie.Cookies);
}
}
@@ -796,7 +890,7 @@ namespace dy.net.job
/// <param name="data">视频信息对象</param>
/// <returns>创建的视频实体对象</returns>
private DouyinVideo CreateVideoEntity(
DouyinUserCookie cookie, Aweme item, VideoBitRate bitRate, string savePath, string saveFolder,
DouyinCookie cookie, Aweme item, VideoBitRate bitRate, string savePath, string saveFolder,
string tag1, string tag2, string tag3, string avatarSavePath, string avatarUrl, DouyinVideoInfo data)
{
var diffs = GetVideoEntityDifferences(cookie, item);
+81
View File
@@ -0,0 +1,81 @@
using dy.net.dto;
using dy.net.model;
using dy.net.service;
using Quartz;
using System.Collections.Generic;
namespace dy.net.job
{
[DisallowConcurrentExecution] // 禁止并发执行,确保同一时间只有一个实例在运行
public class DouyinFollowedUsersSyncJob : IJob
{
/// <summary>
/// 抖音Cookie服务,用于获取和管理用户Cookie
/// </summary>
protected readonly DouyinCookieService _dyCookieService;
/// <summary>
/// 抖音HTTP客户端服务,用于发送HTTP请求
/// </summary>
protected readonly DouyinHttpClientService _douyinService;
/// <summary>
/// 抖音关注服务,用于管理关注数据
/// </summary>
protected readonly DouyinFollowService _followService;
public DouyinFollowedUsersSyncJob(DouyinCookieService dyCookieService, DouyinHttpClientService douyinService, DouyinFollowService followService)
{
_dyCookieService = dyCookieService;
_douyinService = douyinService;
_followService = followService;
}
public async Task Execute(IJobExecutionContext context)
{
var cookies = await _dyCookieService.GetAllOpendAsync();
if (cookies != null && cookies.Any())
{
foreach (var ck in cookies)
{
string count = "20";
string offset = "0";
bool hasmore = true;
int total= 0;
List<FollowingsItem> follows = new List<FollowingsItem>();
while (hasmore)
{
var data = await _douyinService.SyncMyFollows(count, offset, ck.SecUserId, ck.Cookies);
if (data != null)
{
// 绑定我的用户ID--之前没有这个字段
if (string.IsNullOrWhiteSpace(ck.MyUserId))
{
ck.MyUserId = data.MySelfUserId;
await _dyCookieService.UpdateAsync(ck);
}
total = data.Total;
hasmore = data.HasMore;
offset = data.Offset.ToString();
if (data.Followings != null && data.Followings.Count > 0)
{
follows.AddRange(data.Followings);
}
}
}
if (follows.Count > 0)
{
await _followService.Sync(follows, ck.MyUserId);
}
Serilog.Log.Debug($"当前Cookie-[{ck.UserName}],本次同步关注列表完成,共同步关注{total}人。");
}
}
}
}
}
+203
View File
@@ -0,0 +1,203 @@
using ClockSnowFlake;
using dy.net.dto;
using dy.net.model;
using dy.net.service;
using dy.net.utils;
using Newtonsoft.Json;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
namespace dy.net.job
{
public class DouyinFollowedViedoSyncJob : DouyinBasicSyncJob
{
public DouyinFollowedViedoSyncJob(DouyinCookieService douyinCookieService, DouyinHttpClientService douyinHttpClientService, DouyinVideoService douyinVideoService, DouyinCommonService douyinCommonService, DouyinFollowService douyinFollowService, DouyinMergeVideoService douyinMergeVideoService) : base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService)
{
}
protected override string JobType => SystemStaticUtil.DY_FOLLOWEDS;
protected override async Task<List<DouyinCookie>> GetValidCookies()
{
return await douyinCookieService.GetAllOpendAsync(x => !string.IsNullOrWhiteSpace(x.UpSavePath));
}
protected override bool IsCookieValid(DouyinCookie cookie)
{
return !string.IsNullOrWhiteSpace(cookie.Cookies)&&!string.IsNullOrWhiteSpace(cookie.UpSavePath);
}
protected override async Task<DouyinVideoInfo> FetchVideoData(DouyinCookie cookie, string cursor, string uperUid = "")
{
return await douyinHttpClientService.SyncUpderPostVideos(count, cursor, uperUid, cookie.Cookies);
}
protected override bool ShouldContinueSync(DouyinCookie cookie, DouyinVideoInfo data, DouyinFollowed followed)
{
return data != null && data.HasMore == 1 && cookie.UperSyncd == 0 && followed.FullSync;
}
protected override string GetNextCursor(DouyinVideoInfo data)
{
return data?.MaxCursor ?? "0";
}
/// <summary>
/// 关注用户特殊处理文件夹存储路径,用户可自定义保存路径
/// </summary>
/// <param name="cookie"></param>
/// <param name="item"></param>
/// <param name="followed"></param>
/// <param name="config"></param>
/// <returns></returns>
protected override string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed)
{
#region 使UP主名称作为文件夹名称使
var authorName = string.IsNullOrWhiteSpace(item.Author?.Nickname) ? "UnknownAuthor" : DouyinFileNameHelper.SanitizePath(item.Author.Nickname);
var folder = Path.Combine(cookie.UpSavePath, authorName);
if (!string.IsNullOrWhiteSpace(followed.SavePath))
{
folder = Path.Combine(cookie.UpSavePath, followed.SavePath);
}
if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
#endregion
if (config.UperSaveTogether)
{
return folder;
}
else
{
var sampleName = DouyinFileNameHelper.GenerateFileName(item.Desc, item.AwemeId);
var (existingName, _) = douyinVideoService.GetUperLastViedoFileName(item.Author.Uid, sampleName).Result;
var fileNameFolder = string.IsNullOrWhiteSpace(existingName) ? sampleName : existingName;
return Path.Combine(folder, fileNameFolder);
}
}
/// <summary>
/// 关注的视频,生成文件名称
/// </summary>
/// <param name="cookie"></param>
/// <param name="item"></param>
/// <param name="config"></param>
/// <returns></returns>
protected override string GetVideoFileName(DouyinCookie cookie, Aweme item,AppConfig config)
{
string Format = "mp4";
string FileHash = "";
string Height = "";
string Width = "";
if (item.Video != null && item.Video.BitRate != null)
{
var bitrate = item.Video.BitRate.FirstOrDefault();
Format = bitrate.Format;
FileHash = bitrate.PlayAddr.FileHash;
Height = bitrate.PlayAddr.Height.ToString();
Width = bitrate.PlayAddr.Width.ToString();
}
else
{
//图片合成视频,参数要自己写。
var image = item.Images?.FirstOrDefault();
if(image != null){
FileHash = IdGener.GetGuid().ToLower().Replace("-","");//使用随机值,避免重复
Height = image.Height.ToString();
Width = image.Width.ToString();
}
}
string fileName;
if (config?.UperUseViedoTitle ?? false)//优先
{
var sampleName = DouyinFileNameHelper.GenerateFileName(item.Desc, item.AwemeId);
var (existingName, _) = douyinVideoService.GetUperLastViedoFileName(item.Author.Uid, sampleName).Result;
fileName = string.IsNullOrWhiteSpace(existingName) ? $"{sampleName}.{Format}" : $"{existingName}.{Format}";
}
else
{
if (!string.IsNullOrWhiteSpace(config.FullFollowedTitleTemplate))
{
var fullName = VideoTitleGenerator.Generate(config.FullFollowedTitleTemplate, new VideoTitleDataTemplate
{
FileHash = FileHash,
Id = item.AwemeId,
ReleaseTime = DateTimeUtil.Convert10BitTimestamp(item.CreateTime),
Resolution = $"{Width}×{Height}",
VideoTitle = DouyinFileNameHelper.GenerateFileName(item.Desc, item.AwemeId)
});
fileName= $"{DouyinFileNameHelper.SanitizePath(fullName)}.{Format}";
}
else
{
fileName = $"{item.AwemeId}.{Format}";
}
}
return fileName;
}
/// <summary>
///
/// </summary>
/// <param name="cookie"></param>
/// <param name="item"></param>
/// <param name="config"></param>
/// <param name="imageType"></param>
/// <returns></returns>
protected override string GetNfoFileName(DouyinCookie cookie, Aweme item, AppConfig config, string imageType)
{
if (config.UperSaveTogether)
{
var videoFileName = GetVideoFileName(cookie, item,config);
return $"{Path.GetFileNameWithoutExtension(videoFileName)}{imageType}";
}
else
{
return base.GetNfoFileName(cookie, item, config, imageType);
}
}
protected override string GetAuthorAvatarBasePath(DouyinCookie cookie)
{
return Path.Combine(cookie.UpSavePath, "author");
}
protected override async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount)
{
if (syncCount > 0)
{
Serilog.Log.Debug($"{JobType}-Cookie-[{cookie.UserName}],本次同步成功{syncCount}条视频");
cookie.UperSyncd = 1;
await douyinCookieService.UpdateAsync(cookie);
}
else
{
Serilog.Log.Debug($"{JobType}-Cookie-[{cookie.UserName}],本次没有查询到新的视频");
}
}
protected override VideoEntityDifferences GetVideoEntityDifferences(DouyinCookie cookie, Aweme item)
{
var config = douyinCommonService.GetConfig();
string simplifiedTitle = string.Empty;
if (config?.UperUseViedoTitle ?? false)
{
simplifiedTitle = DouyinFileNameHelper.GenerateFileName(item.Desc, item.AwemeId);
}
return new VideoEntityDifferences
{
VideoType = VideoTypeEnum.UperPost,
VideoTitleSimplify = simplifiedTitle
};
}
}
}