using ClockSnowFlake;
using dy.net.model.dto;
using dy.net.model.entity;
using dy.net.model.response;
using dy.net.service;
using dy.net.utils;
using Newtonsoft.Json;
using Quartz;
using Serilog;
namespace dy.net.job
{
///
/// 抖音数据同步任务基类
/// 所有具体的抖音同步任务(如收藏、关注、作品等)都应继承此类
/// 提供了通用的同步逻辑,如Cookie处理、视频下载、数据存储等
///
[DisallowConcurrentExecution] // 禁止并发执行,确保同一时间只有一个实例在运行
public abstract class DouyinBasicSyncJob : IJob
{
#region 受保护字段
///
/// 抖音Cookie服务,用于获取和管理用户Cookie
///
protected readonly DouyinCookieService douyinCookieService;
///
/// 抖音HTTP客户端服务,用于发送HTTP请求
///
protected readonly DouyinHttpClientService douyinHttpClientService;
///
/// 抖音视频服务,用于视频信息的数据库操作
///
protected readonly DouyinVideoService douyinVideoService;
///
/// 抖音通用服务,用于获取应用配置等
///
protected readonly DouyinCommonService douyinCommonService;
///
/// 抖音关注列表
///
private readonly DouyinFollowService douyinFollowService;
///
/// 图文合成视频
///
private readonly DouyinMergeVideoService douyinMergeVideoService;
///
/// 收藏夹、短剧、合集
///
private readonly DouyinCollectCateService douyinCollectCateService;
///
/// 随机数生成器,用于生成随机延迟,模拟人类操作
///
protected readonly Random _random = new Random();
///
/// 每页请求的视频数量.不可修改
///
protected string count = "18";
#endregion
#region 私有字段
// 类级别静态字段:映射视频类型与同步完成状态判断逻辑(复用+易维护)
//private static readonly Dictionary> _syncStatusCheckMap = new()
//{
// [VideoTypeEnum.dy_favorite] = cookie => cookie.FavHasSyncd == 1,
// [VideoTypeEnum.dy_collects] = cookie => cookie.CollHasSyncd == 1,
// [VideoTypeEnum.dy_follows] = cookie => cookie.UperSyncd == 1,
// [VideoTypeEnum.ImageVideo] = cookie => cookie.UperSyncd == 1
// && cookie.CollHasSyncd == 1
// && cookie.FavHasSyncd == 1
//};
#endregion
#region 抽象属性
///
/// 同步类型
///
protected abstract VideoTypeEnum VideoType { get; }
#endregion
#region 构造函数
///
/// 初始化 类的新实例
///
/// 抖音Cookie服务
/// 抖音HTTP客户端服务
/// 抖音视频服务
/// 抖音通用服务
/// 抖音关注的
/// 视频合成
///
protected DouyinBasicSyncJob(
DouyinCookieService douyinCookieService,
DouyinHttpClientService douyinHttpClientService,
DouyinVideoService douyinVideoService,
DouyinCommonService douyinCommonService,
DouyinFollowService douyinFollowService,
DouyinMergeVideoService douyinMergeVideoService,
DouyinCollectCateService douyinCollectCateService)
{
this.douyinCookieService = douyinCookieService ?? throw new ArgumentNullException(nameof(douyinCookieService));
this.douyinHttpClientService = douyinHttpClientService ?? throw new ArgumentNullException(nameof(douyinHttpClientService));
this.douyinVideoService = douyinVideoService ?? throw new ArgumentNullException(nameof(douyinVideoService));
this.douyinCommonService = douyinCommonService ?? throw new ArgumentNullException(nameof(douyinCommonService));
this.douyinFollowService = douyinFollowService;
this.douyinMergeVideoService = douyinMergeVideoService;
this.douyinCollectCateService = douyinCollectCateService;
}
#endregion
#region 公共方法
///
/// 执行任务的主入口点
/// 由Quartz调度器调用,负责协调整个同步流程
///
/// 作业执行上下文
/// 一个表示异步操作的任务
public async Task Execute(IJobExecutionContext context)
{
//if (VideoType.GetVideoTypeDesc() != VideoType.GetVideoTypeDesc()Enum.dy_follows)
// return;
// 获取应用配置
var config = douyinCommonService.GetConfig();
// 在处理Cookie之前执行的预处理
await BeforeProcessCookies();
// 获取所有有效的Cookie
var cookies = await GetSyncCookies();
if (cookies == null || !cookies.Any())
{
Log.Debug($"[{VideoType.GetDesc()}]-Cookie无效或已关闭同步...");
return;
}
Log.Debug($"[{VideoType.GetDesc()}]共发现{cookies.Count}个有效Cookie,同步开始...");
// 遍历每个有效的Cookie,执行同步
foreach (var cookie in cookies)
{
await ProcessSyncUserCookie(cookie, config);
}
}
#endregion
#region 受保护方法
///
/// 在处理Cookie之前执行的预处理操作-AOP
///
/// 一个表示异步操作的任务
protected virtual Task BeforeProcessCookies() => Task.CompletedTask;
///
/// 获取所有有效的Cookie
/// 子类必须实现此方法,根据具体任务类型筛选有效的Cookie
///
/// 有效的Cookie列表
protected virtual async Task> GetSyncCookies()
{
return await douyinCookieService.GetOpendCookiesAsync(x => !string.IsNullOrWhiteSpace(x.SavePath));
}
///
/// 根据Cookie和游标获取视频数据
/// 子类必须实现此方法,调用具体的API接口获取视频列表
///
/// 用户Cookie
/// 分页游标,用于获取下一页数据
/// 关注的人
/// 自定义收藏夹、合集、短剧
/// 视频信息对象,包含视频列表和分页信息
protected abstract Task FetchVideoData(DouyinCookie cookie, string cursor, DouyinFollowed followed, DouyinCollectCate cate);
/////
///// 判断是否应该继续同步下一页数据
/////
///// 用户Cookie
///// 当前获取到的视频数据
///// 关注博主
///// 如果应该继续同步,则为true;否则为false
//protected bool ShouldContinueSync(DouyinCookie cookie, DouyinVideoInfoResponse data, DouyinFollowed followed, AppConfig config) {
//}
///
/// 获取下一页数据的游标
///
/// 当前获取到的视频数据
/// 下一页数据的游标
private static string GetNextCursor(DouyinVideoInfoResponse data)
{
return data?.Cursor ?? (data?.MaxCursor ?? "0");
}
///
/// 创建视频保存文件夹
///
/// 用户Cookie
/// 视频信息
/// 关注用户
///
/// 应用配置
/// 创建的视频保存文件夹路径
protected virtual string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed, DouyinCollectCate cate)
{
var folder = Path.Combine(cookie.SavePath, DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId, true));
if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
return folder;
}
///
/// 获取视频文件名,默认就用id作文件名
///
/// 用户Cookie
/// 视频信息
/// 配置信息
///
/// 生成的视频文件名
protected virtual string GetVideoFileName(DouyinCookie cookie, Aweme item, AppConfig config, DouyinCollectCate cate)
{
if (cate != null && cate.CateType == VideoTypeEnum.dy_custom_collect)
{
if (item.Video != null && item.Video.BitRate != null)
return $"{item.AwemeId}.{item.Video.BitRate.FirstOrDefault().Format}";
return $"{item.AwemeId}.mp4";
}
else
{
if ((VideoType == VideoTypeEnum.dy_series || VideoType == VideoTypeEnum.dy_mix) && item.MixInfo?.Statis?.CurrentEpisode != null)
{
return $"{item.MixInfo.Statis.CurrentEpisode}.mp4";
}
return $"{item.AwemeId}.mp4";
}
}
///
/// 获取作者头像保存的基础路径
/// 子类必须实现此方法,指定头像的存储位置
///
/// 用户Cookie
/// 作者头像保存的基础路径
protected abstract string GetAuthorAvatarBasePath(DouyinCookie cookie);
///
/// 处理同步完成后的操作
///
/// 用户Cookie
/// 本次同步成功的视频数量
///
///
/// 一个表示异步操作的任务
protected async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount, DouyinFollowed followed = null, DouyinCollectCate cate = null)
{
var tag = cate?.Name ?? followed?.UperName ?? string.Empty;
tag = !string.IsNullOrWhiteSpace(tag) ? $"-[{tag}]" : tag;
Log.Debug($"[{VideoType.GetDesc()}][{cookie.UserName}]{tag},本次成功同步{syncCount}条视频");
if (cate != null)
{
//更新合集短剧完结状态
await douyinCollectCateService.UpdateCate2EndStatus(cate);
}
}
///
/// 获取NFO文件中的图片(如海报)文件名
///
/// 当前操作的Cookie
/// 视频信息
/// 应用配置
/// 原文件名称(如poster.jpg)
///
/// 封面图片的文件名
protected virtual string GetNfoFileName(DouyinCookie cookie, Aweme item, AppConfig config, string fileName, DouyinCollectCate cate)
{
return fileName;
}
///
/// 处理单个用户Cookie的同步逻辑
/// 负责循环获取视频数据、处理视频、保存视频信息等
///
/// 用户Cookie
/// 应用配置
/// 一个表示异步操作的任务
protected async Task ProcessSyncUserCookie(DouyinCookie cookie, AppConfig config)
{
try
{
Log.Debug($"[{VideoType.GetDesc()}][{cookie.UserName}]开始同步...");
switch (VideoType)
{
case VideoTypeEnum.dy_follows:
{
//查询关注列表开启了同步的关注
var follows = await douyinFollowService.GetSyncFollows(cookie.MyUserId);
if (follows != null && follows.Any())
{
//int totalSyncCount = 0;
foreach (var followed in follows)
{
int syncCount = 0; // 本次同步成功的视频数量
string cursor = "0";
bool hasMore = true;
(syncCount, cursor, hasMore) = await GetAndSaveViedos(cookie, config, syncCount, cursor, hasMore, followed);
await HandleSyncCompletion(cookie, syncCount, followed);
//totalSyncCount += syncCount;
//if (totalSyncCount >= config.BatchCount)
//{
// Log.Debug($"[{VideoType.GetVideoTypeDesc()}][{cookie.UserName}]:本次同步数量{totalSyncCount},已达配置上限,等下次任务继续同步");
// break;
//}
}
}
}
break;
case VideoTypeEnum.dy_favorite:
case VideoTypeEnum.dy_collects:
if (VideoType == VideoTypeEnum.dy_collects && cookie.UseCollectFolder)
{
Serilog.Log.Debug($"[{VideoType.GetDesc()}]-已开启自定义收藏夹同步...break;");
break;
}
else
{
int syncCount = 0;
string cursor = "0";
bool hasMore = true;
(syncCount, cursor, hasMore) = await GetAndSaveViedos(cookie, config, syncCount, cursor, hasMore);
await HandleSyncCompletion(cookie, syncCount);
}
break;
case VideoTypeEnum.dy_mix:
case VideoTypeEnum.dy_series:
case VideoTypeEnum.dy_custom_collect:
{
var cates = await douyinCollectCateService.GetSyncCates(cookie.Id, VideoType);
if (cates != null && cates.Any())
{
foreach (var cate in cates)
{
int syncCount = 0; // 本次同步成功的视频数量
string cursor = "0";
bool hasMore = true;
(syncCount, cursor, hasMore) = await GetAndSaveViedos(cookie, config, syncCount, cursor, hasMore, null, cate);
await HandleSyncCompletion(cookie, syncCount, null);
}
}
else
{
Serilog.Log.Debug($"[{VideoType.GetDesc()}]没有查询到已开启的对象");
}
}
break;
case VideoTypeEnum.ImageVideo:
default:
break;
}
}
catch (Exception ex)
{
Log.Error(ex, $"[{VideoType.GetDesc()}][{cookie.UserName}]同步出错!!!,{ex.StackTrace}");
}
}
private async Task<(int syncCount, string cursor, bool hasMore)> GetAndSaveViedos(DouyinCookie cookie, AppConfig config, int syncCount, string cursor, bool hasMore, DouyinFollowed followed = null, DouyinCollectCate cate = null)
{
// 循环获取视频数据
while (hasMore)
{
// 获取视频数据
var data = await FetchVideoData(cookie, cursor, followed, cate);
if (data == null || data.AwemeList == null || !data.AwemeList.Any())
{
Serilog.Log.Debug($"[{VideoType.GetDesc()}][{cookie.UserName}] 没有新的视频");
break;
}
// 判断是否还有更多数据
//hasMore = ShouldContinueSync(cookie, data, followed, config);
// 获取下一页游标
cursor = GetNextCursor(data);
hasMore = data.HasMore == 1;
// 处理视频列表
(List videos, int syncCountx) = await ProcessVideoList(syncCount, cookie, data, config, followed, cate);
if (videos != null && videos.Any())
{
// 保存视频信息到数据库
await SaveVideos(videos);
}
syncCount += syncCountx;
if (IsSyncLimitReached(cookie, config, syncCount, cate, followed))
{
break;
}
//随机等待
await Task.Delay(_random.Next(2, 10) * 1000);
}
return (syncCount, cursor, hasMore);
}
///
/// 检查是否达到同步批次上限且满足状态条件,需要终止循环
///
/// 抖音Cookie
/// 同步配置
/// 已同步数量
///
///
/// 是否需要终止循环
private bool IsSyncLimitReached(DouyinCookie cookie, AppConfig config, int syncCount, DouyinCollectCate cate, DouyinFollowed followed)
{
if (cate != null && cate.CateType != VideoTypeEnum.dy_custom_collect)
{
if (syncCount >= 15)
{
Log.Debug($"[{VideoType.GetDesc()}][{cookie.UserName}]:本次同步数量{syncCount},等下次任务继续同步");
return true;
}
}
else
{
if (syncCount >= config.BatchCount)
{
Log.Debug($"[{VideoType.GetDesc()}][{cookie.UserName}]:本次同步数量{syncCount},已达配置上限{config.BatchCount},等下次任务继续同步");
return true;
}
}
if (VideoType == VideoTypeEnum.dy_collects || VideoType == VideoTypeEnum.dy_favorite)
return config.OnlySyncNew;
return VideoType == VideoTypeEnum.dy_follows && !followed.FullSync;
//// 获取当前视频类型对应的状态判断逻辑
//if (!_syncStatusCheckMap.TryGetValue(VideoType, out var isSyncCompleted))
//{
// //Log.Debug($"[{VideoType.GetVideoTypeDesc()}]无匹配的同步状态判断规则,不终止循环");
// return false;
//}
//// 状态满足则记录日志并返回「终止循环」
//if (isSyncCompleted.Invoke(cookie))
//{
// Log.Debug($"[{VideoType.GetDesc()}][{cookie.UserName}]本次同步达到上限{config.BatchCount},停止同步!!!");
// return true;
//}
}
///
/// 遍历视频列表,分别处理每个视频和图片集
///
/// 用户Cookie
/// 视频信息对象
/// 应用配置
/// 关注的
/// 收藏夹、合集、短剧
/// 处理后的视频实体列表
protected async Task<(List videos, int currentCount)> ProcessVideoList(int syncCount1, DouyinCookie cookie, DouyinVideoInfoResponse data, AppConfig config, DouyinFollowed followed = null, DouyinCollectCate cate = null)
{
int syncCount = 0;
var videos = new List();
foreach (var item in data.AwemeList)
{
//if (item.AwemeId != "7440444948844956985")
//{
// continue;
//}
//if (!item.Desc.Contains("抖音各种隐藏功能,每个都好用到离谱#抖音#隐藏功能"))
//{
// continue;
//}
// 1. 查询数据库中是否已存在该视频(通过 AwemeId 唯一标识)
var exitVideo = await douyinVideoService.GetByAwemeId(item.AwemeId);
//判断视频是否是强制删除且不再下载的视频
var deleteVideo = await douyinCommonService.ExistDeleteVideo(item.AwemeId);
if (deleteVideo)
{
//Log.Debug($"[{VideoType.GetVideoTypeDesc()}]-视频-{item.AwemeId}-[{item.Desc}]已被标记为强制删除,跳过下载");
continue;
}
if (cate == null)
{
bool Goon = await AutoDistinct(config, exitVideo);
if (!Goon)
{
continue;
}
//如果存在,但是因为去重规则,选择的最高优先级变更了,需要删除原来的,重新下载。
if (exitVideo != null)
{
await douyinVideoService.DeleteById(exitVideo.Id);
}
var uper = await douyinFollowService.GetByUperId(item.AuthorUserId.ToString(), cookie.MyUserId);
if (uper != null && uper.FullSync)
{
followed ??= uper;
}
}
else
{
if (exitVideo != null && exitVideo.ViedoType == cate.CateType && File.Exists(exitVideo.VideoSavePath))
{
continue;
}
}
// 处理单个视频
var video = await ProcessSingleVideo(cookie, item, data, config, followed, cate);
if (video != null)
{
videos.Add(video);
syncCount++;
if (syncCount + syncCount1 >= config.BatchCount)
{
return (videos, syncCount);
}
}
else
{
//处理多个视频-组合的图文视频--类似动图。
List dynamicVideoUrls = new List();
// 当需要下载动态视频时,获取其他URL
if (config.DownDynamicVideo && item.Images != null && item.Images.Count > 0)
{
foreach (var img in item.Images)
{
if (img.DynamicVideo?.BitRate?.Count > 0)
{
foreach (var btv in img.DynamicVideo.BitRate)
{
var targetUrl = btv.PlayAddr?.UrlList?.FirstOrDefault(x => x.StartsWith("https://www.douyin.com/aweme/v1/play"));
if (targetUrl != null)
{
var height = btv.PlayAddr?.Height ?? 1920;
var width = btv.PlayAddr?.Width ?? 1080;
DouyinDynamicVideoDto info = new DouyinDynamicVideoDto
{
Path = targetUrl,
Height = height,
Width = width
};
dynamicVideoUrls.Add(info);
}
}
}
}
}
// 处理核心逻辑
if (dynamicVideoUrls.Count > 0)
{
// 处理动态视频
var dynamicVideo = await ProcessDynamicVideo(dynamicVideoUrls, cookie, item, config, followed, cate);
if (dynamicVideo != null)
{
if (!string.IsNullOrEmpty(dynamicVideo.DynamicVideos))
{
var dynamicVideos = JsonConvert.DeserializeObject>(dynamicVideo.DynamicVideos);
Log.Debug($"[{VideoType.GetDesc()}][{cookie.UserName}]-动态视频[{item.Desc}],下载成功 ,共{dynamicVideos?.Count}个视频...");
if (dynamicVideos != null && dynamicVideos.Count > 0)
{
var savePath = DouyinFileNameHelper.RemoveNumberSuffix(dynamicVideo.VideoSavePath);
//音频文件下载地址
var mp3Url = item?.Music?.PlayUrl?.UrlList?.FirstOrDefault();
var outPath = await douyinMergeVideoService.MergeMultipleVideosAsync(dynamicVideos, mp3Url, savePath, cookie.Cookies);
if (!string.IsNullOrWhiteSpace(outPath.mp4Path))
{
if (File.Exists(outPath.mp4Path))
{
dynamicVideo.VideoSavePath = outPath.mp4Path;
if (!config.KeepDynamicVideo)
{
if (File.Exists(outPath.mp3Path))
{
File.Delete(outPath.mp3Path);
}
//不保留原视频-删除
foreach (var opath in dynamicVideos)
{
if (File.Exists(opath.Path))
File.Delete(opath.Path);
}
}
}
}
}
}
videos.Add(dynamicVideo);
syncCount++;
if (syncCount + syncCount1 >= config.BatchCount)
{
return (videos, syncCount);
}
}
else
{
Log.Debug($"[{VideoType.GetDesc()}][{cookie.UserName}]-动态视频[{item.Desc}],下载失败...");
}
}
else
{
// 处理图文视频逻辑
if (config.DownImageVideo || config.DownMp3 || config.DownImage)
{
var mergevideo = await ProcessImageSetAndMergeToVideo(cookie, item, config, followed, cate);
if (mergevideo != null)
{
videos.Add(mergevideo);
syncCount++;
if (syncCount + syncCount1 >= config.BatchCount)
{
return (videos, syncCount);
}
}
}
}
}
}
return (videos, syncCount);
}
private async Task AutoDistinct(AppConfig config, DouyinVideo exitVideo)
{
// 去重,检查视频是否已存在(按优先级下载)
if (config.AutoDistinct)
{
if (exitVideo != null)
{
// 2. 已存在视频:先判断本地文件是否存在
if (File.Exists(exitVideo.VideoSavePath))
{
List priLevs = new List();
if (!string.IsNullOrWhiteSpace(config.PriorityLevel))
{
priLevs = JsonConvert.DeserializeObject>(config.PriorityLevel);
}
// 4. 处理优先级:获取「最高优先级」(Sort 越小优先级越高)
PriorityLevelDto maxPriority = null;
if (priLevs.Any())
{
// 前端已配置优先级:取 Sort 最小的(1最高)
maxPriority = priLevs.OrderBy(x => x.Sort).FirstOrDefault();
}
else
{
// 前端未配置:使用默认优先级(喜欢 > 收藏 > 关注)
maxPriority = new PriorityLevelDto { Id = 1, Sort = 1, Name = "喜欢的" }; // 默认「喜欢的视频」最高
}
// 5. 转换为当前上下文的视频类型
var maxPriorityType = (VideoTypeEnum)maxPriority.Id; // 配置的最高优先级类型
// 6. 获取已存在视频的类型(从数据库中 exitVideo 读取,需确保字段存在)
var exitVideoType = exitVideo.ViedoType; // 假设数据库存储了 VideoType.GetVideoTypeDesc()(1/2/3)
// 7. 优先级逻辑判断(核心)
if (VideoType == maxPriorityType)
{
// 情况1:当前要下载的是「最高优先级」视频
if (exitVideoType == VideoType)
{
// 已存在同优先级视频 → 跳过下载(避免重复)
//Log.Debug($"[{VideoType.GetVideoTypeDesc()}]-视频-{exitVideo.AwemeId}-[{exitVideo.VideoTitle}]已存在(同最高优先级),跳过");
return false;
}
else
{
// 已存在「低优先级」视频 → 替换(删除旧文件,继续下载新的最高优先级视频)
//Log.Debug($"[{VideoType.GetVideoTypeDesc()}]-视频-{exitVideo.AwemeId}-[{exitVideo.VideoTitle}]已存在(低优先级:{exitVideoType.GetVideoTypeDesc()}),替换为最高优先级:{currentVideoType.GetVideoTypeDesc()}");
// 删除旧的低优先级文件(可选:也可保留备份,根据需求调整)
try
{
//File.Delete(exitVideo.VideoSavePath);
DeleteOldViedo(config, exitVideo);
//Log.Debug($"已删除旧文件:{exitVideo.VideoSavePath}");
}
catch (Exception ex)
{
Log.Error($"[{VideoType.GetDesc()}]-删除重复的旧文件失败:{ex.Message}", ex);
// 即使删除失败,仍继续下载(新文件会覆盖旧文件,或按路径规则重命名)
}
// 继续执行下载逻辑(覆盖旧数据)
}
}
else
{
// 情况2:当前要下载的是「非最高优先级」视频
if (exitVideoType == maxPriorityType)
{
// 已存在「最高优先级」视频 → 跳过(不替换最高优先级)
//Log.Debug($"[{VideoType.GetVideoTypeDesc()}]-视频-{exitVideo.AwemeId}-[{exitVideo.VideoTitle}]已存在最高优先级视频({maxPriorityType}),当前类型({currentVideoType.GetVideoTypeDesc()})优先级低,跳过");
return false;
}
else
{
// 已存在「其他非最高优先级」视频 → 比较两者优先级
// 获取当前类型和已存在类型的 Sort 值
var currentSort = priLevs.FirstOrDefault(x => x.Id == (int)VideoType)?.Sort ?? int.MaxValue;
var exitSort = priLevs.FirstOrDefault(x => x.Id == (int)exitVideoType)?.Sort ?? int.MaxValue;
if (currentSort < exitSort)
{
// 当前类型优先级更高 → 替换旧视频
//Log.Debug($"[{VideoType.GetVideoTypeDesc()}]-视频-{exitVideo.AwemeId}-[{exitVideo.VideoTitle}]已存在低优先级视频({exitVideoType.GetVideoTypeDesc()}),替换为当前优先级:{currentVideoType.GetVideoTypeDesc()}");
// 删除旧文件
DeleteOldViedo(config, exitVideo);
// 继续下载
}
else
{
// 当前类型优先级更低或相等 → 跳过
//Log.Debug($"[{VideoType.GetVideoTypeDesc()}]-视频-{exitVideo.AwemeId}-[{exitVideo.VideoTitle}]已存在更高/同等优先级视频({exitVideoType.GetVideoTypeDesc()}),当前类型({currentVideoType.GetVideoTypeDesc()})跳过");
return false;
}
}
}
}
else
{
if (exitVideo.OnlyImgOrOnlyMp3)
{
return true;//说明是图文视频,不需要再下载视频了
}
else
{
//记录存在,但本地文件不存在,则继续下载。
//Log.Debug($"[{VideoType.GetVideoTypeDesc()}]-视频-{exitVideo.AwemeId}记录存在,但本地文件缺失,删除记录,重新下载");
//删除原来的记录
await douyinVideoService.DeleteById(exitVideo.Id);
}
}
}
}
return true;
}
private void DeleteOldViedo(AppConfig config, DouyinVideo exitVideo)
{
if (File.Exists(exitVideo.VideoSavePath))
{
var dirPath = Path.GetDirectoryName(exitVideo.VideoSavePath);
if (Directory.Exists(dirPath))
{
Directory.Delete(dirPath, true);
//Log.Debug($"[{VideoType.GetVideoTypeDesc()}]-已删除旧文件夹:{dirPath}");
}
//查看是否还有其他文件,如果没有则删除文件夹
var parentDir = Path.GetDirectoryName(exitVideo.VideoSavePath);
if (Directory.Exists(parentDir) && !Directory.EnumerateFileSystemEntries(parentDir).Any())
{
Directory.Delete(parentDir);
//Log.Debug($"[{VideoType.GetVideoTypeDesc()}]-已删除空文件夹:{parentDir}");
}
}
}
///
/// 处理单个视频--正常的收藏喜欢关注的 视频
/// 负责下载视频、封面、头像,生成NFO文件,创建视频实体等
///
/// 用户Cookie
/// 视频信息
/// 视频信息对象
/// 应用配置
/// 关注
///
/// 处理后的视频实体,如果处理失败则为null
protected async Task ProcessSingleVideo(DouyinCookie cookie, Aweme item, DouyinVideoInfoResponse data, AppConfig config, DouyinFollowed followed = null, DouyinCollectCate cate = null)
{
// 检查视频数据是否有效
if (!IsAwemeValid(item)) return null;
// 获取视频的码率信息
var v = item.Video.BitRate.Where(x => !string.IsNullOrEmpty(x.Format)).FirstOrDefault();
if (v == null) return null;
// 获取视频播放地址
var videoUrl = v.PlayAddr.UrlList.Where(x => !string.IsNullOrEmpty(x))?.FirstOrDefault();
if (string.IsNullOrWhiteSpace(videoUrl)) return null;
// 创建保存文件夹
var saveFolder = CreateSaveFolder(cookie, item, config, followed, cate);
// 获取视频文件名
var fileName = GetVideoFileName(cookie, item, config, cate);
// 拼接视频保存路径
var savePath = Path.Combine(saveFolder, fileName);
// 如果文件已存在,跳过
if (File.Exists(savePath))
{
//Log.Debug($"[{VideoType.GetVideoTypeDesc()}]-视频[{DouyinFileNameHelper.SanitizePath(item.Desc)}]已存在,跳过下载.");
return null;
}
Log.Debug($"[{VideoType.GetDesc()}][{item?.Author?.Nickname ?? ""}]-视频[{DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId)}]开始下载...");
// 随机延迟
await Task.Delay(_random.Next(1, 4) * 1000);
// 下载视频
var (Success, ActualSavePath) = await douyinHttpClientService.DownloadAsync(videoUrl, savePath, cookie.Cookies);
if (!Success)
{
Log.Error($"[{VideoType.GetDesc()}][{item?.Author?.Nickname ?? ""}]-视频[{DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId)}]下载失败!!!");
(bool flowControl, DouyinVideo value) = await SwitchOtherUrlAddressDown(cookie, item, videoUrl, savePath);
if (!flowControl)
{
return value;
}
}
else
{
Log.Debug($"[{VideoType.GetDesc()}][{item?.Author?.Nickname ?? ""}]-视频[{DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId)}]下载完成.");
}
// 下载视频封面
await GetDownVideoCover(item, saveFolder, cookie, config, cate);
// 下载作者头像
var (avatarSavePath, avatarUrl) = await DownAuthorAvatar(cookie, item);
// 创建视频实体
return await CreateVideoEntity(config, cookie, item, v, savePath, saveFolder, avatarSavePath, null, cate);
}
///
/// 动态视频处理
///
///
///
///
///
///
///
///
protected async Task ProcessDynamicVideo(List dynamicUrls, DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed = null, DouyinCollectCate cate = null)
{
// 创建保存文件夹
var saveFolder = CreateSaveFolder(cookie, item, config, followed, cate);
// 获取视频文件名
//var fileName = DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId) + ".mp4";
var fileName = $"{item.AwemeId}.mp4";
// 拼接视频保存路径
var savePath = Path.Combine(saveFolder, fileName);
Log.Debug($"[{VideoType.GetDesc()}]-动态视频[{DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId)}]开始下载...");
// 随机延迟
int i = 1;
List dynamicSavePaths = new List();
foreach (var dynamicUrl in dynamicUrls)
{
// 下载动态视频
var dynamicSavePath = savePath.Replace(".mp4", $"_00{i}.mp4");
DouyinDynamicVideoDto v = new DouyinDynamicVideoDto() { Height = dynamicUrl.Height, Width = dynamicUrl.Width, Path = dynamicSavePath };
i++;
// 如果文件已存在,跳过
if (File.Exists(dynamicSavePath))
{
//Log.Debug($"[{VideoType.GetVideoTypeDesc()}]-视频[{DouyinFileNameHelper.SanitizePath(item.Desc)}]已存在,跳过下载.");
dynamicSavePaths.Add(v);
continue;
}
var (Success, ActualSavePath) = await douyinHttpClientService.DownloadAsync(dynamicUrl.Path, dynamicSavePath, cookie.Cookies);
if (!Success)
{
Log.Error($"[{VideoType.GetDesc()}][{item?.Author?.Nickname ?? ""}]-动态视频[{dynamicSavePath}]-00{i},下载失败!!!");
}
else
{
dynamicSavePaths.Add(v);
Log.Debug($"[{VideoType.GetDesc()}][{item?.Author?.Nickname ?? ""}]-动态视频[{dynamicSavePath}]-00{i},下载完成.");
}
await Task.Delay(_random.Next(1, 4) * 1000);
}
//// 下载视频封面
//await DownVideoCover(item, saveFolder, cookie, config);
//// 下载作者头像
var (avatarSavePath, avatarUrl) = await DownAuthorAvatar(cookie, item);
// 创建视频实体
var virtualBitRate = new VideoBitRate
{
PlayAddr = new PlayAddr
{
Width = item?.Images?.FirstOrDefault()?.Width ?? 0,
Height = item?.Images?.FirstOrDefault()?.Height ?? 0,
DataSize = DouyinFileUtils.GetTotalFileSize(dynamicSavePaths.Select(x => x.Path).ToList()) // 合成视频的文件大小
}
};
return await CreateVideoEntity(config, cookie, item, virtualBitRate, dynamicSavePaths.FirstOrDefault()?.Path, saveFolder, avatarSavePath, dynamicSavePaths, cate);
}
///
/// 寻找其他视频地址下载...
///
///
///
///
///
///
private async Task<(bool flowControl, DouyinVideo value)> SwitchOtherUrlAddressDown(DouyinCookie cookie, Aweme item, string videoUrl, string savePath)
{
Log.Debug($"[ {VideoType.GetDesc()} ][ {item?.Author?.Nickname ?? ""} ]-视频[{DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId)}],正在尝试切换其他地址再次重新下载...");
var otherUrls = new List();
foreach (var bit in item.Video.BitRate)
{
if (bit == null)
{
continue;
}
var payUrls = bit.PlayAddr;
if (payUrls == null || payUrls.UrlList == null || payUrls.UrlList.Count == 0)
{
continue;
}
foreach (var payurl in payUrls.UrlList)
{
if (payurl == videoUrl)// 排除最开始下载失败的视频地址
continue;
otherUrls.Add(payurl);
}
}
if (otherUrls.Count > 0)
{
//Log.Debug($"[{VideoType.GetVideoTypeDesc()}]-额外找到{otherUrls.Count}个视频地址,即将再次开始下载...");
var (Success, ActualSavePath) = await douyinHttpClientService.DownloadAsync(videoUrl, savePath, cookie.Cookies, otherUrls);
if (Success)
{
Log.Debug($"[{VideoType.GetDesc()}]-尝试多个地址后,下载成功,{savePath}");
}
else
{
Log.Error($"[{VideoType.GetDesc()}]-尝试多个地址后,依旧下载失败,{item.Desc}");
return (flowControl: false, value: null);
}
}
else
{
Log.Error($"[{VideoType.GetDesc()}]-没有找到额外下载链接可以下载了," + item.Desc);
return (flowControl: false, value: null);
}
return (flowControl: true, value: null);
}
///
/// 负责下载图片、合成视频、处理封面和头像等
///
/// 用户Cookie
/// 视频信息(包含图片集)
/// 视频信息对象
/// 应用配置
/// 应用配置
///
/// 合成后的视频实体,如果处理失败则为null
protected async Task ProcessImageSetAndMergeToVideo(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed, DouyinCollectCate cate)
{
try
{
// 提取图片URL列表
List imageUrls = item.Images?
.Where(img => img.UrlList != null && img.UrlList.Any())
.Select(img => img.UrlList.FirstOrDefault())
.Where(url => !string.IsNullOrWhiteSpace(url))
.ToList();
// 如果没有图片,返回null
if (imageUrls == null || !imageUrls.Any())
{
return null;
}
// 创建图片保存文件夹
var fileNamefolder = string.Empty;
fileNamefolder = CreateSaveFolder(cookie, item, config, followed, cate);
if (!Directory.Exists(fileNamefolder)) Directory.CreateDirectory(fileNamefolder);
var fileName = GetVideoFileName(cookie, item, config, cate);
// 合成视频的保存路径
var savePath = Path.Combine(fileNamefolder, fileName);
// 如果文件已存在,返回null
if (File.Exists(savePath))
{
FileInfo fileInfo = new FileInfo(savePath);
if (fileInfo.Length > 0)
return null;
}
// 获取音乐URL
var mp3Url = item.Music?.PlayUrl?.UrlList?.FirstOrDefault();
var firstImage = item.Images.FirstOrDefault();
int height = firstImage.Height;
int width = firstImage.Width;
// 准备合成视频的请求参数
var reqParams = new MediaMergeRequest
{
ImageDurationPerSecond = 3, // 每张图片显示的时长(秒)
OutputFormat = "mp4", // 输出视频格式
VideoFps = 30, // 视频帧率
AudioUrls = string.IsNullOrWhiteSpace(mp3Url) ? new List() : new List { mp3Url }, // 音频URL列表
ImageUrls = imageUrls, // 图片URL列表
VideoWidth = width > 0 ? width : 1080, // 视频宽度
VideoHeight = height > 0 ? height : 1920, // 视频高度
};
// 执行图片合成视频操作
var mergeResult = await douyinMergeVideoService.MergeToVideo(cookie.Cookies, AppContext.BaseDirectory, reqParams, savePath, fileNamefolder, config.DownImageVideo, config.DownImage, config.DownMp3);
if (!mergeResult)
{
Log.Error($"[{VideoType.GetDesc()}]-图文视频-[{DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId)}]合成失败!!!");
return null;
}
// 获取不带扩展名的完整路径
//string fullPathWithoutExtension = Path.Combine(Path.GetDirectoryName(savePath),Path.GetFileNameWithoutExtension(savePath) );
if (config.DownImageVideo)
{
// 检查合成后的视频文件是否有效
if (!File.Exists(savePath) || new FileInfo(savePath).Length <= 0)
{
Log.Error($"[{VideoType.GetDesc()}]-图文视频-[{DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId)}]合成失败!!!");
// 清理无效的文件和文件夹
if (Directory.Exists(fileNamefolder))
{
File.Delete(savePath);
Directory.Delete(fileNamefolder, true);
Log.Error($"[{VideoType.GetDesc()}]-图文视频-删除合成失败的视频文件和目录...");
}
return null;
}
}
else
{
savePath = "";
}
var coverUrl = cate is not null && cate.CateType != VideoTypeEnum.dy_custom_collect
? (item.MixInfo?.CoverUrl?.UrlList?.FirstOrDefault() ?? imageUrls.FirstOrDefault() ?? item.Music?.CoverHd?.UrlList?.FirstOrDefault())
: imageUrls.FirstOrDefault();
// 下载视频封面(使用第一张图片作为封面)
if (!string.IsNullOrWhiteSpace(coverUrl))
await DownVideoCover(coverUrl, fileNamefolder, cookie, item, config, cate);
// 下载作者头像
var (avatarSavePath, avatarUrl) = await DownAuthorAvatar(cookie, item);
// 为合成的视频创建一个“虚拟”的BitRate对象,以便复用CreateVideoEntity方法
var virtualBitRate = new VideoBitRate
{
PlayAddr = new PlayAddr
{
Width = reqParams.VideoWidth,
Height = reqParams.VideoHeight,
DataSize = string.IsNullOrWhiteSpace(savePath) || !File.Exists(savePath) ? 0 : new FileInfo(savePath).Length // 合成视频的文件大小
}
};
// 创建视频实体
var videoEntity = await CreateVideoEntity(config,
cookie, item, virtualBitRate, savePath, fileNamefolder, avatarSavePath, null, cate);
// 特殊处理合成视频的字段
videoEntity.FileHash = string.Empty; // 合成视频没有原始文件哈希
videoEntity.VideoUrl = "/"; // 合成视频没有原始URL
videoEntity.ViedoType = VideoType;
videoEntity.IsMergeVideo = 1;// 标记为图片合成视频
return videoEntity;
}
catch (Exception ex)
{
Log.Error(ex, $"[{VideoType.GetDesc()}]-图片视频同步-处理图片集并合成视频时出错");
return null;
}
}
///
/// 保存视频信息到数据库
/// 批量插入视频实体列表到数据库中
///
/// 要保存的视频实体列表
/// 保存成功的视频数量
protected async Task SaveVideos(List videos)
{
if (!videos.Any()) return 0;
try
{
await douyinVideoService.BatchInsertOrUpdate(videos);
return videos.Count;
}
catch (Exception ex)
{
Log.Error(ex, $"[{VideoType.GetDesc()}]-批量保存视频到数据库失败");
// 清理保存失败的视频文件
await CleanupFailedVideos(videos);
return 0;
}
}
///
/// 下载视频封面
/// 从视频信息中提取封面URL并下载到指定文件夹
///
/// 视频信息
/// 封面保存文件夹
/// 用户Cookie
/// 应用配置
///
/// 一个表示异步操作的任务
protected async Task GetDownVideoCover(Aweme item, string saveFolder, DouyinCookie cookie, AppConfig config, DouyinCollectCate cate)
{
// 定义封面URL变量
string coverUrl;
// 按照优先级获取封面URL
if (cate is not null)
{
// cate不为空时:优先MixInfo封面 → 其次Music高清封面 → 最后Video封面
coverUrl = item.MixInfo?.CoverUrl?.UrlList?.FirstOrDefault()
?? item.Video.Cover.UrlList?.LastOrDefault()
?? item.Music?.CoverHd?.UrlList?.FirstOrDefault();
}
else
{
// cate为空时:只取Video封面
coverUrl = item.Video.Cover.UrlList?.FirstOrDefault();
}
// 如果封面URL为空,直接返回
if (string.IsNullOrWhiteSpace(coverUrl)) return;
// 调用下载封面的方法
await DownVideoCover(coverUrl, saveFolder, cookie, item, config, cate);
}
///
/// 下载作者头像
/// 从视频信息中提取作者头像URL并下载到指定文件夹
///
/// 用户Cookie
/// 视频信息
/// 一个元组,包含头像保存路径和头像URL
protected async Task<(string savePath, string url)> DownAuthorAvatar(DouyinCookie cookie, Aweme item)
{
if (item.Author == null) return (null, null);
// 优先获取高清头像
var avatarUrl = item.Author.AvatarLarger?.UrlList?.FirstOrDefault() ?? item.Author.AvatarThumb?.UrlList?.FirstOrDefault();
if (string.IsNullOrWhiteSpace(avatarUrl)) return (null, null);
// 拼接头像保存路径
var avatarSavePath = Path.Combine(GetAuthorAvatarBasePath(cookie), $"{item.Author.Uid}.jpg");
var avatarDir = Path.GetDirectoryName(avatarSavePath);
// 创建头像保存文件夹
if (!Directory.Exists(avatarDir)) Directory.CreateDirectory(avatarDir);
// 如果头像文件不存在,则下载
if (!File.Exists(avatarSavePath))
{
await douyinHttpClientService.DownloadAsync(avatarUrl, avatarSavePath, cookie.Cookies);
}
return (avatarSavePath, avatarUrl);
}
#endregion
#region 私有方法
///
/// 检查视频数据是否有效
/// 验证视频信息是否包含必要的字段
///
/// 视频信息
/// 如果视频数据有效,则为true;否则为false
private static bool IsAwemeValid(Aweme item) => item != null && item.Video != null && item.Video.BitRate != null;
///
/// 获取视频标签
/// 从视频信息中提取三个级别的标签
///
/// 视频信息
/// 一个元组,包含三个级别的视频标签
protected (string tag1, string tag2, string tag3) GetVideoTags(Aweme item)
{
var tags = item.VideoTags;
return (
tags?.FirstOrDefault(x => x.Level == 1)?.TagName,
tags?.FirstOrDefault(x => x.Level == 2)?.TagName,
tags?.FirstOrDefault(x => x.Level == 3)?.TagName
);
}
///
/// 清理保存失败的视频文件
/// 当数据库保存失败时,删除已下载的视频文件和文件夹
///
/// 保存失败的视频实体列表
/// 一个表示异步操作的任务
private async Task CleanupFailedVideos(List videos)
{
Log.Debug($"[{VideoType.GetDesc()}]-数据库保存失败,开始清理本次下载的视频目录...");
foreach (var video in videos)
{
// 异步删除文件和文件夹,避免阻塞主线程
await Task.Run(() =>
{
try
{
if (!string.IsNullOrWhiteSpace(video.VideoSavePath) && File.Exists(video.VideoSavePath))
{
File.Delete(video.VideoSavePath);
}
string directory = Path.GetDirectoryName(video.VideoSavePath);
if (!string.IsNullOrWhiteSpace(directory) && Directory.Exists(directory) && Directory.GetFileSystemEntries(directory).Length == 0)
{
Directory.Delete(directory);
}
Log.Debug($"[{VideoType.GetDesc()}]-清理失败视频文件成功: {video.VideoSavePath}!!!");
}
catch (Exception ex)
{
Log.Warning(ex, $"[{VideoType.GetDesc()}]-清理失败视频文件出错: {video.VideoSavePath}!!!");
}
});
}
}
///
/// 下载视频封面(重载)
/// 根据指定的封面URL下载封面图片,并复制为fanart.jpg
///
/// 封面图片URL
/// 封面保存文件夹
/// 用户Cookie
/// 视频信息
/// 应用配置
///
/// 一个表示异步操作的任务
private async Task DownVideoCover(string coverUrl, string saveFolder, DouyinCookie cookie, Aweme item, AppConfig config, DouyinCollectCate cate)
{
if (string.IsNullOrWhiteSpace(coverUrl)) return;
// 获取封面图片文件名
var coverImgName = GetNfoFileName(cookie, item, config, "poster.jpg", cate);
var coverSavePath = Path.Combine(saveFolder, coverImgName);
// 如果封面文件不存在,则下载
if (!File.Exists(coverSavePath))
{
await douyinHttpClientService.DownloadAsync(coverUrl, coverSavePath, cookie.Cookies);
}
}
///
/// 创建视频实体
/// 根据视频信息、下载路径等创建DouyinVideo实体对象
///
/// 配置
/// 用户Cookie
/// 视频信息
/// 视频码率信息
/// 视频保存路径
/// 视频保存文件夹
///
/// 动态视频
/// 短剧、合集、自定义收藏夹
/// 创建的视频实体对象
private async Task CreateVideoEntity(AppConfig config,
DouyinCookie cookie, Aweme item, VideoBitRate bitRate, string savePath, string saveFolder, string avatorPath, List dynamicVideos = null, DouyinCollectCate cate = null)
{
// 获取视频标签
var (tag1, tag2, tag3) = GetVideoTags(item);
var video = new DouyinVideo
{
ViedoType = VideoType,
AwemeId = item.AwemeId,
Author = item.Author?.Nickname,
AuthorId = item.Author?.Uid,
AuthorAvatar = avatorPath,
AuthorAvatarUrl = item.Author.AvatarLarger?.UrlList?.FirstOrDefault() ?? item.Author.AvatarThumb?.UrlList?.FirstOrDefault(),
CreateTime = DateTimeUtil.Convert10BitTimestamp(item.CreateTime),
VideoTitle = string.IsNullOrWhiteSpace(item.Desc) ? $"{item.Author?.Nickname}-{item.CreateTime}" : item.Desc,
//VideoTitleSimplify = VideoType == VideoTypeEnum.dy_follows? GetVideoSimplifyTitle(item):string.Empty,
Id = IdGener.GetLong().ToString(),
Resolution = $"{bitRate.PlayAddr.Width}×{bitRate.PlayAddr.Height}",
FileSize = bitRate.PlayAddr.DataSize,
FileHash = bitRate.PlayAddr.FileHash,
Tag1 = tag1,
Tag2 = tag2,
Tag3 = tag3,
VideoUrl = bitRate.PlayAddr.UrlList?.FirstOrDefault(),
VideoCoverUrl = item.Video.Cover.UrlList?.FirstOrDefault(),
VideoSavePath = savePath,
VideoCoverSavePath = Path.Combine(saveFolder, "poster.jpg"),
SyncTime = DateTime.Now,
DyUserId = item.AuthorUserId == 0 ? item.Author?.Uid : item.AuthorUserId.ToString(),
CookieId = cookie.Id,
OnlyImgOrOnlyMp3 = string.IsNullOrWhiteSpace(savePath) && !config.DownImageVideo && (config.DownImage || config.DownMp3),
CateId = cate?.Id,
CateXId = cate?.XId,
};
if (cate != null && cate.CateType != VideoTypeEnum.dy_custom_collect)
{
video.VideoTitle = (string.IsNullOrWhiteSpace(item.Desc) ? cate.Name : $"[{cate.Name}]" + "_" + item.Desc) + "_" + item.MixInfo.Statis.CurrentEpisode;
}
if (dynamicVideos != null && dynamicVideos.Count > 0)
{
video.DynamicVideos = JsonConvert.SerializeObject(dynamicVideos);
}
// 生成NFO文件
NfoFileGenerator.GenerateVideoNfoFile(video);
return video;
}
///
///
///
///
///
//private string GetVideoSimplifyTitle(Aweme item)
//{
// var config = douyinCommonService.GetConfig();
// string simplifiedTitle = string.Empty;
// if (config?.UperUseViedoTitle ?? false)
// {
// simplifiedTitle = DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId);
// }
// return simplifiedTitle;
//}
#endregion
}
}