1、删除视频,永不下载

2、去重-优先级
3、视频合成优化
4、其他优化
This commit is contained in:
jianzhichu
2025-12-03 23:43:10 +08:00
parent bd3a103f8f
commit 25e2a22dd2
29 changed files with 1023 additions and 702 deletions
+109
View File
@@ -0,0 +1,109 @@
using dy.net.dto;
using dy.net.service;
using Quartz;
using Serilog;
using static Quartz.Logging.OperationName;
namespace dy.net.job
{ /// <summary>
/// 抖音任务依赖监听器(独立公共类)
/// 作用:监听任务执行完成事件,触发下一个依赖任务,实现顺序执行
/// </summary>
public class DouyinJobDependencyListener : IJobListener
{
// 监听器名称(唯一标识,不可重复)
public string Name => "DouyinJobDependencyListener";
/// <summary>
/// 任务配置字典(从外部注入)
/// </summary>
private readonly Dictionary<string, JobConfig> _jobConfigs;
/// <summary>
/// 任务依赖关系(从外部注入,定义执行顺序)
/// </summary>
private readonly Dictionary<string, string> _jobDependency;
/// <summary>
/// 任务服务(用于触发下一个任务,从外部注入)
/// </summary>
private readonly DouyinQuartzJobService _jobService;
/// <summary>
/// 构造函数(依赖注入)
/// </summary>
/// <param name="jobConfigs">任务配置</param>
/// <param name="jobDependency">任务依赖关系</param>
/// <param name="jobService">任务服务</param>
public DouyinJobDependencyListener(
Dictionary<string, JobConfig> jobConfigs,
Dictionary<string, string> jobDependency,
DouyinQuartzJobService jobService)
{
_jobConfigs = jobConfigs ?? throw new ArgumentNullException(nameof(jobConfigs), "任务配置不能为空");
_jobDependency = jobDependency ?? throw new ArgumentNullException(nameof(jobDependency), "任务依赖关系不能为空");
_jobService = jobService ?? throw new ArgumentNullException(nameof(jobService), "任务服务不能为空");
}
/// <summary>
/// 任务执行前触发(无需处理)
/// </summary>
public Task JobToBeExecuted(IJobExecutionContext context, CancellationToken cancellationToken = default)
{
return Task.CompletedTask;
}
/// <summary>
/// 任务被否决执行时触发(无需处理)
/// </summary>
public Task JobExecutionVetoed(IJobExecutionContext context, CancellationToken cancellationToken = default)
{
return Task.CompletedTask;
}
/// <summary>
/// 任务执行完成后触发(核心逻辑:触发下一个依赖任务)
/// </summary>
public async Task JobWasExecuted(IJobExecutionContext context, JobExecutionException? jobException, CancellationToken cancellationToken = default)
{
var currentJobKey = context.JobDetail.Key;
Log.Information("【任务监听】任务执行完成 - 任务名称: {JobName}, 执行状态: {Status}",
currentJobKey.Name, jobException == null ? "成功" : "失败");
// 1. 若当前任务执行失败,终止后续依赖任务(避免无效执行)
if (jobException != null)
{
Log.Error(jobException, "【任务监听】任务 {JobName} 执行失败,终止后续任务链条", currentJobKey.Name);
return;
}
// 2. 根据当前任务的 JobKey,找到对应的配置 Key(如:dy.job.key.collect → collect
var currentConfigKey = _jobConfigs.FirstOrDefault(kv => kv.Value.JobKey == currentJobKey.Name).Key;
if (string.IsNullOrEmpty(currentConfigKey))
{
Log.Warning("【任务监听】未找到任务 {JobName} 的配置信息,任务链条终止", currentJobKey.Name);
return;
}
// 3. 查找下一个依赖任务的配置 Key
if (!_jobDependency.TryGetValue(currentConfigKey, out var nextConfigKey) || string.IsNullOrEmpty(nextConfigKey))
{
Log.Information("【任务监听】任务 {JobName} 是最后一个任务,本次任务链条执行完毕", currentJobKey.Name);
return;
}
// 4. 触发下一个任务(标记为「依赖触发」,立即执行)
Log.Information("【任务监听】准备触发下一个任务: {NextJobName}(依赖触发)", nextConfigKey);
var triggerSuccess = await _jobService.StartJobAsync(nextConfigKey, "", isDependencyTrigger: true);
if (triggerSuccess)
{
Log.Information("【任务监听】下一个任务 {NextJobName} 触发成功", nextConfigKey);
}
else
{
Log.Error("【任务监听】下一个任务 {NextJobName} 触发失败,任务链条中断", nextConfigKey);
}
}
}
}
+8 -12
View File
@@ -15,9 +15,8 @@ namespace dy.net.job
DouyinCommonService douyinCommonService,DouyinFollowService douyinFollowService,DouyinMergeVideoService douyinMergeVideoService)
: base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService,douyinFollowService, douyinMergeVideoService) { }
protected override string JobType => SystemStaticUtil.DY_COLLECTS;
protected override VideoTypeEnum VideoType => VideoTypeEnum.Collect;
protected override VideoTypeEnum VideoType => VideoTypeEnum.dy_collects;
protected override async Task BeforeProcessCookies()
{
@@ -61,34 +60,31 @@ namespace dy.net.job
return Path.Combine(cookie.SavePath, "author");
}
protected override async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount)
protected override async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount,DouyinFollowed followed)
{
if (syncCount > 0)
{
Serilog.Log.Debug($"{JobType}-Cookie-[{cookie.UserName}],本次同步成功{syncCount}条视频");
Serilog.Log.Debug($"{VideoType}-Cookie-[{cookie.UserName}],本次同步成功{syncCount}条视频");
cookie.CollHasSyncd = 1;
await douyinCookieService.UpdateAsync(cookie);
}
else
{
Serilog.Log.Debug($"{JobType}-Cookie-[{cookie.UserName}],本次没有查询到新的视频");
Serilog.Log.Debug($"{VideoType}-Cookie-[{cookie.UserName}],没有可以同步的新视频");
}
}
protected override VideoEntityDifferences GetVideoEntityDifferences(DouyinCookie cookie, Aweme item)
{
return new VideoEntityDifferences
{
VideoType = VideoTypeEnum.Collect,
};
return new VideoEntityDifferences();
}
protected override string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed)
{
var (tag1, _, _) = GetVideoTags(item);
var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : DouyinFileNameHelper.SanitizePath(tag1);
var folder = Path.Combine(cookie.SavePath, safeTag1, $"{DouyinFileNameHelper.SanitizePath(item.Desc)}@{item.AwemeId}");
var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : DouyinFileNameHelper.SanitizeLinuxFileName(tag1);
var folder = Path.Combine(cookie.SavePath, safeTag1, $"{DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc)}@{item.AwemeId}");
if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
return folder;
}
+6 -15
View File
@@ -17,9 +17,8 @@ namespace dy.net.job
DouyinCommonService douyinCommonService,DouyinFollowService douyinFollowService,DouyinMergeVideoService douyinMergeVideoService)
: base(douyinCookieService, douyinHttpClientService, douyinVideoService, douyinCommonService, douyinFollowService, douyinMergeVideoService) { }
protected override string JobType => SystemStaticUtil.DY_FAVORITES;
protected override VideoTypeEnum VideoType => VideoTypeEnum.Favorite;
protected override VideoTypeEnum VideoType => VideoTypeEnum.dy_favorite;
protected override async Task<List<DouyinCookie>> GetValidCookies()
{
@@ -54,33 +53,25 @@ namespace dy.net.job
return Path.Combine(cookie.FavSavePath, "author");
}
protected override async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount)
protected override async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount, DouyinFollowed followed)
{
if (syncCount > 0)
{
Serilog.Log.Debug($"{JobType}-Cookie-[{cookie.UserName}],本次同步成功{syncCount}条视频");
Serilog.Log.Debug($"{VideoType}-Cookie-[{cookie.UserName}],本次同步成功{syncCount}条视频");
cookie.FavHasSyncd = 1;
await douyinCookieService.UpdateAsync(cookie);
}
else
{
Serilog.Log.Debug($"{JobType}-Cookie-[{cookie.UserName}],本次没有查询到新的视频");
Serilog.Log.Debug($"{VideoType}-Cookie-[{cookie.UserName}],没有可以同步的新视频");
}
}
protected override VideoEntityDifferences GetVideoEntityDifferences(DouyinCookie cookie, Aweme item)
{
return new VideoEntityDifferences
{
VideoType = VideoTypeEnum.Favorite
};
}
protected override string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed)
{
var (tag1, _, _) = GetVideoTags(item);
var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : DouyinFileNameHelper.SanitizePath(tag1);
var folder = Path.Combine(cookie.FavSavePath, safeTag1, $"{DouyinFileNameHelper.SanitizePath(item.Desc)}@{item.AwemeId}");
var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : DouyinFileNameHelper.SanitizeLinuxFileName(tag1);
var folder = Path.Combine(cookie.FavSavePath, safeTag1, $"{DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc)}@{item.AwemeId}");
if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
return folder;
}
+113 -82
View File
@@ -82,7 +82,7 @@ namespace dy.net.job
/// 任务类型名称,用于日志记录和区分不同的同步任务
/// 子类必须实现此属性并返回具体的任务类型
/// </summary>
protected abstract string JobType { get; }
//protected abstract string VideoType { get; }
protected abstract VideoTypeEnum VideoType { get; }
@@ -131,14 +131,13 @@ namespace dy.net.job
var config = douyinCommonService.GetConfig();
if (config == null)
{
Log.Debug($"{JobType}-未获取到系统配置,任务终止!!!");
Log.Debug($"{VideoType}-未获取到系统配置,任务终止!!!");
return;
}
// 2. 从配置中获取每页请求数量
if (config.BatchCount > 0)
count = config.BatchCount.ToString();
// 2. 从配置中获取每页请求数量--固定18
//if (config.BatchCount > 0)
// count = config.BatchCount.ToString();
// 3. 在处理Cookie之前执行的预处理操作
await BeforeProcessCookies();
@@ -147,11 +146,11 @@ namespace dy.net.job
var cookies = await GetValidCookies();
if (cookies == null || !cookies.Any())
{
Log.Debug($"{JobType}-无有效Cookie,任务终止!!!");
Log.Debug($"{VideoType}-无有效Cookie,任务终止!!!");
return;
}
Log.Debug($"{JobType}-共发现{cookies.Count}个Cookie,同步任务即将开始...");
Log.Debug($"{VideoType}-共发现{cookies.Count}个Cookie,同步任务即将开始...");
// 6. 遍历每个有效的Cookie,执行同步操作
foreach (var cookie in cookies)
@@ -262,8 +261,9 @@ namespace dy.net.job
/// </summary>
/// <param name="cookie">用户Cookie</param>
/// <param name="syncCount">本次同步成功的视频数量</param>
/// <param name="followed"></param>
/// <returns>一个表示异步操作的任务</returns>
protected abstract Task HandleSyncCompletion(DouyinCookie cookie, int syncCount);
protected abstract Task HandleSyncCompletion(DouyinCookie cookie, int syncCount,DouyinFollowed followed=null);
/// <summary>
/// 获取视频实体的差异信息
@@ -272,7 +272,10 @@ namespace dy.net.job
/// <param name="cookie">用户Cookie</param>
/// <param name="item">视频信息</param>
/// <returns>视频实体的差异信息,包含视频类型和简化标题</returns>
protected abstract VideoEntityDifferences GetVideoEntityDifferences(DouyinCookie cookie, Aweme item);
protected virtual VideoEntityDifferences GetVideoEntityDifferences(DouyinCookie cookie, Aweme item)
{
return new VideoEntityDifferences();
}
/// <summary>
/// 获取NFO文件中的图片(如海报)文件名
@@ -302,13 +305,13 @@ namespace dy.net.job
// 检查Cookie是否有效
if (!IsCookieValid(cookie))
{
Log.Debug($"{JobType}-Cookie[{cookie.UserName}]无效,任务终止!!!");
Log.Debug($"{VideoType}-Cookie[{cookie.UserName}]无效,任务终止!!!");
return;
}
Log.Debug($"{JobType}- Cookie-[{cookie.UserName}]开始同步...");
Log.Debug($"{VideoType}- Cookie-[{cookie.UserName}]开始同步...");
//up主上传视频特殊处理
if (JobType == SystemStaticUtil.DY_FOLLOWEDS)
if (VideoType ==VideoTypeEnum.dy_follows)
{
int syncCount = 0; // 本次同步成功的视频数量
string cursor = "0"; // 初始游标
@@ -324,18 +327,18 @@ namespace dy.net.job
}
if (string.IsNullOrWhiteSpace(firstUp.SecUid))
{
Log.Debug($"{JobType}-Cookie[{firstUp.UperName}]无效,没有sec_userid,任务终止!!!");
Log.Debug($"{VideoType}-Cookie[{firstUp.UperName}]无效,没有sec_userid,任务终止!!!");
return;
}
foreach (var item in follows)
foreach (var followed in follows)
{
cursor = "0"; // 初始游标
await GetViedos(cookie, config, syncCount, cursor, hasMore, item);
await GetViedos(cookie, config, syncCount, cursor, hasMore, followed);
hasMore = true;
// 处理同步完成后的操作
await HandleSyncCompletion(cookie, syncCount);
await HandleSyncCompletion(cookie, syncCount,followed);
}
}
else
@@ -353,7 +356,7 @@ namespace dy.net.job
}
catch (Exception ex)
{
Log.Error(ex, $"{JobType}-Cookie[{cookie.Id}]同步出错!!!");
Log.Error(ex, $"{VideoType}-Cookie[{cookie.Id}]同步出错!!!");
}
}
@@ -366,7 +369,7 @@ namespace dy.net.job
var data = await FetchVideoData(cookie, cursor, followed == null ? "" : followed?.SecUid);
if (data == null)
{
Log.Debug($"{JobType}-Cookie[{cookie.UserName}]读取数据失败!!!");
Log.Debug($"{VideoType}-Cookie[{cookie.UserName}]读取数据失败!!!");
break;
}
@@ -387,7 +390,7 @@ namespace dy.net.job
//当syncCount达到上限时,跳出循环
if (config.BatchCount > 0 && syncCount >= config.BatchCount)
{
Log.Debug($"{JobType}-Cookie[{cookie.UserName}]本次同步达到上限{config.BatchCount},停止同步!!!");
Log.Debug($"{VideoType}-Cookie[{cookie.UserName}]本次同步达到上限{config.BatchCount},停止同步!!!");
break;
}
@@ -412,12 +415,28 @@ namespace dy.net.job
var videos = new List<DouyinVideo>();
foreach (var item in data.AwemeList)
{
//判断视频是否是强制删除且不再下载的视频
var deleteVideo = await douyinCommonService.ExistDeleteVideo(item.AwemeId);
if (deleteVideo)
{
Log.Debug($"{VideoType}-视频-{item.AwemeId}-[{item.Desc}]已被标记为强制删除,跳过下载");
continue;
}
//判断是否存在视频,是否根据去重规则进行去重处理。。
bool Goon = await AutoDistinct(config, item);
// 1. 查询数据库中是否已存在该视频(通过 AwemeId 唯一标识)
var exitVideo = await douyinVideoService.GetByAwemeId(item.AwemeId);
bool Goon = await AutoDistinct(config, item, cookie, exitVideo);
if (!Goon)
{
continue;
}
if (exitVideo!=null) {
await douyinVideoService.DeleteById(exitVideo.Id);
}
var uper = await douyinFollowService.GetByUperId(item.AuthorUserId.ToString(), cookie.MyUserId);
if (uper != null)
{
@@ -442,13 +461,11 @@ namespace dy.net.job
return videos;
}
private async Task<bool> AutoDistinct(AppConfig config, Aweme item)
private async Task<bool> AutoDistinct(AppConfig config, Aweme item,DouyinCookie cookie,DouyinVideo exitVideo)
{
// 去重,检查视频是否已存在(按优先级下载)
if (config.AutoDistinct)
{
// 1. 查询数据库中是否已存在该视频(通过 AwemeId 唯一标识)
var exitVideo = await douyinVideoService.GetByAwemeId(item.AwemeId);
if (exitVideo != null)
{
// 2. 已存在视频:先判断本地文件是否存在
@@ -465,7 +482,7 @@ namespace dy.net.job
}
catch (Exception ex)
{
Log.Error($"解析优先级配置失败:{ex.Message}", ex);
Log.Error($"{VideoType}-解析优先级配置失败:{ex.Message}", ex);
priLevs = new List<PriorityLevelDto>(); // 解析失败则用默认优先级
}
}
@@ -497,23 +514,24 @@ namespace dy.net.job
if (exitVideoType == currentVideoType)
{
// 已存在同优先级视频 → 跳过下载(避免重复)
Log.Debug($"视频-{exitVideo.AwemeId}-[{exitVideo.VideoTitle}]已存在(同最高优先级),跳过");
//Log.Debug($"{VideoType}-视频-{exitVideo.AwemeId}-[{exitVideo.VideoTitle}]已存在(同最高优先级),跳过");
return false;
}
else
{
// 已存在「低优先级」视频 → 替换(删除旧文件,继续下载新的最高优先级视频)
Log.Debug($"视频-{exitVideo.AwemeId}-[{exitVideo.VideoTitle}]已存在(低优先级:{exitVideoType}),替换为最高优先级:{currentVideoType}");
//Log.Debug($"{VideoType}-视频-{exitVideo.AwemeId}-[{exitVideo.VideoTitle}]已存在(低优先级:{exitVideoType}),替换为最高优先级:{currentVideoType}");
// 删除旧的低优先级文件(可选:也可保留备份,根据需求调整)
try
{
File.Delete(exitVideo.VideoSavePath);
Log.Debug($"已删除旧文件:{exitVideo.VideoSavePath}");
//File.Delete(exitVideo.VideoSavePath);
DeleteOldViedo(config, exitVideo);
//Log.Debug($"已删除旧文件:{exitVideo.VideoSavePath}");
}
catch (Exception ex)
{
Log.Error($"删除旧文件失败:{ex.Message}", ex);
Log.Error($"{VideoType}-删除旧文件失败:{ex.Message}", ex);
// 即使删除失败,仍继续下载(新文件会覆盖旧文件,或按路径规则重命名)
}
@@ -526,7 +544,7 @@ namespace dy.net.job
if (exitVideoType == maxPriorityType)
{
// 已存在「最高优先级」视频 → 跳过(不替换最高优先级)
Log.Debug($"视频-{exitVideo.AwemeId}-[{exitVideo.VideoTitle}]已存在最高优先级视频({maxPriorityType}),当前类型({currentVideoType})优先级低,跳过");
//Log.Debug($"{VideoType}-视频-{exitVideo.AwemeId}-[{exitVideo.VideoTitle}]已存在最高优先级视频({maxPriorityType}),当前类型({currentVideoType})优先级低,跳过");
return false;
}
else
@@ -539,39 +557,70 @@ namespace dy.net.job
if (currentSort < exitSort)
{
// 当前类型优先级更高 → 替换旧视频
Log.Debug($"视频-{exitVideo.AwemeId}-[{exitVideo.VideoTitle}]已存在低优先级视频({exitVideoType}),替换为当前优先级:{currentVideoType}");
//Log.Debug($"{VideoType}-视频-{exitVideo.AwemeId}-[{exitVideo.VideoTitle}]已存在低优先级视频({exitVideoType}),替换为当前优先级:{currentVideoType}");
// 删除旧文件
if (File.Exists(exitVideo.VideoSavePath))
{
File.Delete(exitVideo.VideoSavePath);
}
DeleteOldViedo(config, exitVideo);
// 继续下载
}
else
{
// 当前类型优先级更低或相等 → 跳过
Log.Debug($"视频-{exitVideo.AwemeId}-[{exitVideo.VideoTitle}]已存在更高/同等优先级视频({exitVideoType}),当前类型({currentVideoType})跳过");
//Log.Debug($"{VideoType}-视频-{exitVideo.AwemeId}-[{exitVideo.VideoTitle}]已存在更高/同等优先级视频({exitVideoType}),当前类型({currentVideoType})跳过");
return false;
}
}
}
}
else
}
}
return true;
}
private void DeleteOldViedo(AppConfig config, DouyinVideo exitVideo)
{
if (File.Exists(exitVideo.VideoSavePath))
{
//如果是关注的视频,并且是保存到同一文件夹,则只删除文件
if (VideoType == VideoTypeEnum.dy_follows && config.UperSaveTogether)
{
File.Delete(exitVideo.VideoSavePath);
//Log.Debug($"{VideoType}-已删除旧文件:{exitVideo.VideoSavePath}");
//删除同名的.nfo 文件
var nfoPath = Path.Combine(Path.GetDirectoryName(exitVideo.VideoSavePath), ".nfo");
if (File.Exists(nfoPath))
{
// 数据库存在记录,但本地文件丢失 → 直接重新下载(无论优先级)
Log.Debug($"视频-{exitVideo.AwemeId}-[{exitVideo.VideoTitle}]数据库存在但本地文件丢失,重新下载");
File.Delete(nfoPath);
//删除同名的封面文件
//Log.Debug($"{VideoType}-已删除旧nfo文件:{nfoPath}");
}
//删除同名的封面文件
var posterPath = Path.Combine(Path.GetDirectoryName(exitVideo.VideoSavePath), "poster.jpg");
if (File.Exists(posterPath))
{
File.Delete(posterPath);
//Log.Debug($"{VideoType}-已删除旧封面文件:{posterPath}");
}
}
else
{
// 数据库和本地均无该视频 → 直接下载
//Log.Debug($"视频-{item.AwemeId}-[{item.Desc}]未存在,开始下载");
//如果是关注的,且不是保存到同一文件夹 或者其他类型视频,直接删除文件夹
//
var dirPath = Path.GetDirectoryName(exitVideo.VideoSavePath);
if (Directory.Exists(dirPath))
{
Directory.Delete(dirPath, true);
//Log.Debug($"{VideoType}-已删除旧文件夹:{dirPath}");
}
}
//查看是否还有其他文件,如果没有则删除文件夹
var parentDir = Path.GetDirectoryName(exitVideo.VideoSavePath);
if (Directory.Exists(parentDir) && !Directory.EnumerateFileSystemEntries(parentDir).Any())
{
Directory.Delete(parentDir);
Log.Debug($"{VideoType}-已删除空文件夹:{parentDir}");
}
}
return true;
}
/// <summary>
@@ -609,22 +658,22 @@ namespace dy.net.job
// 如果文件已存在,跳过
if (File.Exists(savePath))
{
//Log.Debug($"{JobType}-视频[{DouyinFileNameHelper.SanitizePath(item.Desc)}]已存在,跳过下载.");
//Log.Debug($"{VideoType}-视频[{DouyinFileNameHelper.SanitizePath(item.Desc)}]已存在,跳过下载.");
return null;
}
Log.Debug($"{JobType}-视频[{DouyinFileNameHelper.SanitizePath(item.Desc)}]开始下载...");
Log.Debug($"{VideoType}-视频[{DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc)}]开始下载...");
// 随机延迟,模拟人类操作
await Task.Delay(_random.Next(1, 4) * 1000);
// 下载视频
if (!await douyinHttpClientService.DownloadAsync(videoUrl, savePath, cookie.Cookies))
{
Log.Error($"{JobType}-{item?.Author?.Nickname ?? ""}-视频[{DouyinFileNameHelper.SanitizePath(item.Desc)}]下载失败!!!");
Log.Error($"{VideoType}-{item?.Author?.Nickname ?? ""}-视频[{DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc)}]下载失败!!!");
return null;
}
else
{
Log.Debug($"{JobType}-{item?.Author?.Nickname ?? ""}-视频[{DouyinFileNameHelper.SanitizePath(item.Desc)}]下载完成.");
Log.Debug($"{VideoType}-{item?.Author?.Nickname ?? ""}-视频[{DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc)}]下载完成.");
}
// 下载视频封面
@@ -672,10 +721,10 @@ namespace dy.net.job
// 检查图片保存路径是否配置
if (string.IsNullOrWhiteSpace(cookie.ImgSavePath))
{
Log.Error($"{JobType}-图文视频同步-没有配置图片存储路径,任务终止!!!");
Log.Error($"{VideoType}-图文视频同步-没有配置图片存储路径,任务终止!!!");
return null;
}
fileNamefolder = Path.Combine(cookie.ImgSavePath, DouyinFileNameHelper.GenerateFileName(item.Desc, item.AwemeId));
fileNamefolder = Path.Combine(cookie.ImgSavePath, DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId));
}
else
{
@@ -713,7 +762,7 @@ namespace dy.net.job
var mergeResult = await douyinMergeVideoService.MergeToVideo(cookie.Cookies, AppContext.BaseDirectory, reqParams, savePath, fileNamefolder, config.DownImageVideo, config.DownImage, config.DownMp3);
if (!mergeResult)
{
Log.Error($"{JobType}-图文视频-[{DouyinFileNameHelper.SanitizePath(item.Desc)}]合成失败!!!");
Log.Error($"{VideoType}-图文视频-[{DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc)}]合成失败!!!");
return null;
}
@@ -722,13 +771,13 @@ namespace dy.net.job
// 检查合成后的视频文件是否有效
if (!File.Exists(savePath) || new FileInfo(savePath).Length <= 0)
{
Log.Error($"{JobType}-图文视频-[{DouyinFileNameHelper.SanitizePath(item.Desc)}]合成失败!!!");
Log.Error($"{VideoType}-图文视频-[{DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc)}]合成失败!!!");
// 清理无效的文件和文件夹
if (Directory.Exists(fileNamefolder))
{
File.Delete(savePath);
Directory.Delete(fileNamefolder, true);
Log.Error($"{JobType}-图文视频-删除合成失败的视频文件和目录...");
Log.Error($"{VideoType}-图文视频-删除合成失败的视频文件和目录...");
}
return null;
}
@@ -765,13 +814,14 @@ namespace dy.net.job
// 特殊处理合成视频的字段
videoEntity.FileHash = string.Empty; // 合成视频没有原始文件哈希
videoEntity.VideoUrl = "/"; // 合成视频没有原始URL
videoEntity.ViedoType = VideoTypeEnum.ImageVideo; // 标记为图片合成视频
videoEntity.ViedoType = VideoType;
videoEntity.IsMergeVideo = 1;// 标记为图片合成视频
return videoEntity;
}
catch (Exception ex)
{
Log.Error(ex, $"{JobType}-图片视频同步-处理图片集并合成视频时出错");
Log.Error(ex, $"{VideoType}-图片视频同步-处理图片集并合成视频时出错");
return null;
}
}
@@ -788,30 +838,11 @@ namespace dy.net.job
try
{
await douyinVideoService.BatchInsertOrUpdate(videos);
var redowns = await douyinCommonService.GetAllRedown();
if (redowns != null && redowns.Any())
{
//找出viedos和redowns重复的
var duplicateVideos = videos.Where(v => redowns.Any(r => r.ViedoId == v.AwemeId)).ToList();
if (duplicateVideos != null && duplicateVideos.Any())
{
var duplicateVideosIds = duplicateVideos.Select(x => x.AwemeId).ToList();
var downeds = redowns.Where(x => duplicateVideosIds.Contains(x.ViedoId))?.ToList();
foreach (var item in downeds)
{
item.Status = 1;
var v = videos.FirstOrDefault(x => x.AwemeId == item.ViedoId);
Log.Debug($"{JobType}-重新下载成功:-{v.VideoTitle}");
}
await douyinCommonService.UpdateRedownStatus(downeds);
}
}
return videos.Count;
}
catch (Exception ex)
{
Log.Error(ex, $"{JobType}-批量保存视频到数据库失败");
Log.Error(ex, $"{VideoType}-批量保存视频到数据库失败");
// 清理保存失败的视频文件
await CleanupFailedVideos(videos);
return 0;
@@ -938,7 +969,7 @@ namespace dy.net.job
/// <returns>一个表示异步操作的任务</returns>
private async Task CleanupFailedVideos(List<DouyinVideo> videos)
{
Log.Debug($"{JobType}-数据库保存失败,开始清理本次下载的视频目录...");
Log.Debug($"{VideoType}-数据库保存失败,开始清理本次下载的视频目录...");
foreach (var video in videos)
{
@@ -956,12 +987,12 @@ namespace dy.net.job
{
Directory.Delete(directory);
}
Log.Debug($"{JobType}-清理失败视频文件成功: {video.VideoSavePath}!!!");
Log.Debug($"{VideoType}-清理失败视频文件成功: {video.VideoSavePath}!!!");
}
catch (Exception ex)
{
Log.Warning(ex, $"{JobType}-清理失败视频文件出错: {video.VideoSavePath}!!!");
Log.Warning(ex, $"{VideoType}-清理失败视频文件出错: {video.VideoSavePath}!!!");
}
});
}
@@ -1015,7 +1046,7 @@ namespace dy.net.job
return new DouyinVideo
{
ViedoType = diffs.VideoType,
ViedoType = VideoType,
AwemeId = item.AwemeId,
Author = item.Author?.Nickname,
AuthorId = item.Author?.Uid,
@@ -1053,7 +1084,7 @@ namespace dy.net.job
/// <summary>
/// 视频类型
/// </summary>
public VideoTypeEnum VideoType { get; set; }
//public VideoTypeEnum VideoType { get; set; }
/// <summary>
/// 简化的视频标题
+2 -2
View File
@@ -84,10 +84,10 @@ namespace dy.net.job
if (follows.Count > 0)
{
await _followService.Sync(follows, ck.MyUserId);
await _followService.Sync(follows, ck );
}
Serilog.Log.Debug($"当前Cookie-[{ck.UserName}],本次同步关注列表完成,共同步关注{total}人。");
//Serilog.Log.Debug($"当前Cookie-[{ck.UserName}],本次同步关注列表完成,当前共关注{total}人。");
}
}
}
+10 -13
View File
@@ -16,9 +16,8 @@ namespace dy.net.job
{
}
protected override string JobType => SystemStaticUtil.DY_FOLLOWEDS;
protected override VideoTypeEnum VideoType => VideoTypeEnum.UperPost;
protected override VideoTypeEnum VideoType => VideoTypeEnum.dy_follows;
protected override async Task<List<DouyinCookie>> GetValidCookies()
{
@@ -56,7 +55,7 @@ namespace dy.net.job
protected override string CreateSaveFolder(DouyinCookie cookie, Aweme item, AppConfig config, DouyinFollowed followed)
{
#region 使UP主名称作为文件夹名称使
var authorName = string.IsNullOrWhiteSpace(item.Author?.Nickname) ? "UnknownAuthor" : DouyinFileNameHelper.SanitizePath(item.Author.Nickname);
var authorName = string.IsNullOrWhiteSpace(item.Author?.Nickname) ? "UnknownAuthor" : DouyinFileNameHelper.SanitizeLinuxFileName(item.Author.Nickname);
var folder = Path.Combine(cookie.UpSavePath, authorName);
if (!string.IsNullOrWhiteSpace(followed.SavePath))
{
@@ -71,7 +70,7 @@ namespace dy.net.job
}
else
{
var sampleName = DouyinFileNameHelper.GenerateFileName(item.Desc, item.AwemeId);
var sampleName = DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId);
var (existingName, _) = douyinVideoService.GetUperLastViedoFileName(item.Author.Uid, sampleName).Result;
var fileNameFolder = string.IsNullOrWhiteSpace(existingName) ? sampleName : existingName;
return Path.Combine(folder, fileNameFolder);
@@ -115,7 +114,7 @@ namespace dy.net.job
string fileName;
if (config?.UperUseViedoTitle ?? false)//优先
{
var sampleName = DouyinFileNameHelper.GenerateFileName(item.Desc, item.AwemeId);
var sampleName = DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId);
var (existingName, _) = douyinVideoService.GetUperLastViedoFileName(item.Author.Uid, sampleName).Result;
fileName = string.IsNullOrWhiteSpace(existingName) ? $"{sampleName}.{Format}" : $"{existingName}.{Format}";
}
@@ -130,11 +129,11 @@ namespace dy.net.job
Id = item.AwemeId,
ReleaseTime = DateTimeUtil.Convert10BitTimestamp(item.CreateTime),
Resolution = $"{Width}×{Height}",
VideoTitle = DouyinFileNameHelper.GenerateFileName(item.Desc, item.AwemeId),
VideoTitle = DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId),
Author = item.Author.Nickname
});
fileName= $"{DouyinFileNameHelper.SanitizePath(fullName)}.{Format}";
fileName= $"{DouyinFileNameHelper.SanitizeLinuxFileName(fullName)}.{Format}";
}
else
{
@@ -172,17 +171,17 @@ namespace dy.net.job
return Path.Combine(cookie.UpSavePath, "author");
}
protected override async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount)
protected override async Task HandleSyncCompletion(DouyinCookie cookie, int syncCount, DouyinFollowed followed)
{
if (syncCount > 0)
{
Serilog.Log.Debug($"{JobType}-Cookie-[{cookie.UserName}],本次同步成功{syncCount}条视频");
Serilog.Log.Debug($"{VideoType}-Cookie-[{cookie.UserName}],本次同步成功{syncCount}条视频");
cookie.UperSyncd = 1;
await douyinCookieService.UpdateAsync(cookie);
}
else
{
Serilog.Log.Debug($"{JobType}-Cookie-[{cookie.UserName}],本次没有查询到新的视频");
Serilog.Log.Debug($"{VideoType}-Cookie-[{cookie.UserName}]-{(followed == null ? "" : $"{followed.UperName}")},没有可以同步的新视频");
}
}
@@ -193,12 +192,10 @@ namespace dy.net.job
if (config?.UperUseViedoTitle ?? false)
{
simplifiedTitle = DouyinFileNameHelper.GenerateFileName(item.Desc, item.AwemeId);
simplifiedTitle = DouyinFileNameHelper.SanitizeLinuxFileName(item.Desc, item.AwemeId);
}
return new VideoEntityDifferences
{
VideoType = VideoTypeEnum.UperPost,
VideoTitleSimplify = simplifiedTitle
};
}