nanxun/douyin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1、删除视频，永不下载

Browse Source 
2、去重-优先级
3、视频合成优化
4、其他优化
This commit is contained in:
jianzhichu
2025-12-03 23:43:10 +08:00
parent bd3a103f8f
commit 25e2a22dd2
29 changed files with 1023 additions and 702 deletions
Download Patch File Download Diff File Expand all files Collapse all files
service/DouyinCommonService.cs
+65 -18
View File
@@ -49,7 +49,7 @@ namespace dy.net.service
LogKeepDay = 10,
UperSaveTogether = false,//博主视频：true-->每个视频单独一个文件夹 false-->所有视频放在同一个文件夹
UperUseViedoTitle = false,//博主视频：true-->使用视频标题作为文件名 false-->使用视频id作为文件名
DownImageVideo = false,//默认不下载图文视频
DownImageVideo = true,//默认下载图文视频
DownMp3 = false,
DownImage = false,
ImageViedoSaveAlone = true,
@@ -99,18 +99,18 @@ namespace dy.net.service
}
/// <summary>
/// 兼容旧版将之前同步了的我收藏的数据进行更新
/// 初始化一些数据，兼容旧版数据结构。
/// </summary>
public void UpdateCollectViedoType()
{
//更新视频类型字段-兼容老版本
string sql = @"UPDATE dy_collect_video
SET ViedoType = CASE
WHEN ViedoType = '0' THEN 0
WHEN ViedoType = '1' THEN 1
WHEN ViedoType = '2' THEN 2
WHEN ViedoType = '3' THEN 3
WHEN ViedoType = '4' THEN 4
WHEN ViedoType = '4' THEN 4
ELSE NULL
END;";
@@ -119,16 +119,53 @@ namespace dy.net.service
//更新关注表的IsNoFollowed字段为空的数据为0--兼容老版本-新加的字段
string followUpdateSql = @"Update dy_follow SET IsNoFollowed=0 WHERE IsNoFollowed is NULL";
sqlSugarClient.Ado.ExecuteCommand(followUpdateSql);
//var collectViedos = sqlSugarClient.Queryable<DouyinVideo>().ToList();
//if (collectViedos.Any())
//{
// collectViedos.ForEach(x =>
// {
// x.ViedoType = x.;
// });
// sqlSugarClient.Updateable(collectViedos).ExecuteCommand();
//}
//更新图片视频的合并状态
string updateIsMergeVideoSql = @"UPDATE dy_collect_video SET IsMergeVideo = 1 WHERE IsMergeVideo IS NULL and ViedoType='4';";
sqlSugarClient.Ado.ExecuteCommand(updateIsMergeVideoSql);
//更新非图片视频的合并状态
string updateNoIsMergeVideoSql = @"UPDATE dy_collect_video SET IsMergeVideo = 0 WHERE IsMergeVideo IS NULL and ViedoType<>'4';";
sqlSugarClient.Ado.ExecuteCommand(updateNoIsMergeVideoSql);
//强制开启去重
sqlSugarClient.Updateable<AppConfig>().SetColumns(x => new AppConfig { AutoDistinct = true }).Where(x => !string.IsNullOrWhiteSpace(x.Id)).ExecuteCommand();
//重新根据保存路径更新图片视频的类型为喜欢，收藏或关注
var collectViedos = sqlSugarClient.Queryable<DouyinVideo>().Where(x=>x.ViedoType==VideoTypeEnum.ImageVideo).ToList();
if (collectViedos != null && collectViedos.Any())
{
var cookies= sqlSugarClient.Queryable<DouyinCookie>().ToList();
collectViedos.ForEach(x =>
{
var ck= cookies.FirstOrDefault(c => c.Id == x.CookieId);
if (ck != null)
{
var savePath = x.VideoSavePath;
if (savePath != null)
{
if (savePath.StartsWith(ck.SavePath))
{
x.ViedoType = VideoTypeEnum.dy_collects;
}
else if(savePath.StartsWith(ck.UpSavePath))
{
x.ViedoType = VideoTypeEnum.dy_follows;
}
else if(savePath.StartsWith(ck.FavSavePath))
{
x.ViedoType = VideoTypeEnum.dy_favorite;
}
else
{
x.ViedoType= VideoTypeEnum.dy_collects;
}
}
}
});
//只更新图片视频的类型
sqlSugarClient.Updateable(collectViedos).UpdateColumns(x => new DouyinVideo { ViedoType = x.ViedoType }).IgnoreColumns(true).ExecuteCommand();
}
}
/// <summary>
/// 重置所有Cookie的同步状态为0
@@ -148,18 +185,28 @@ namespace dy.net.service
}
/// <summary>
/// 查询需要下载的所有重下载视频记录
/// 查询是否已删除
/// </summary>
/// <param name="videoId"></param>
/// <returns></returns>
public async Task<List<ViedoReDown>> GetAllRedown()
public async Task<bool> ExistDeleteVideo(string videoId)
{
return await sqlSugarClient.Queryable<ViedoReDown>().Where(x => x.Status == 0 || x.Status == 2).ToListAsync();
var count= await sqlSugarClient.Queryable<DouyinVideoDelete>().Where(x=>x.ViedoId==videoId).CountAsync();
return count > 0;
}
public async Task<bool> UpdateRedownStatus(List<ViedoReDown> list)
/// <summary>
/// 新增要删除的视频
/// </summary>
/// <param name="dto"></param>
/// <returns></returns>
public async Task<bool> AddDeleteVideo(DouyinVideoDelete dto)
{
return await sqlSugarClient.Updateable(list).ExecuteCommandAsync() > 0;
dto.Id=IdGener.GetLong().ToString();
dto.DeleteTime = DateTime.Now;
return sqlSugarClient.Insertable(dto).ExecuteCommand() > 0;
}
#region 测试创建数据库
///// <summary>
service/DouyinFollowService.cs
+3 -3
View File
@@ -55,11 +55,11 @@ namespace dy.net.service
///
/// </summary>
/// <param name="followInfos"></param>
/// <param name="myselfUserId"></param>
/// <param name="ck"></param>
/// <returns></returns>
public async Task<bool> Sync(List<FollowingsItem> followInfos, string myselfUserId)
public async Task<bool> Sync(List<FollowingsItem> followInfos, DouyinCookie ck)
{
return await _followRepository.Sync(followInfos, myselfUserId);
return await _followRepository.Sync(followInfos, ck);
}
public async Task<DouyinFollowed> GetByUperId(string uperId,string myUid)
service/DouyinHttpClientService.cs
+108
View File
@@ -15,11 +15,14 @@ namespace dy.net.service
public static readonly string DouYinApi = "https://www.douyin.com/aweme/v1/web/aweme";
// 随机数生成器（避免重复实例化，保证随机性）
private readonly IHttpClientFactory _clientFactory;
// 下载信号量锁：初始计数1，最大并发1（同时只能一个下载任务）
private readonly SemaphoreSlim _downloadSemaphore = new SemaphoreSlim(1, 1);
public DouyinHttpClientService(IHttpClientFactory clientFactory)
{
_clientFactory = clientFactory;
}
/// <summary>
/// 查询用户收藏的视频
/// </summary>
@@ -371,6 +374,111 @@ namespace dy.net.service
}
}
/// <summary>
/// 下载文件并保存到本地（支持重试机制+单线程限制，同时只能一个下载）
/// </summary>
/// <param name="videoUrl">文件地址</param>
/// <param name="savePath">保存路径</param>
/// <param name="cookie">请求Cookie</param>
/// <param name="httpclientName">HttpClient名称（默认"dy_down1"）</param>
/// <param name="cancellationToken">取消令牌（用于终止任务）</param>
/// <param name="streamTimeout">流读取超时时间（默认60秒）</param>
/// <param name="maxRetryCount">最大重试次数（默认3次）</param>
/// <param name="initialRetryDelay">初始重试延迟（默认1秒，指数退避）</param>
/// <returns>是否下载成功</returns>
//public async Task<bool> DownloadAsync(
// string videoUrl,
// string savePath,
// string cookie,
// CancellationToken cancellationToken = default,
// TimeSpan? streamTimeout = null,
// int maxRetryCount = 3,
// TimeSpan? initialRetryDelay = null)
//{
// bool lockAcquired = false;
// try
// {
// // 申请锁：如果已有下载任务在执行，会阻塞等待直到锁释放
// // 传入cancellationToken支持取消等待
// await _downloadSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
// lockAcquired = true; // 标记锁已获取
// // 重试参数初始化
// int retryCount = 0;
// var retryDelay = initialRetryDelay ?? TimeSpan.FromSeconds(1);
// streamTimeout ??= TimeSpan.FromSeconds(60);
// while (true)
// {
// try
// {
// return await TryDownloadOnceAsync(
// videoUrl, savePath, cookie, cancellationToken, streamTimeout.Value);
// }
// catch (Exception ex) when (IsRetryableException(ex) && retryCount < maxRetryCount)
// {
// retryCount++;
// var delay = TimeSpan.FromMilliseconds(retryDelay.TotalMilliseconds * Math.Pow(2, retryCount - 1));
// Serilog.Log.Warning(ex, $"下载失败（第{retryCount}/{maxRetryCount}次重试）：{videoUrl}，将在{delay.TotalSeconds:F1}秒后重试");
// try
// {
// await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
// }
// catch (OperationCanceledException)
// {
// Serilog.Log.Information($"重试等待被取消：{videoUrl}");
// CleanupIncompleteFile(savePath);
// return false;
// }
// }
// catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
// {
// Serilog.Log.Information($"下载被取消：{videoUrl}");
// CleanupIncompleteFile(savePath);
// return false;
// }
// catch (Exception ex)
// {
// Serilog.Log.Error(ex, $"下载失败（不可重试）：{videoUrl}");
// CleanupIncompleteFile(savePath);
// return false;
// }
// }
// }
// finally
// {
// // 确保锁一定释放（无论是否发生异常）
// if (lockAcquired)
// {
// _downloadSemaphore.Release();
// Serilog.Log.Debug($"下载锁已释放，下一个任务可执行");
// }
// }
//}
// 辅助类：用于using语句自动释放SemaphoreSlim
//public sealed class SemaphoreReleaser : IDisposable
//{
// private readonly SemaphoreSlim _semaphore;
// private bool _disposed;
// public SemaphoreReleaser(SemaphoreSlim semaphore)
// {
// _semaphore = semaphore ?? throw new ArgumentNullException(nameof(semaphore));
// }
// public void Dispose()
// {
// if (!_disposed)
// {
// _semaphore.Release(); // 释放锁，允许下一个任务执行
// _disposed = true;
// }
// }
//}
/// <summary>
/// 单次下载尝试（核心下载逻辑）
/// </summary>
service/DouyinMergeVideoService.cs
+1 -1
View File
@@ -370,7 +370,7 @@ namespace dy.net.service
// 延迟清理：给 FFmpeg 进程足够时间释放文件句柄（1秒）
await Task.Delay(1000);
Directory.Delete(tempDir, recursive: true);
Log.Debug($"临时目录已清理：{tempDir}");
//Log.Debug($"临时目录已清理：{tempDir}");
}
catch (Exception ex)
{
service/DouyinQuartzJobService.cs
+277 -243
View File
@@ -1,6 +1,7 @@
using dy.net.dto;
using dy.net.job;
using Quartz;
using Quartz.Impl.Matchers;
using Serilog;
using System;
using System.Threading.Tasks;
@@ -13,242 +14,21 @@ namespace dy.net.service
public class DouyinQuartzJobService
{
private readonly ISchedulerFactory _schedulerFactory;
private const string DefaultJobGroup = "group1";
private const string DefaultJobGroup = "dysync.net";
private const int DefaultIntervalMinutes = 30;
private const int DefaultCronStartDelaySeconds = 30;
private const int DefaultSimpleStartDelaySeconds = 3;
public DouyinQuartzJobService(ISchedulerFactory schedulerFactory)
// 任务顺序依赖配置（核心：定义执行顺序，供 Listener 使用）
public Dictionary<string, string> JobDependency { get; } = new()
{
_schedulerFactory = schedulerFactory ?? throw new ArgumentNullException(nameof(schedulerFactory));
}
{"collect", "favorite"}, // collect 执行完 → 触发 favorite
{"favorite", "followed"}, // favorite 执行完 → 触发 followed
{"followed", null}, // followed 执行完 → 一轮任务结束
};
/// <summary>
/// 启动所有抖音相关定时任务
/// </summary>
/// <param name="expression">Cron表达式或间隔分钟数</param>
/// <param name="delayBetweenJobs">任务之间的启动延迟(毫秒)</param>
/// <returns>是否启动成功</returns>
public async Task<bool> InitOrReStartAllJobs(string expression, int delayBetweenJobs = 5000)
{
if (string.IsNullOrWhiteSpace(expression))
{
Log.Warning("定时任务表达式为空，使用默认配置");
expression = DefaultIntervalMinutes.ToString();
}
// 按顺序启动任务，避免并发
var jobTasks = new List<Task<bool>>
{
//关注列表
//StartJobAsync("follow_user", expression),
//我收藏的作品
StartJobAsync("collect", expression),
//我喜欢的作品
DelayAndStartJobAsync("favorite", expression, delayBetweenJobs),
//关注的用户的作品
DelayAndStartJobAsync("uper", expression, delayBetweenJobs * 2),
//关注列表
DelayAndStartJobAsync("follow_user", (Convert.ToInt32(expression)*2*24).ToString(), delayBetweenJobs * 3)
};
var results = await Task.WhenAll(jobTasks);
return results.All(success => success);
}
/// <summary>
/// 启动关注同步任务（单次执行）
/// </summary>
/// <returns>是否启动成功</returns>
public async Task<bool> StartFollowJobOnceAsync()
{
return await StartOneTimeJobAsync("follow_user_once");
}
/// <summary>
/// 延迟后启动任务
/// </summary>
private async Task<bool> DelayAndStartJobAsync(string jobKey, string expression, int delayMs)
{
if (delayMs > 0)
{
await Task.Delay(delayMs);
}
// 如果是数字表达式，自动递增避免并发
var adjustedExpression = AdjustExpressionForConcurrency(jobKey, expression);
return await StartJobAsync(jobKey, adjustedExpression);
}
/// <summary>
/// 调整任务表达式以避免并发
/// </summary>
private string AdjustExpressionForConcurrency(string jobKey, string expression)
{
if (!int.TryParse(expression, out int interval))
return expression;
// 根据任务类型递增间隔，避免所有任务同时执行
var jobIndex = _jobConfigs.Keys.ToList().IndexOf(jobKey);
return (interval + jobIndex).ToString();
}
/// <summary>
/// 启动指定定时任务
/// </summary>
private async Task<bool> StartJobAsync(string configKey, string expression)
{
if (!_jobConfigs.TryGetValue(configKey, out var jobConfig))
{
Log.Error("找不到任务配置: {ConfigKey}", configKey);
return false;
}
try
{
var scheduler = await _schedulerFactory.GetScheduler();
var jobKey = new JobKey(jobConfig.JobKey, DefaultJobGroup);
var triggerKey = new TriggerKey(jobConfig.TriggerKey, DefaultJobGroup);
// 删除已存在的任务
await RemoveExistingJobAsync(scheduler, jobKey);
// 创建任务详情
var jobDetail = JobBuilder.Create(jobConfig.JobType)
.WithIdentity(jobKey)
.WithDescription(jobConfig.Description)
.Build();
// 创建触发器
var trigger = CreateTrigger(triggerKey, expression, jobConfig.Description);
if (trigger == null)
{
Log.Error("创建触发器失败: {JobDescription}", jobConfig.Description);
return false;
}
// 调度任务
await scheduler.ScheduleJob(jobDetail, trigger);
Log.Information("启动定时任务成功 - {JobDescription}, 表达式: {Expression}",
jobConfig.Description, expression);
return true;
}
catch (Exception ex)
{
Log.Error(ex, "启动定时任务失败 - {JobDescription}", jobConfig.Description);
return false;
}
}
/// <summary>
/// 启动单次执行任务
/// </summary>
private async Task<bool> StartOneTimeJobAsync(string configKey)
{
if (!_jobConfigs.TryGetValue(configKey, out var jobConfig))
{
Log.Error("找不到任务配置: {ConfigKey}", configKey);
return false;
}
try
{
var scheduler = await _schedulerFactory.GetScheduler();
var jobKey = new JobKey(jobConfig.JobKey, DefaultJobGroup);
var triggerKey = new TriggerKey(jobConfig.TriggerKey, DefaultJobGroup);
// 删除已存在的任务
await RemoveExistingJobAsync(scheduler, jobKey);
// 创建任务详情
var jobDetail = JobBuilder.Create(jobConfig.JobType)
.WithIdentity(jobKey)
.WithDescription(jobConfig.Description)
.Build();
// 创建立即执行的触发器（只执行一次）
var trigger = TriggerBuilder.Create()
.WithIdentity(triggerKey)
.WithDescription($"{jobConfig.Description} - 单次执行")
.StartNow()
.Build();
// 调度任务
await scheduler.ScheduleJob(jobDetail, trigger);
Log.Information("启动单次任务成功 - {JobDescription}", jobConfig.Description);
return true;
}
catch (Exception ex)
{
Log.Error(ex, "启动单次任务失败 - {JobDescription}", jobConfig.Description);
return false;
}
}
/// <summary>
/// 创建触发器（支持Cron表达式和简单间隔）
/// </summary>
private ITrigger? CreateTrigger(TriggerKey triggerKey, string expression, string jobDescription)
{
// Cron表达式格式
if (CronExpression.IsValidExpression(expression))
{
return TriggerBuilder.Create()
.WithIdentity(triggerKey)
.WithDescription($"{jobDescription} - Cron调度")
.WithCronSchedule(expression)
.StartAt(DateTime.Now.AddSeconds(DefaultCronStartDelaySeconds))
.Build();
}
// 数字间隔格式（分钟）
if (int.TryParse(expression, out int intervalMinutes))
{
intervalMinutes = Math.Max(1, intervalMinutes); // 最小间隔1分钟
return TriggerBuilder.Create()
.WithIdentity(triggerKey)
.WithDescription($"{jobDescription} - 间隔{intervalMinutes}分钟")
.StartAt(DateTime.Now.AddSeconds(DefaultSimpleStartDelaySeconds))
.WithSimpleSchedule(x => x
.WithIntervalInMinutes(intervalMinutes)
.RepeatForever())
.Build();
}
// 无效表达式，使用默认配置
Log.Warning("无效的任务表达式: {Expression}，使用默认间隔{DefaultMinutes}分钟",
expression, DefaultIntervalMinutes);
return TriggerBuilder.Create()
.WithIdentity(triggerKey)
.WithDescription($"{jobDescription} - 默认间隔调度")
.StartAt(DateTime.Now.AddSeconds(DefaultSimpleStartDelaySeconds))
.WithSimpleSchedule(x => x
.WithIntervalInMinutes(DefaultIntervalMinutes)
.RepeatForever())
.Build();
}
/// <summary>
/// 移除已存在的任务
/// </summary>
private async Task RemoveExistingJobAsync(IScheduler scheduler, JobKey jobKey)
{
if (await scheduler.CheckExists(jobKey))
{
Log.Information("移除已存在的任务: {JobKey}", jobKey);
await scheduler.DeleteJob(jobKey);
}
}
/// <summary>
/// 任务配置信息
/// </summary>
private readonly Dictionary<string, JobConfig> _jobConfigs = new()
// 任务配置信息（保持原有配置不变，改为 public 供 Listener 访问）
public Dictionary<string, JobConfig> JobConfigs { get; } = new()
{
{
"collect",
@@ -267,11 +47,11 @@ namespace dy.net.service
"抖音点赞同步任务")
},
{
"uper",
"followed",
new JobConfig(
typeof(DouyinFollowedViedoSyncJob),
"dy.job.key.uper",
"dy.trigger.key.uper",
"dy.job.key.followed",
"dy.trigger.key.followed",
"抖音UP主作品同步任务")
},
{
@@ -290,16 +70,270 @@ namespace dy.net.service
"dy.trigger.key.follow_user_once",
"抖音关注同步任务(单次执行)")
}
,
//{
// "redown_once",
// new JobConfig(
// typeof(DouyinReDownSyncJob),
// "dy.job.key.redown_once",
// "dy.trigger.key.redown_once",
// "抖音重新下载任务(单次执行)")
//}
};
}
public DouyinQuartzJobService(ISchedulerFactory schedulerFactory)
{
_schedulerFactory = schedulerFactory ?? throw new ArgumentNullException(nameof(schedulerFactory));
}
/// <summary>
/// 启动所有抖音相关定时任务（顺序执行模式）
/// </summary>
/// <param name="expression">Cron表达式或间隔分钟数（控制整个链条的执行频率）</param>
/// <returns>是否启动成功</returns>
public async Task<bool> InitOrReStartAllJobs(string expression)
{
if (string.IsNullOrWhiteSpace(expression))
{
Log.Debug("定时任务表达式为空，使用默认配置（{DefaultMinutes}分钟）", DefaultIntervalMinutes);
expression = DefaultIntervalMinutes.ToString();
}
try
{
var scheduler = await _schedulerFactory.GetScheduler();
// 1. 注册独立的 JobListener（核心：注入配置和服务）
await RegisterJobListener(scheduler);
// 2. 移除所有已存在的任务（避免重复调度）
await RemoveAllExistingJobs(scheduler);
// 3. 只启动第一个任务（collect），后续任务由 Listener 自动触发
var firstJobConfigKey = "collect";
var startSuccess = await StartJobAsync(firstJobConfigKey, expression);
if (startSuccess)
{
Log.Debug("【任务服务】任务链条启动成功！执行顺序：collect → favorite → followed → follow_user", expression);
}
else
{
Log.Error("【任务服务】任务链条启动失败（第一个任务 {FirstJob} 启动失败）", firstJobConfigKey);
}
//启动follow_user--这个与其他几个任务没有依赖关系，所以单独启动
await StartJobAsync("follow_user", expression);
return startSuccess;
}
catch (Exception ex)
{
Log.Error(ex, "【任务服务】初始化任务链条异常");
return false;
}
}
/// <summary>
/// 启动关注同步任务（单次执行）
/// </summary>
public async Task<bool> StartFollowJobOnceAsync()
{
return await StartOneTimeJobAsync("follow_user_once");
}
/// <summary>
/// 注册独立的 JobListener（核心步骤）
/// </summary>
private async Task RegisterJobListener(IScheduler scheduler)
{
// 创建独立的 Listener 实例，注入依赖（任务配置、依赖关系、当前服务）
var dependencyListener = new DouyinJobDependencyListener(
JobConfigs, // 任务配置
JobDependency, // 依赖顺序
this // 任务服务（用于触发下一个任务）
);
// 注册 Listener：仅监听 DefaultJobGroup 分组的任务（精准匹配，避免影响其他任务）
scheduler.ListenerManager.AddJobListener(
dependencyListener,
GroupMatcher<JobKey>.GroupEquals(DefaultJobGroup)
);
Log.Information("【任务服务】JobListener 注册成功：{ListenerName}", dependencyListener.Name);
}
/// <summary>
/// 移除所有已存在的任务（避免重复调度）
/// </summary>
private async Task RemoveAllExistingJobs(IScheduler scheduler)
{
var jobKeys = JobConfigs.Values.Select(config => new JobKey(config.JobKey, DefaultJobGroup)).ToList();
foreach (var jobKey in jobKeys)
{
if (await scheduler.CheckExists(jobKey))
{
Log.Information("【任务服务】移除已存在的任务: {JobKey}", jobKey);
await scheduler.DeleteJob(jobKey);
}
}
}
/// <summary>
/// 启动指定定时任务（public 修饰，供 Listener 调用）
/// </summary>
/// <param name="configKey">任务配置Key（如：collect、favorite）</param>
/// <param name="expression">定时表达式（依赖触发时传空）</param>
/// <param name="isDependencyTrigger">是否为依赖触发（true=立即执行，false=定时执行）</param>
/// <returns>是否启动成功</returns>
public async Task<bool> StartJobAsync(string configKey, string expression, bool isDependencyTrigger = false)
{
if (!JobConfigs.TryGetValue(configKey, out var jobConfig))
{
Log.Error("【任务服务】找不到任务配置: {ConfigKey}", configKey);
return false;
}
try
{
var scheduler = await _schedulerFactory.GetScheduler();
var jobKey = new JobKey(jobConfig.JobKey, DefaultJobGroup);
// 触发器Key：区分「定时触发」和「依赖触发」，避免冲突
var triggerKey = new TriggerKey(
$"{jobConfig.TriggerKey}_{(isDependencyTrigger ? "dependency" : "main")}",
DefaultJobGroup
);
// 移除已存在的任务（防止重复执行）
await RemoveExistingJobAsync(scheduler, jobKey);
// 创建任务详情（添加禁止并发执行特性，避免顺序混乱）
var jobDetail = JobBuilder.Create(jobConfig.JobType)
.WithIdentity(jobKey)
.WithDescription(jobConfig.Description)
.DisallowConcurrentExecution() // 关键：禁止同一任务并发执行
.Build();
// 创建立触发器
ITrigger trigger = isDependencyTrigger
? CreateDependencyTrigger(triggerKey, jobConfig.Description) // 依赖触发：立即执行
: CreateScheduledTrigger(triggerKey, expression, jobConfig.Description); // 定时触发：按表达式执行
// 调度任务
await scheduler.ScheduleJob(jobDetail, trigger);
Log.Information("【任务服务】启动任务成功 - 任务描述: {JobDescription}, 触发类型: {TriggerType}, 表达式: {Expression}",
jobConfig.Description,
isDependencyTrigger ? "依赖触发（立即执行）" : "定时触发",
isDependencyTrigger ? "无" : expression);
return true;
}
catch (Exception ex)
{
Log.Error(ex, "【任务服务】启动任务失败 - 任务描述: {JobDescription}", jobConfig.Description);
return false;
}
}
/// <summary>
/// 启动单次执行任务（保持原有逻辑不变）
/// </summary>
private async Task<bool> StartOneTimeJobAsync(string configKey)
{
if (!JobConfigs.TryGetValue(configKey, out var jobConfig))
{
Log.Error("【任务服务】找不到任务配置: {ConfigKey}", configKey);
return false;
}
try
{
var scheduler = await _schedulerFactory.GetScheduler();
var jobKey = new JobKey(jobConfig.JobKey, DefaultJobGroup);
var triggerKey = new TriggerKey(jobConfig.TriggerKey, DefaultJobGroup);
await RemoveExistingJobAsync(scheduler, jobKey);
var jobDetail = JobBuilder.Create(jobConfig.JobType)
.WithIdentity(jobKey)
.WithDescription(jobConfig.Description)
.DisallowConcurrentExecution()
.Build();
var trigger = TriggerBuilder.Create()
.WithIdentity(triggerKey)
.WithDescription($"{jobConfig.Description} - 单次执行")
.StartNow()
.Build();
await scheduler.ScheduleJob(jobDetail, trigger);
Log.Information("【任务服务】启动单次任务成功 - 任务描述: {JobDescription}", jobConfig.Description);
return true;
}
catch (Exception ex)
{
Log.Error(ex, "【任务服务】启动单次任务失败 - 任务描述: {JobDescription}", jobConfig.Description);
return false;
}
}
/// <summary>
/// 创建「定时触发器」（按表达式执行，仅第一个任务使用）
/// </summary>
private ITrigger CreateScheduledTrigger(TriggerKey triggerKey, string expression, string jobDescription)
{
// Cron表达式格式
if (CronExpression.IsValidExpression(expression))
{
return TriggerBuilder.Create()
.WithIdentity(triggerKey)
.WithDescription($"{jobDescription} - Cron调度")
.WithCronSchedule(expression)
.StartAt(DateTime.Now.AddSeconds(DefaultCronStartDelaySeconds))
.Build();
}
// 数字间隔格式（分钟）
if (int.TryParse(expression, out int intervalMinutes))
{
intervalMinutes = Math.Max(1, intervalMinutes); // 最小间隔1分钟
return TriggerBuilder.Create()
.WithIdentity(triggerKey)
.WithDescription($"{jobDescription} - 间隔{intervalMinutes}分钟调度")
.StartAt(DateTime.Now.AddSeconds(DefaultSimpleStartDelaySeconds))
.WithSimpleSchedule(x => x
.WithIntervalInMinutes(intervalMinutes)
.RepeatForever())
.Build();
}
// 无效表达式，使用默认配置
Log.Warning("【任务服务】无效的任务表达式: {Expression}，使用默认间隔{DefaultMinutes}分钟",
expression, DefaultIntervalMinutes);
return TriggerBuilder.Create()
.WithIdentity(triggerKey)
.WithDescription($"{jobDescription} - 默认间隔调度")
.StartAt(DateTime.Now.AddSeconds(DefaultSimpleStartDelaySeconds))
.WithSimpleSchedule(x => x
.WithIntervalInMinutes(DefaultIntervalMinutes)
.RepeatForever())
.Build();
}
/// <summary>
/// 创建「依赖触发器」（立即执行，仅执行一次）
/// </summary>
private ITrigger CreateDependencyTrigger(TriggerKey triggerKey, string jobDescription)
{
return TriggerBuilder.Create()
.WithIdentity(triggerKey)
.WithDescription($"{jobDescription} - 依赖触发（立即执行）")
.StartNow() // 立即触发
.Build();
}
/// <summary>
/// 移除已存在的任务（保持原有逻辑不变）
/// </summary>
private async Task RemoveExistingJobAsync(IScheduler scheduler, JobKey jobKey)
{
if (await scheduler.CheckExists(jobKey))
{
Log.Information("【任务服务】移除已存在的任务: {JobKey}", jobKey);
await scheduler.DeleteJob(jobKey);
}
}
}
}
service/DouyinVideoService.cs
+33 -52
View File
@@ -22,6 +22,11 @@ namespace dy.net.service
}
public async Task<bool> DeleteById(string Id)
{
return await _dyCollectVideoRepository.DeleteByIdAsync(Id);
}
public async Task<bool> BatchInsertOrUpdate(List<DouyinVideo> videos)
{
// 边界处理：传入列表为空直接返回成功
@@ -99,22 +104,22 @@ namespace dy.net.service
CategoryCount = list.Select(x => x.Tag1).Distinct().Count(),
VideoCount = list.Count,
Categories = Categories,
FavoriteCount = list.Count(x => x.ViedoType == VideoTypeEnum.Favorite),
CollectCount = list.Count(x => x.ViedoType == VideoTypeEnum.Collect),
FollowCount = list.Count(x => x.ViedoType == VideoTypeEnum.UperPost),
GraphicVideoCount = list.Count(x => x.ViedoType == VideoTypeEnum.ImageVideo),
FavoriteCount = list.Count(x => x.ViedoType == VideoTypeEnum.dy_favorite),
CollectCount = list.Count(x => x.ViedoType == VideoTypeEnum.dy_collects),
FollowCount = list.Count(x => x.ViedoType == VideoTypeEnum.dy_follows),
GraphicVideoCount = list.Count(x => x.IsMergeVideo == 1),
VideoSizeTotal = ByteToGbConverter.ConvertBytesToGb(list.Sum(x => x.FileSize)),
VideoFavoriteSize = ByteToGbConverter.ConvertBytesToGb(list.Where(x => x.ViedoType == VideoTypeEnum.Favorite).Sum(x => x.FileSize)),
VideoCollectSize = ByteToGbConverter.ConvertBytesToGb(list.Where(x => x.ViedoType == VideoTypeEnum.Collect).Sum(x => x.FileSize)),
VideoFollowSize = ByteToGbConverter.ConvertBytesToGb(list.Where(x => x.ViedoType == VideoTypeEnum.UperPost).Sum(x => x.FileSize)),
GraphicVideoSize = ByteToGbConverter.ConvertBytesToGb(list.Where(x => x.ViedoType == VideoTypeEnum.ImageVideo).Sum(x => x.FileSize)),
VideoFavoriteSize = ByteToGbConverter.ConvertBytesToGb(list.Where(x => x.ViedoType == VideoTypeEnum.dy_favorite).Sum(x => x.FileSize)),
VideoCollectSize = ByteToGbConverter.ConvertBytesToGb(list.Where(x => x.ViedoType == VideoTypeEnum.dy_collects).Sum(x => x.FileSize)),
VideoFollowSize = ByteToGbConverter.ConvertBytesToGb(list.Where(x => x.ViedoType == VideoTypeEnum.dy_follows).Sum(x => x.FileSize)),
GraphicVideoSize = ByteToGbConverter.ConvertBytesToGb(list.Where(x => x.IsMergeVideo == 1).Sum(x => x.FileSize)),
//TotalDiskSize= ByteToGbConverter.GetHostTotalDiskSpaceGB(),
};
if (data.GraphicVideoSize == "0.00")
{
if (list.Where(x => x.ViedoType == VideoTypeEnum.ImageVideo).Sum(x => x.FileSize) > 0)
if (list.Where(x => x.IsMergeVideo == 1).Sum(x => x.FileSize) > 0)
{
data.GraphicVideoSize = "<0.01";//避免显示0.00误导用户
}
@@ -166,6 +171,8 @@ namespace dy.net.service
return await _dyCollectVideoRepository.GetByIdAsync(id);
}
/// <summary>
/// 重新下载选中的视频
/// </summary>
@@ -194,7 +201,7 @@ namespace dy.net.service
}
// 3. 构建重新下载记录（提前准备数据，避免事务内耗时操作）
var reDownList = new List<ViedoReDown>();
var reDownList = new List<DouyinReDownload>();
var filePathsToDelete = new List<string>(); // 收集待删除文件路径，统一处理
foreach (var video in videos)
@@ -207,7 +214,7 @@ namespace dy.net.service
}
// 构建重新下载记录
reDownList.Add(new ViedoReDown
reDownList.Add(new DouyinReDownload
{
Id = IdGener.GetLong().ToString(),
CreateTime = DateTime.UtcNow, // 统一使用UTC时间，避免时区问题
@@ -233,7 +240,7 @@ namespace dy.net.service
var transactionResult = await _dyCollectVideoRepository.UseTranAsync(async () =>
{
// 4.1 批量插入重新下载记录（SqlSugar批量插入效率更高）
_dyCollectVideoRepository.InsertReDowns(reDownList);
_dyCollectVideoRepository.InsertReDowns(reDownList);
// 4.2 批量删除原视频记录（使用视频实际存在的ID，避免无效删除）
var actualDeleteIds = videos.Select(v => v.Id).ToList();
var deleteCount = await _dyCollectVideoRepository.DeleteByIdsAsync(actualDeleteIds); // 建议仓储层提供异步删除方法
@@ -242,75 +249,49 @@ namespace dy.net.service
Serilog.Log.Error(e, "数据库事务执行失败：Ids={0}", string.Join(",", videoIds));
});
// 5. 文件夹删除（非事务操作，失败不回滚数据库，可根据业务调整）
// 关键：先通过文件路径获取父文件夹，再删除整个文件夹（含子内容）
foreach (var filePath in filePathsToDelete)
// 5. 文件删除（非事务操作，失败不回滚数据库，可根据业务调整）
// 采用异步文件操作，避免同步IO阻塞线程（需.NET 5+支持）
foreach (var path in filePathsToDelete)
{
try
{
// 1. 验证文件路径有效性
if (string.IsNullOrWhiteSpace(filePath))
if (File.Exists(path))
{
Serilog.Log.Error("文件路径为空，跳过文件夹删除");
continue;
}
// 2. 获取文件对应的父文件夹路径（无论文件是否存在，只要路径合法就能拿到父目录）
string parentDirPath = Path.GetDirectoryName(filePath);
if (string.IsNullOrWhiteSpace(parentDirPath))
{
Serilog.Log.Error("无法获取父文件夹路径，文件路径无效：Filepath={0}", filePath);
continue;
}
// 3. 验证父文件夹是否存在
if (Directory.Exists(parentDirPath))
{
Directory.Delete(parentDirPath, recursive: true);
Serilog.Log.Debug("文件夹删除成功（含子内容）：ParentDir={0}，关联文件路径={1}", parentDirPath, filePath);
File.Delete(path); // 异步删除，提升并发性能
Serilog.Log.Debug("视频文件删除成功：Path={0}", path);
}
else
{
Serilog.Log.Error("父文件夹不存在，跳过删除：ParentDir={0}，关联文件路径={1}", parentDirPath, filePath);
Serilog.Log.Error("视频文件不存在，跳过删除：Path={0}", path);
}
}
catch (IOException ex)
{
Serilog.Log.Error(ex, "文件夹删除失败（IO异常）：Filepath={0}，父文件夹路径={1}", filePath, Path.GetDirectoryName(filePath));
}
catch (UnauthorizedAccessException ex)
{
// 单独捕获权限异常，更精准的日志提示
Serilog.Log.Error(ex, "文件夹删除失败（权限不足）：Filepath={0}，父文件夹路径={1}", filePath, Path.GetDirectoryName(filePath));
}
catch (Exception ex)
{
// 捕获所有其他异常，避免循环中断
Serilog.Log.Error(ex, "文件夹删除失败（未知异常）：Filepath={0}，父文件夹路径={1}", filePath, Path.GetDirectoryName(filePath));
Serilog.Log.Error(ex, "视频文件删除失败：Path={0}", path);
}
}
var CookieIds = reDownList.Select(x => x.CookieId).Distinct();
foreach (var ck in CookieIds)
{
var cookie= douyinCookieRepository.GetById(ck);
var cookie = douyinCookieRepository.GetById(ck);
if (cookie == null)
continue;
var viedoTypes = videos.Where(x => x.CookieId == ck).Select(x => x.ViedoType).Distinct();
if(viedoTypes!=null&& viedoTypes.Any())
if (viedoTypes != null && viedoTypes.Any())
{
foreach (VideoTypeEnum item in viedoTypes)
{
switch (item)
{
case VideoTypeEnum.Favorite:
case VideoTypeEnum.dy_favorite:
cookie.FavHasSyncd = 0;
break;
case VideoTypeEnum.Collect:
case VideoTypeEnum.dy_collects:
cookie.CollHasSyncd = 0;
break;
case VideoTypeEnum.UperPost:
case VideoTypeEnum.dy_follows:
cookie.UperSyncd = 0;
break;
case VideoTypeEnum.ImageVideo:
@@ -338,7 +319,7 @@ namespace dy.net.service
/// 获取待重新下载的视频列表
/// </summary>
/// <returns></returns>
public async Task<List<ViedoReDown>> GetViedoReDowns()
public async Task<List<DouyinReDownload>> GetViedoReDowns()
{
return await _dyCollectVideoRepository.GetViedoReDowns();
}