nanxun/douyin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1、增加图片视频

Browse Source 
2、优化代码结构
3、up主下载分类逻辑可配置
This commit is contained in:
jianzhichu
2025-11-23 00:46:41 +08:00
parent 7cacdaedd4
commit 8cef13c926
51 changed files with 1604 additions and 1667 deletions
Download Patch File Download Diff File Expand all files Collapse all files
job/DouYinCollectSyncJob.cs
+72 -342
View File
@@ -1,367 +1,97 @@
using ClockSnowFlake;
using dy.net.dto;
using dy.net.dto;
using dy.net.model;
using dy.net.service;
using dy.net.utils;
using Quartz;
namespace dy.net.job
{
[DisallowConcurrentExecution]
public class DouYinCollectSyncJob : IJob
public class DouyinCollectSyncJob : DouyinBaseSyncJob
{
private readonly DyCookieService _dyCookieService;
public DouyinCollectSyncJob(
DouyinCookieService dyCookieService,
DouyinHttpClientService dyHttpClientService,
DouyinVideoService dyCollectVideoService,
DouyinCommonService commonService,IServiceProvider serviceProvider,IWebHostEnvironment webHostEnvironment)
: base(dyCookieService, dyHttpClientService, dyCollectVideoService, commonService, serviceProvider,webHostEnvironment) { }
private readonly DyHttpClientService _douyinService;
protected override string JobType => "collect";
private readonly DyCollectVideoService _douyinVideoService;
private readonly CommonService commonService;
private readonly Random _random = new Random();
private string count = "18"; // 每页请求的视频数量，默认18
public DouYinCollectSyncJob(DyCookieService dyCookieService, DyHttpClientService dyHttpClientService, DyCollectVideoService dyCollectVideoService, CommonService commonService)
protected override async Task BeforeProcessCookies()
{
_dyCookieService = dyCookieService;
_douyinService = dyHttpClientService;
_douyinVideoService = dyCollectVideoService;
this.commonService = commonService;
var now = DateTime.Now;
if (now.Hour == 1 && now.Minute < 30)
{
_commonService.UpdateAllCookieSyncedToZero();
await Task.Delay(200);
}
}
public async Task Execute(IJobExecutionContext context)
protected override async Task<List<DouyinUserCookie>> GetValidCookies()
{
var config = commonService.GetConfig();
if (config == null)
{
Serilog.Log.Debug("collect-请先在设置中初始化配置，再执行同步任务");
return;
}
if (config.BatchCount > 0)
{
count = config.BatchCount.ToString();
}
var cookies = await _dyCookieService.GetAllCookies();
cookies= cookies.Where(c => !string.IsNullOrWhiteSpace(c.SavePath)).ToList();
if (cookies == null || !cookies.Any())
return cookies.Where(c => !string.IsNullOrWhiteSpace(c.SavePath)).ToList();
}
protected override bool IsCookieValid(DouyinUserCookie cookie)
{
return !string.IsNullOrWhiteSpace(cookie.Cookies) && cookie.Cookies.Length >= 1000 &&
!string.IsNullOrWhiteSpace(cookie.SavePath);
}
protected override async Task<DouyinVideoInfo> FetchVideoData(DouyinUserCookie cookie, string cursor)
{
return await _douyinService.SyncCollectVideos(cursor, count, cookie.Cookies);
}
protected override bool ShouldContinueSync(DouyinUserCookie cookie, DouyinVideoInfo data)
{
return data != null && data.HasMore == 1 && cookie.CollHasSyncd == 0;
}
protected override string GetNextCursor(DouyinVideoInfo data)
{
return data?.Cursor ?? "0";
}
protected override string CreateSaveFolder(DouyinUserCookie cookie, Aweme item, string tag1, string tag2)
{
var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : TikTokFileNameHelper.SanitizePath(tag1);
var folder = Path.Combine(cookie.SavePath, safeTag1, $"{TikTokFileNameHelper.SanitizePath(item.Desc)}@{item.AwemeId}");
if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
return folder;
}
protected override string GetVideoFileName(DouyinUserCookie cookie, Aweme item, VideoBitRate bitRate)
{
return $"{item.AwemeId}.{bitRate.Format}";
}
protected override string GetAuthorAvatarBasePath(DouyinUserCookie cookie)
{
return Path.Combine(cookie.SavePath, "author");
}
protected override async Task HandleSyncCompletion(DouyinUserCookie cookie, int syncCount)
{
if (syncCount > 0)
{
Serilog.Log.Debug("collect-无可用Cookie，任务终止");
return;
Serilog.Log.Debug($"{JobType}-Cookie-[{cookie.UserName}],本次同步成功{syncCount}条视频");
cookie.CollHasSyncd = 1;
await _dyCookieService.UpdateAsync(cookie);
}
else
{
//Serilog.Log.Debug($"collect-当前有{cookies.Count}个cookie开启了同步,即将开始同步");
//return;
}
//每天凌晨1点到1点半之间执行一次重置同步状态-全部cookie
UpdateCookieSyncedToZero();
foreach (var cookie in cookies)
{
Serilog.Log.Debug($"collect-开始同步 Cookie-[{cookie.UserName}]收藏视频");
if (string.IsNullOrWhiteSpace(cookie.Cookies) || cookie.Cookies.Length < 1000)
{
Serilog.Log.Debug($"collect-Cookie-[{cookie.UserName}]无效，跳过");
continue;
}
//if (string.IsNullOrWhiteSpace(cookie.SavePath))
//{
// Serilog.Log.Debug($"collect-Cookie-[{cookie.UserName}]未设置保存路径，跳过");
// continue;
//}
try
{
int syncCount = 0;// 记录本次Cookie同步的视频数量
int index = 0;
bool hasMore = true;
string cursor = "0";
while (hasMore)
{
var data = await _douyinService.SyncCollectVideos(cursor, count, cookie.Cookies);
hasMore = data != null && data.HasMore == 1 && cookie.CollHasSyncd == 0;
break;
if (cookie.CollHasSyncd == 1)
{
Serilog.Log.Debug($"collect-Cookie[{cookie.UserName}]已完整同步过，后续只获取最新一页数据");
}
//Serilog.Log.Debug($"还有数据需要同步吗？{(hasMore ? "YES" : "NO")}");
if (data == null)
{
Serilog.Log.Debug($"collect-Cookie[{cookie.UserName}]获取收藏数据失败，请检查一下Cookie");
break;
}
cursor = data != null && !string.IsNullOrWhiteSpace(data.Cursor) ? data.Cursor : "0";
if (data.AwemeList == null || !data.AwemeList.Any())
{
break;
}
List<DyCollectVideo> videos = new List<DyCollectVideo>();
foreach (var item in data.AwemeList)
{
if (item == null)
continue;
if (item.Video == null)
continue;
if (item.Video.BitRate == null)
continue;
var v = item.Video.BitRate.FirstOrDefault();
var tags = item.VideoTags;
if (v == null) continue;
var videoUrl = v.PlayAddr.UrlList != null && v.PlayAddr.UrlList.Any() ? v.PlayAddr.UrlList[0] : null;
if (string.IsNullOrWhiteSpace(videoUrl)) continue;
var tag1 = tags.FirstOrDefault(x => x.Level == 1)?.TagName;
var tag2 = tags.FirstOrDefault(x => x.Level == 2)?.TagName;
var tag3 = tags.FirstOrDefault(x => x.Level == 3)?.TagName;
string saveFolder = CreateSaveFolder(cookie, item, tag1, tag2);
var fileName = $"{item.AwemeId}.{v.Format}"; // 用ID做文件名，避免特殊字符
var savePath = Path.Combine(saveFolder, fileName);
if (!File.Exists(savePath))
{
Serilog.Log.Debug($"collect-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]开始下载");
await Task.Delay(_random.Next(1, 4) * 1000);
var downVideo = await _douyinService.DownloadAsync(videoUrl, savePath, cookie.Cookies);
if (downVideo)
{
Serilog.Log.Debug($"collect-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]下载{(downVideo ? "成功" : "失败")}");
}
else
{
Serilog.Log.Error($"collect-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]下载{(downVideo ? "成功" : "失败")}");
}
if (downVideo)
{
await DownVideoCover(item, saveFolder, cookie.Cookies);
// 用AuthorId做文件名，避免昵称特殊字符
var avatarImgName = $"{item.Author.Uid}.jpg";
var avatarSavePath = Path.Combine(cookie.SavePath, "author", avatarImgName);
await DownAuthorAvatar(cookie.SavePath, item, avatarSavePath, cookie.Cookies);
var avatarUrl = item.Author?.AvatarLarger?.UrlList != null && item.Author.AvatarLarger.UrlList.Any()
? item.Author.AvatarLarger.UrlList[0] : null;
// 备用头像链接
if (string.IsNullOrWhiteSpace(avatarUrl))
{
avatarUrl = item.Author?.AvatarThumb?.UrlList != null && item.Author.AvatarThumb.UrlList.Any()
? item.Author.AvatarThumb.UrlList[0] : null;
}
// 构造视频数据
DyCollectVideo video = new()
{
ViedoType = "2",
AwemeId = item.AwemeId,
Author = item.Author?.Nickname,
AuthorId = item.Author?.Uid,
AuthorAvatar = avatarSavePath,
AuthorAvatarUrl = avatarUrl,
CreateTime = DateTimeUtil.Convert10BitTimestamp(item.CreateTime),
VideoTitle = item.Desc,
Id = IdGener.GetLong().ToString(),
Resolution = $"{v.PlayAddr.Width}×{v.PlayAddr.Height}",
FileSize = v.PlayAddr.DataSize,
FileHash = v.PlayAddr.FileHash,
Tag1 = tag1,
Tag2 = tag2,
Tag3 = tag3,
VideoUrl = videoUrl,
VideoCoverUrl = item.Video.Cover.UrlList != null && item.Video.Cover.UrlList.Any()
? item.Video.Cover.UrlList[0] : null,
VideoSavePath = savePath,
VideoCoverSavePath = Path.Combine(saveFolder, "poster.jpg"),
SyncTime = DateTime.Now,
DyUserId = data.Uid,
CookieId = cookie.Id
};
videos.Add(video);
var nfoPath = Path.Combine(saveFolder, $"{item.AwemeId}.nfo");
NfoFileGenerator.GenerateNfoFile(new VideoNfo
{
Author = video.Author,
Poster = Path.Combine(saveFolder, "poster.jpg"),
Title = video.VideoTitle,
Thumbnail = Path.Combine(saveFolder, "fanart.jpg"),
ReleaseDate = video.CreateTime,
Genres = new List<string> { tag1, tag2, tag3 }.Where(t => !string.IsNullOrWhiteSpace(t)).ToList()
}, nfoPath);
}
}
else
{
//Serilog.Log.Debug($"视频[{item.AwemeId}]已存在，跳过下载");
}
}
index++;
// 批量保存到数据库（减少数据库操作频率）
if (videos.Any())
{
//Serilog.Log.Debug($"处理Cookie[{cookie.UserName}]的第{index}页数据，共{videos.Count}条");
try
{
await _douyinVideoService.batchInsert(videos);
syncCount += videos.Count;
}
catch (Exception ex)
{
Serilog.Log.Error($"collect-批量保存视频到数据库失败：{ex.Message}", ex);
// 收集所有需要删除的目录（去重）
foreach (var video in videos)
{
if (!string.IsNullOrWhiteSpace(video.VideoSavePath) && Directory.Exists(video.VideoSavePath))
{
var deletePath = Path.GetDirectoryName(video.VideoSavePath);
Directory.Delete(deletePath, recursive: true);
//Serilog.Log.Debug($"已删除失败视频目录：{deletePath}");
}
}
Serilog.Log.Debug("collect-因为数据库没有保存成功，本次下载的视频目录已尝试删除，将继续处理下一页数据");
}
}
else
{
//Serilog.Log.Debug($"没有查询到新的视频");
}
await Task.Delay(_random.Next(5, 10) * 1000);
}
if (syncCount > 0)
{
Serilog.Log.Debug($"collect-Cookie-[{cookie.UserName}],本次共同步成功{syncCount}条视频");
// 更新同步状态为已同步
cookie.CollHasSyncd = 1;
await _dyCookieService.UpdateAsync(cookie);
}
else
{
Serilog.Log.Debug($"collect-Cookie-[{cookie.UserName}],本次没有查询到新的视频");
}
}
catch (Exception ex)
{
Serilog.Log.Error($"collect-处理Cookie[{cookie.Id}]时出错：{ex.Message}");
Serilog.Log.Error($"collect-处理Cookie[{cookie.Id}]时出错22：{ex.StackTrace}");
}
Serilog.Log.Debug($"{JobType}-Cookie-[{cookie.UserName}],本次没有查询到新的视频");
}
}
/// <summary>
/// 下载视频封面
/// </summary>
/// <param name="item"></param>
/// <param name="saveFolder"></param>
/// <param name="cookie"></param>
/// <returns></returns>
private async Task DownVideoCover(Aweme item, string saveFolder, string cookie)
protected override VideoEntityDifferences GetVideoEntityDifferences(DouyinUserCookie cookie, Aweme item)
{
var coverUrl = item.Video.Cover.UrlList != null && item.Video.Cover.UrlList.Any()
? item.Video.Cover.UrlList[0] : null;
if (string.IsNullOrWhiteSpace(coverUrl)) return;
var coverImgName = "poster.jpg";
var coverSavePath = Path.Combine(saveFolder, coverImgName);
if (!File.Exists(coverSavePath))
return new VideoEntityDifferences
{
// 封面下载前随机延迟
var downRes = await _douyinService.DownloadAsync(coverUrl, coverSavePath, cookie);
if (downRes)
{
var copyPath = Path.Combine(saveFolder, "fanart.jpg");
File.Copy(coverSavePath, copyPath, true);
}
}
}
/// <summary>
/// 下载作者头像
/// </summary>
/// <param name="mainPath"></param>
/// <param name="item"></param>
/// <param name="avatarSavePath"></param>
/// <param name="cookie"></param>
/// <returns></returns>
private async Task DownAuthorAvatar(string mainPath, Aweme item, string avatarSavePath, string cookie)
{
if (item.Author == null) return;
var avatarUrl = item.Author.AvatarLarger?.UrlList != null && item.Author.AvatarLarger.UrlList.Any()
? item.Author.AvatarLarger.UrlList[0] : null;
if (string.IsNullOrWhiteSpace(avatarUrl)) return;
var path = Path.Combine(mainPath, "author");
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
if (!File.Exists(avatarSavePath))
{
// 头像下载前随机延迟
await _douyinService.DownloadAsync(avatarUrl, avatarSavePath, cookie);
}
}
/// <summary>
/// 创建目录
/// </summary>
/// <param name="cookie"></param>
/// <param name="item"></param>
/// <param name="tag1"></param>
/// <param name="tag2"></param>
/// <returns></returns>
private static string CreateSaveFolder(DyUserCookies cookie, Aweme item, string? tag1, string? tag2)
{
// 路径中避免特殊字符，用ID替代描述
var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : TikTokFileNameHelper.SanitizePath(tag1);
List<string> pathParts = new List<string> { cookie.SavePath, safeTag1 };
var saveFolder = Path.Combine(pathParts[0], string.Join("-", pathParts.Skip(1)), TikTokFileNameHelper.SanitizePath(item.Desc) + "@" + item.AwemeId);
// 创建文件夹（提前创建，避免下载时才操作）
if (!Directory.Exists(saveFolder))
{
Directory.CreateDirectory(saveFolder);
}
return saveFolder;
//if (string.IsNullOrWhiteSpace(tag2))
// return Path.Combine(pathParts[0], string.Join("-", pathParts.Skip(1)), SanitizePath(item.Desc) + "@" + item.AwemeId);
//else
// return Path.Combine(pathParts[0], string.Join("-", pathParts.Skip(1)), tag2 + "@" + item.AwemeId);
}
/// <summary>
/// 重置Cookie的同步状态为0（每天凌晨1点到1点半之间执行一次）
/// </summary>
/// <returns></returns>
private void UpdateCookieSyncedToZero()
{
var now = DateTime.Now;
//如果当时间为凌晨1点到1点半之间，则重置为0
if (now.Hour == 1 && now.Minute < 30)
{
commonService.UpdateAllCookieSyncedToZero();
}
VideoType = VideoTypeEnum.Collect,
};
}
}
}
}
job/DouYinFavoritSyncJob.cs
+65 -325
View File
@@ -1,349 +1,89 @@
using ClockSnowFlake;
using dy.net.dto;
using dy.net.dto;
using dy.net.model;
using dy.net.service;
using dy.net.utils;
using Quartz;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
namespace dy.net.job
{
[DisallowConcurrentExecution]
public class DouYinFavoritSyncJob : IJob
public class DouyinFavoritSyncJob : DouyinBaseSyncJob
{
private readonly DyCookieService _dyCookieService;
public DouyinFavoritSyncJob(
DouyinCookieService dyCookieService,
DouyinHttpClientService dyHttpClientService,
DouyinVideoService dyCollectVideoService,
DouyinCommonService commonService, IServiceProvider serviceProvider, IWebHostEnvironment webHostEnvironment)
: base(dyCookieService, dyHttpClientService, dyCollectVideoService, commonService, serviceProvider, webHostEnvironment) { }
private readonly DyHttpClientService _douyinService;
private readonly DyCollectVideoService _douyinVideoService;
private readonly CommonService commonService;
private readonly Random _random = new Random();
private string count = "18"; // 每页请求的视频数量，默认18
public DouYinFavoritSyncJob(DyCookieService dyCookieService, DyHttpClientService dyHttpClientService, DyCollectVideoService dyCollectVideoService,CommonService commonService)
protected override string JobType => "favorite";
protected override async Task<List<DouyinUserCookie>> GetValidCookies()
{
_dyCookieService = dyCookieService;
_douyinService = dyHttpClientService;
_douyinVideoService = dyCollectVideoService;
this.commonService = commonService;
var cookies = await _dyCookieService.GetAllCookies();
return cookies.Where(c => !string.IsNullOrWhiteSpace(c.FavSavePath)).ToList();
}
public async Task Execute(IJobExecutionContext context)
protected override bool IsCookieValid(DouyinUserCookie cookie)
{
return !string.IsNullOrWhiteSpace(cookie.Cookies) && cookie.Cookies.Length >= 1000 &&
!string.IsNullOrWhiteSpace(cookie.FavSavePath) &&
!string.IsNullOrWhiteSpace(cookie.SecUserId) && cookie.SecUserId.Length >= 10;
}
var config = commonService.GetConfig();
if (config == null)
protected override async Task<DouyinVideoInfo> FetchVideoData(DouyinUserCookie cookie, string cursor)
{
return await _douyinService.SyncFavoriteVideos(count, cursor, cookie.SecUserId, cookie.Cookies);
}
protected override bool ShouldContinueSync(DouyinUserCookie cookie, DouyinVideoInfo data)
{
return data != null && data.HasMore == 1 && cookie.FavHasSyncd == 0;
}
protected override string GetNextCursor(DouyinVideoInfo data)
{
return data?.MaxCursor ?? "0";
}
protected override string CreateSaveFolder(DouyinUserCookie cookie, Aweme item, string tag1, string tag2)
{
var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : TikTokFileNameHelper.SanitizePath(tag1);
var folder = Path.Combine(cookie.FavSavePath, safeTag1, $"{TikTokFileNameHelper.SanitizePath(item.Desc)}@{item.AwemeId}");
if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
return folder;
}
protected override string GetVideoFileName(DouyinUserCookie cookie, Aweme item, VideoBitRate bitRate)
{
return $"{item.AwemeId}.{bitRate.Format}";
}
protected override string GetAuthorAvatarBasePath(DouyinUserCookie cookie)
{
return Path.Combine(cookie.FavSavePath, "author");
}
protected override async Task HandleSyncCompletion(DouyinUserCookie cookie, int syncCount)
{
if (syncCount > 0)
{
Serilog.Log.Debug("favorite-请先在设置中初始化配置，再执行同步任务");
return;
}
if (config.BatchCount > 0)
{
count = config.BatchCount.ToString();
}
var cookies = await _dyCookieService.GetAllCookies();
cookies=cookies.Where(c => !string.IsNullOrWhiteSpace(c.FavSavePath)).ToList();
if (cookies == null || !cookies.Any())
{
Serilog.Log.Debug("favorite-无可用Cookie，任务终止");
return;
Serilog.Log.Debug($"{JobType}-Cookie-[{cookie.UserName}],本次同步成功{syncCount}条视频");
cookie.FavHasSyncd = 1;
await _dyCookieService.UpdateAsync(cookie);
}
else
{
Serilog.Log.Debug($"favorite-当前有{cookies.Count}个cookie开启了同步,即将开始同步");
//return;
}
foreach (var cookie in cookies)
{
Serilog.Log.Debug($"favorite-开始同步 Cookie-[{cookie.UserName}]喜欢的视频");
if (string.IsNullOrWhiteSpace(cookie.Cookies)|| cookie.Cookies.Length<1000)
{
Serilog.Log.Debug($"favorite-Cookie-[{cookie.UserName}]无效，跳过");
continue;
}
if(string.IsNullOrWhiteSpace(cookie.FavSavePath))
{
Serilog.Log.Debug($"favorite-Cookie-[{cookie.UserName}]未设置保存路径，跳过");
continue;
}
if (string.IsNullOrWhiteSpace(cookie.SecUserId)|| cookie.SecUserId.Length<10) {
Serilog.Log.Debug($"favorite-Cookie-[{cookie.UserName}]未设置SecUserId，跳过");
continue;
}
try
{
int syncCount = 0;// 记录本次Cookie同步的视频数量
int index = 0;
bool hasMore = true;
string cursor = "0";
while (hasMore)
{
var data = await _douyinService.SyncFavoriteVideos(count,cursor, cookie.SecUserId, cookie.Cookies);
hasMore = data != null && data.HasMore == 1 && cookie.FavHasSyncd == 0;
if (cookie.FavHasSyncd == 1)
{
Serilog.Log.Debug($"favorite-Cookie[{cookie.UserName}]已完整同步过，后续只获取最新一页数据");
}
//Serilog.Log.Debug($"还有数据需要同步吗？{(hasMore ? "YES" : "NO")}");
if (data == null)
{
Serilog.Log.Debug($"favorite-Cookie[{cookie.UserName}]获取喜欢的数据失败，请检查一下Cookie和sec_user_id");
break;
}
cursor = data != null && !string.IsNullOrWhiteSpace(data.MaxCursor) ? data.MaxCursor : "0";
if (data.AwemeList == null || !data.AwemeList.Any())
{
break;
}
List<DyCollectVideo> videos = new List<DyCollectVideo>();
foreach (var item in data.AwemeList)
{
if (item == null)
continue;
if (item.Video == null)
continue;
if (item.Video.BitRate == null)
continue;
var v = item.Video.BitRate.FirstOrDefault();
var tags = item.VideoTags;
if (v == null) continue;
var videoUrl = v.PlayAddr.UrlList != null && v.PlayAddr.UrlList.Any() ? v.PlayAddr.UrlList[0] : null;
if (string.IsNullOrWhiteSpace(videoUrl)) continue;
var tag1 = tags.FirstOrDefault(x => x.Level == 1)?.TagName;
var tag2 = tags.FirstOrDefault(x => x.Level == 2)?.TagName;
var tag3 = tags.FirstOrDefault(x => x.Level == 3)?.TagName;
string saveFolder = CreateSaveFolder(cookie, item, tag1, tag2);
var fileName = $"{item.AwemeId}.{v.Format}"; // 用ID做文件名，避免特殊字符
var savePath = Path.Combine(saveFolder, fileName);
if (!File.Exists(savePath))
{
Serilog.Log.Debug($"favorite-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]开始下载");
await Task.Delay(_random.Next(1, 4) * 1000);
var downVideo = await _douyinService.DownloadAsync(videoUrl, savePath, cookie.Cookies);
if (downVideo)
{
Serilog.Log.Debug($"favorite-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]下载{(downVideo ? "成功" : "失败")}");
}
else
{
Serilog.Log.Error($"favorite-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]下载{(downVideo ? "成功" : "失败")}");
}
if (downVideo)
{
await DownVideoCover(item, saveFolder, cookie.Cookies);
// 用AuthorId做文件名，避免昵称特殊字符
var avatarImgName = $"{item.Author.Uid}.jpg";
var avatarSavePath = Path.Combine(cookie.FavSavePath, "author", avatarImgName);
await DownAuthorAvatar(cookie.FavSavePath, item, avatarSavePath, cookie.Cookies);
var avatarUrl = item.Author?.AvatarLarger?.UrlList != null && item.Author.AvatarLarger.UrlList.Any()
? item.Author.AvatarLarger.UrlList[0] : null;
// 尝试使用AvatarThumb作为备用头像URL
if (string.IsNullOrWhiteSpace(avatarUrl))
{
avatarUrl = item.Author?.AvatarThumb?.UrlList != null && item.Author.AvatarThumb.UrlList.Any()
? item.Author.AvatarThumb.UrlList[0] : null;
}
// 构造视频数据
DyCollectVideo video = new()
{
ViedoType="1",
AwemeId = item.AwemeId,
Author = item.Author?.Nickname,
AuthorId = item.Author?.Uid,
AuthorAvatar = avatarSavePath,
AuthorAvatarUrl = avatarUrl,
CreateTime = DateTimeUtil.Convert10BitTimestamp(item.CreateTime),
VideoTitle = item.Desc,
Id = IdGener.GetLong().ToString(),
Resolution = $"{v.PlayAddr.Width}×{v.PlayAddr.Height}",
FileSize = v.PlayAddr.DataSize,
FileHash = v.PlayAddr.FileHash,
Tag1 = tag1,
Tag2 = tag2,
Tag3 = tag3,
VideoUrl = videoUrl,
VideoCoverUrl = item.Video.Cover.UrlList != null && item.Video.Cover.UrlList.Any()
? item.Video.Cover.UrlList[0] : null,
VideoSavePath = savePath,
VideoCoverSavePath = Path.Combine(saveFolder, "poster.jpg"),
SyncTime = DateTime.Now,
DyUserId = data.Uid,
CookieId = cookie.Id
};
videos.Add(video);
var nfoPath = Path.Combine(saveFolder, $"{item.AwemeId}.nfo");
NfoFileGenerator.GenerateNfoFile(new VideoNfo
{
Author = video.Author,
Poster = Path.Combine(saveFolder, "poster.jpg"),
Title = video.VideoTitle,
Thumbnail= Path.Combine(saveFolder, "fanart.jpg"),
ReleaseDate = video.CreateTime,
Genres = new List<string> { tag1, tag2, tag3 }.Where(t => !string.IsNullOrWhiteSpace(t)).ToList()
}, nfoPath);
}
}
else
{
//Serilog.Log.Debug($"视频[{item.AwemeId}]已存在，跳过下载");
}
}
index++;
// 批量保存到数据库（减少数据库操作频率）
if (videos.Any())
{
//Serilog.Log.Debug($"处理Cookie[{cookie.UserName}]的第{index}页数据，共{videos.Count}条");
try
{
await _douyinVideoService.batchInsert(videos);
syncCount += videos.Count;
}
catch (Exception ex)
{
Serilog.Log.Error("favorite-批量保存视频到数据库失败：{ex.Message}", ex);
// 收集所有需要删除的目录（去重）
foreach (var video in videos)
{
if (!string.IsNullOrWhiteSpace(video.VideoSavePath) && Directory.Exists(video.VideoSavePath))
{
var deletePath = Path.GetDirectoryName(video.VideoSavePath);
Directory.Delete(deletePath, recursive: true);
//Serilog.Log.Debug($"已删除失败视频目录：{deletePath}");
}
}
Serilog.Log.Debug("favorite-因为数据库没有保存成功，本次下载的视频目录已尝试删除，将继续处理下一页数据");
}
}
else {
//Serilog.Log.Debug($"没有查询到新的视频");
}
await Task.Delay(_random.Next(5, 10) * 1000);
}
if (syncCount > 0)
{
Serilog.Log.Debug($"Cookie-[{cookie.UserName}],本次共同步成功{syncCount}条视频");
// 更新同步状态为已同步
cookie.FavHasSyncd = 1;
await _dyCookieService.UpdateAsync(cookie);
}
else {
Serilog.Log.Debug($"favorite-Cookie-[{cookie.UserName}],本次没有查询到新的视频");
}
}
catch (Exception ex)
{
Serilog.Log.Error($"favorite-处理Cookie[{cookie.Id}]时出错：{ex.Message}");
Serilog.Log.Error($"favorite-处理Cookie[{cookie.Id}]时出错22：{ex.StackTrace}");
}
Serilog.Log.Debug($"{JobType}-Cookie-[{cookie.UserName}],本次没有查询到新的视频");
}
}
/// <summary>
/// 下载视频封面
/// </summary>
/// <param name="item"></param>
/// <param name="saveFolder"></param>
/// <param name="cookie"></param>
/// <returns></returns>
private async Task DownVideoCover(Aweme item, string saveFolder, string cookie)
protected override VideoEntityDifferences GetVideoEntityDifferences(DouyinUserCookie cookie, Aweme item)
{
var coverUrl = item.Video.Cover.UrlList != null && item.Video.Cover.UrlList.Any()
? item.Video.Cover.UrlList[0] : null;
if (string.IsNullOrWhiteSpace(coverUrl)) return;
var coverImgName = "poster.jpg";
var coverSavePath = Path.Combine(saveFolder, coverImgName);
if (!File.Exists(coverSavePath))
return new VideoEntityDifferences
{
// 封面下载前随机延迟
var downRes = await _douyinService.DownloadAsync(coverUrl, coverSavePath, cookie);
if (downRes)
{
var copyPath = Path.Combine(saveFolder, "fanart.jpg");
File.Copy(coverSavePath, copyPath, true);
}
}
VideoType = VideoTypeEnum.Favorite
};
}
/// <summary>
/// 下载作者头像
/// </summary>
/// <param name="mainPath"></param>
/// <param name="item"></param>
/// <param name="avatarSavePath"></param>
/// <param name="cookie"></param>
/// <returns></returns>
private async Task DownAuthorAvatar(string mainPath, Aweme item, string avatarSavePath, string cookie)
{
if (item.Author == null) return;
var avatarUrl = item.Author.AvatarLarger?.UrlList != null && item.Author.AvatarLarger.UrlList.Any()
? item.Author.AvatarLarger.UrlList[0] : null;
if (string.IsNullOrWhiteSpace(avatarUrl)) {
avatarUrl = item.Author.AvatarThumb?.UrlList != null && item.Author.AvatarThumb.UrlList.Any()
? item.Author.AvatarThumb.UrlList[0] : null;
}
if (string.IsNullOrWhiteSpace(avatarUrl)) return;
var path = Path.Combine(mainPath, "author");
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
if (!File.Exists(avatarSavePath))
{
// 头像下载前随机延迟
await _douyinService.DownloadAsync(avatarUrl, avatarSavePath, cookie);
}
}
/// <summary>
/// 创建目录
/// </summary>
/// <param name="cookie"></param>
/// <param name="item"></param>
/// <param name="tag1"></param>
/// <param name="tag2"></param>
/// <returns></returns>
private static string CreateSaveFolder(DyUserCookies cookie, Aweme item, string? tag1, string? tag2)
{
// 路径中避免特殊字符，用ID替代描述
var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : TikTokFileNameHelper.SanitizePath(tag1);
List<string> pathParts = new List<string> { cookie.FavSavePath, safeTag1 };
var saveFolder= Path.Combine(pathParts[0], string.Join("-", pathParts.Skip(1)), TikTokFileNameHelper.SanitizePath(item.Desc) + "@" + item.AwemeId);
// 创建文件夹（提前创建，避免下载时才操作）
if (!Directory.Exists(saveFolder))
{
Directory.CreateDirectory(saveFolder);
}
return saveFolder;
//if (string.IsNullOrWhiteSpace(tag2))
// return Path.Combine(pathParts[0], string.Join("-", pathParts.Skip(1)), SanitizePath(item.Desc) + "@" + item.AwemeId);
//else
// return Path.Combine(pathParts[0], string.Join("-", pathParts.Skip(1)), tag2 + "@" + item.AwemeId);
}
}
}
}
job/DouYinUperPostSyncJob.cs
+83 -353
View File
@@ -1,381 +1,111 @@
using ClockSnowFlake;
using dy.net.dto;
using dy.net.dto;
using dy.net.model;
using dy.net.service;
using dy.net.utils;
using Newtonsoft.Json;
using Quartz;
using SqlSugar.Extensions;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
namespace dy.net.job
{
[DisallowConcurrentExecution]
public class DouYinUperPostSyncJob : IJob
public class DouyinUperPostSyncJob : DouyinBaseSyncJob
{
private readonly DyCookieService _dyCookieService;
public DouyinUperPostSyncJob(
DouyinCookieService dyCookieService,
DouyinHttpClientService dyHttpClientService,
DouyinVideoService dyCollectVideoService,
DouyinCommonService commonService, IServiceProvider serviceProvider, IWebHostEnvironment webHostEnvironment)
: base(dyCookieService, dyHttpClientService, dyCollectVideoService, commonService, serviceProvider, webHostEnvironment) { }
private readonly DyHttpClientService _douyinService;
protected override string JobType => "dyuploder";
private readonly DyCollectVideoService _douyinVideoService;
private readonly CommonService commonService;
private readonly Random _random = new Random();
private string count = "18"; // 每页请求的视频数量，默认18
private readonly string httpDownName = "dy_down_uper";
public DouYinUperPostSyncJob(DyCookieService dyCookieService, DyHttpClientService dyHttpClientService, DyCollectVideoService dyCollectVideoService, CommonService commonService)
protected override async Task<List<DouyinUserCookie>> GetValidCookies()
{
_dyCookieService = dyCookieService;
_douyinService = dyHttpClientService;
_douyinVideoService = dyCollectVideoService;
this.commonService = commonService;
var cookies = await _dyCookieService.GetAllCookies();
return cookies.Where(x => !string.IsNullOrWhiteSpace(x.UpSecUserIds) && !string.IsNullOrWhiteSpace(x.UpSavePath)).ToList();
}
public async Task Execute(IJobExecutionContext context)
protected override bool IsCookieValid(DouyinUserCookie cookie)
{
var config = commonService.GetConfig();
if (config == null)
{
Serilog.Log.Debug("dyuploder-请先在设置中初始化配置，再执行同步任务");
return;
}
if (config.BatchCount > 0)
{
count = config.BatchCount.ToString();
}
return !string.IsNullOrWhiteSpace(cookie.Cookies) && !string.IsNullOrWhiteSpace(cookie.UpSavePath) && !string.IsNullOrWhiteSpace(cookie.UpSecUserIds);
}
var cookies = await _dyCookieService.GetAllCookies();
cookies = cookies.Where(x => !string.IsNullOrWhiteSpace(x.UpSecUserIds) && !string.IsNullOrWhiteSpace(x.UpSavePath))?.ToList();
if (!cookies.Any())
protected override async Task<DouyinVideoInfo> FetchVideoData(DouyinUserCookie cookie, string cursor)
{
// 简化处理：假设只同步第一个UP主
var ups = JsonConvert.DeserializeObject<List<DouyinUpSecUserIdDto>>(cookie.UpSecUserIds);
var firstUpId = ups?.FirstOrDefault()?.uid;
if (string.IsNullOrEmpty(firstUpId)) return null;
return await _douyinService.SyncUpderPostVideos(count, cursor, firstUpId, cookie.Cookies);
}
protected override bool ShouldContinueSync(DouyinUserCookie cookie, DouyinVideoInfo data)
{
return data != null && data.HasMore == 1 && cookie.UperSyncd == 0;
}
protected override string GetNextCursor(DouyinVideoInfo data)
{
return data?.MaxCursor ?? "0";
}
protected override string CreateSaveFolder(DouyinUserCookie cookie, Aweme item, string tag1, string tag2)
{
// UP主视频通常按作者名创建文件夹
var authorName = string.IsNullOrWhiteSpace(item.Author?.Nickname) ? "UnknownAuthor" : TikTokFileNameHelper.SanitizePath(item.Author.Nickname);
var folder = Path.Combine(cookie.UpSavePath, authorName);
if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
return folder;
}
protected override string GetVideoFileName(DouyinUserCookie cookie, Aweme item, VideoBitRate bitRate)
{
var config = _commonService.GetConfig();
if (config?.UperUseViedoTitle ?? false)
{
Serilog.Log.Debug("dyuploder-无可用cookie，任务终止");
return;
var sampleName = TikTokFileNameHelper.GenerateFileName(item.Desc, item.AwemeId);
var (existingName, _) = _douyinVideoService.GetUperLastViedoFileName(item.Author.Uid, sampleName).Result;
return string.IsNullOrWhiteSpace(existingName) ? $"{sampleName}.{bitRate.Format}" : $"{existingName}.{bitRate.Format}";
}
return $"{item.AwemeId}.{bitRate.Format}";
}
protected override string GetAuthorAvatarBasePath(DouyinUserCookie cookie)
{
return Path.Combine(cookie.UpSavePath, "author");
}
protected override async Task HandleSyncCompletion(DouyinUserCookie cookie, int syncCount)
{
if (syncCount > 0)
{
Serilog.Log.Debug($"{JobType}-Cookie-[{cookie.UserName}],本次同步成功{syncCount}条视频");
cookie.UperSyncd = 1;
await _dyCookieService.UpdateAsync(cookie);
}
else
{
//Serilog.Log.Debug($"dyuploder-当前有{cookies.Count}个cookie开启了同步,即将开始同步");
//return;
}
foreach (var cookie in cookies)
{
//if(string.IsNullOrWhiteSpace(cookie.UpSavePath))
//{
// Serilog.Log.Debug($"dyuploder-Cookie[{cookie.UserName}]的UP主保存路径无效，跳过");
// continue;
//}
var ups = JsonConvert.DeserializeObject<List<DyUpSecUserIdDto>>(cookie.UpSecUserIds);
foreach (var uper in ups)
{
Serilog.Log.Debug($"dyuploder-开始同步-[{uper.uper}]主页作品数据");
if (string.IsNullOrWhiteSpace(uper.uid) || uper.uid.Length < 10)
{
Serilog.Log.Debug($"dyuploder-[{uper.uper}]的SecUserId无效，跳过");
continue;
}
try
{
int syncCount = 0;// 记录本次Cookie同步的视频数量
int index = 0;
bool hasMore = true;
string cursor = "0";
while (hasMore)
{
var data = await _douyinService.SyncUpderPostVideos(count, cursor, uper.uid, cookie.Cookies);
hasMore = data != null && data.HasMore == 1 && cookie.UperSyncd == 0 && uper.syncAll;
if (cookie.UperSyncd == 1)
{
Serilog.Log.Debug($"dyuploder-Cookie[{cookie.UserName}]已完整同步过，后续只获取最新一页数据");
}
//Serilog.Log.Debug($"还有数据需要同步吗？{(hasMore ? "YES" : "NO")}");
if (data == null)
{
Serilog.Log.Debug($"dyuploder-[{uper.uper}]获取数据失败，请检查Up主SecUserId");
break;
}
cursor = data != null && !string.IsNullOrWhiteSpace(data.MaxCursor) ? data.MaxCursor : "0";
if (data.AwemeList == null || !data.AwemeList.Any())
{
break;
}
List<DyCollectVideo> videos = new List<DyCollectVideo>();
foreach (var item in data.AwemeList)
{
if (item == null)
continue;
if (item.Video == null)
continue;
if (item.Video.BitRate == null)
continue;
var v = item.Video.BitRate.FirstOrDefault();
var tags = item.VideoTags;
if (v == null) continue;
var videoUrl = v.PlayAddr.UrlList != null && v.PlayAddr.UrlList.Any() ? v.PlayAddr.UrlList[0] : null;
if (string.IsNullOrWhiteSpace(videoUrl)) continue;
var tag1 = tags.FirstOrDefault(x => x.Level == 1)?.TagName;
var tag2 = tags.FirstOrDefault(x => x.Level == 2)?.TagName;
var tag3 = tags.FirstOrDefault(x => x.Level == 3)?.TagName;
string saveFolder = CreateSaveFolder(cookie, uper.uper);
var fileName = $"{item.AwemeId}.{v.Format}"; // 用ID做文件名，避免特殊字符
string samplePrefix = null;
string sampleName = null;
if (config.UperUseViedoTitle)// 使用视频标题作为文件名
{
sampleName = TikTokFileNameHelper.GenerateFileName(item.Desc, item.AwemeId);
var sampleViedo = await _douyinVideoService.GetUperLastViedoFileName(item.Author.Uid, sampleName);
if (string.IsNullOrWhiteSpace(sampleViedo.Item1))
{
fileName = $"{sampleName}.{v.Format}";
}
else
{
fileName = $"{sampleViedo.Item1}.{v.Format}";
samplePrefix = sampleViedo.Item2;
}
}
var savePath = Path.Combine(saveFolder, fileName);
if (!File.Exists(savePath))
{
Serilog.Log.Debug($"dyuploder-[{uper.uper}]-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]开始下载");
await Task.Delay(_random.Next(1, 4) * 1000);
var downVideo = await _douyinService.DownloadAsync(videoUrl, savePath, cookie.Cookies);
if (downVideo)
{
Serilog.Log.Debug($"dyuploder-[{uper.uper}]-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]下载{(downVideo ? "成功" : "失败")}");
}
else
{
Serilog.Log.Error($"dyuploder-[{uper.uper}]-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]下载{(downVideo ? "成功" : "失败")}");
}
if (downVideo)
{
await DownVideoCover(item, saveFolder, cookie.Cookies);
// 用AuthorId做文件名，避免昵称特殊字符
var avatarImgName = $"{item.Author.Uid}.jpg";
var avatarSavePath = Path.Combine(cookie.UpSavePath, "author", avatarImgName);
await DownAuthorAvatar(cookie.UpSavePath, item, avatarSavePath, cookie.Cookies);
var avatarUrl = item.Author?.AvatarLarger?.UrlList != null && item.Author.AvatarLarger.UrlList.Any()
? item.Author.AvatarLarger.UrlList[0] : null;
// 有可能下载大头像失败，尝试用缩略图地址
if (string.IsNullOrWhiteSpace(avatarUrl))
{
avatarUrl = item.Author?.AvatarThumb?.UrlList != null && item.Author.AvatarThumb.UrlList.Any()
? item.Author.AvatarThumb.UrlList[0] : null;
}
// 构造视频数据
DyCollectVideo video = new()
{
ViedoType = "3",// UP主视频
AwemeId = item.AwemeId,
Author = item.Author?.Nickname,
AuthorId = item.Author?.Uid,
AuthorAvatar = avatarSavePath,
AuthorAvatarUrl = avatarUrl,
CreateTime = DateTimeUtil.Convert10BitTimestamp(item.CreateTime),
VideoTitle = item.Desc,
Id = IdGener.GetLong().ToString(),
Resolution = $"{v.PlayAddr.Width}×{v.PlayAddr.Height}",
FileSize = v.PlayAddr.DataSize,
FileHash = v.PlayAddr.FileHash,
Tag1 = tag1,
Tag2 = tag2,
Tag3 = tag3,
VideoUrl = videoUrl,
VideoCoverUrl = item.Video.Cover.UrlList != null && item.Video.Cover.UrlList.Any()
? item.Video.Cover.UrlList[0] : null,
VideoSavePath = savePath,
VideoCoverSavePath = Path.Combine(saveFolder, "poster.jpg"),
SyncTime = DateTime.Now,
DyUserId = data.Uid,
CookieId = cookie.Id,
};
if (!string.IsNullOrEmpty(sampleName))
{
video.VideoTitleSimplify = sampleName;
if (!string.IsNullOrEmpty(samplePrefix))
video.VideoTitleSimplifyPrefix = samplePrefix;
}
videos.Add(video);
var nfoPath = Path.Combine(saveFolder, $"{item.AwemeId}.nfo");
NfoFileGenerator.GenerateNfoFile(new VideoNfo
{
Author = video.Author,
Poster = Path.Combine(saveFolder, "poster.jpg"),
Title = video.VideoTitle,
Thumbnail = Path.Combine(saveFolder, "fanart.jpg"),
ReleaseDate = video.CreateTime,
Genres = new List<string> { tag1, tag2, tag3 }.Where(t => !string.IsNullOrWhiteSpace(t)).ToList()
}, nfoPath);
}
}
else
{
//Serilog.Log.Debug($"视频[{item.AwemeId}]已存在，跳过下载");
}
}
index++;
// 批量保存到数据库（减少数据库操作频率）
if (videos.Any())
{
//Serilog.Log.Debug($"处理Cookie[{cookie.UserName}]的第{index}页数据，共{videos.Count}条");
try
{
await _douyinVideoService.batchInsert(videos);
syncCount += videos.Count;
}
catch (Exception ex)
{
Serilog.Log.Error($"dyuploder-[{uper.uper}]-批量保存视频到数据库失败：{ex.Message}", ex);
// 收集所有需要删除的目录（去重）
foreach (var video in videos)
{
if (!string.IsNullOrWhiteSpace(video.VideoSavePath) && Directory.Exists(video.VideoSavePath))
{
var deletePath = Path.GetDirectoryName(video.VideoSavePath);
Directory.Delete(deletePath, recursive: true);
//Serilog.Log.Debug($"已删除失败视频目录：{deletePath}");
}
}
Serilog.Log.Debug($"dyuploder-[{uper.uper}]-因为数据库没有保存成功，本次下载的视频目录已尝试删除，将继续处理下一页数据");
}
}
else
{
//Serilog.Log.Debug($"没有查询到新的视频");
}
await Task.Delay(_random.Next(5, 10) * 1000);
}
if (syncCount > 0)
{
Serilog.Log.Debug($"dyuploder-[{uper.uper}],本次共同步成功{syncCount}条视频");
}
else
{
Serilog.Log.Debug($"dyuploder-[{uper.uper}],本次没有查询到新的视频");
}
}
catch (Exception ex)
{
Serilog.Log.Error($"dyuploder-[{uper.uper}]同步时出错：{ex.Message}");
Serilog.Log.Error($"dyuploder-[{uper.uper}]同步时出错222：{ex.StackTrace}");
}
}
// 更新为已同步过
cookie.UperSyncd = 1;
await _dyCookieService.UpdateAsync(cookie);
Serilog.Log.Debug($"{JobType}-Cookie-[{cookie.UserName}],本次没有查询到新的视频");
}
}
/// <summary>
/// 下载视频封面
/// </summary>
/// <param name="item"></param>
/// <param name="saveFolder"></param>
/// <param name="cookie"></param>
/// <returns></returns>
private async Task DownVideoCover(Aweme item, string saveFolder, string cookie)
protected override VideoEntityDifferences GetVideoEntityDifferences(DouyinUserCookie cookie, Aweme item)
{
var coverUrl = item.Video.Cover.UrlList != null && item.Video.Cover.UrlList.Any()
? item.Video.Cover.UrlList[0] : null;
if (string.IsNullOrWhiteSpace(coverUrl)) return;
var config = _commonService.GetConfig();
string simplifiedTitle = string.Empty;
var coverImgName = "poster.jpg";
var coverSavePath = Path.Combine(saveFolder, coverImgName);
if (!File.Exists(coverSavePath))
if (config?.UperUseViedoTitle ?? false)
{
// 封面下载前随机延迟
var downRes = await _douyinService.DownloadAsync(coverUrl, coverSavePath, cookie);
if (downRes)
{
var copyPath = Path.Combine(saveFolder, "fanart.jpg");
File.Copy(coverSavePath, copyPath, true);
}
simplifiedTitle = TikTokFileNameHelper.GenerateFileName(item.Desc, item.AwemeId);
}
return new VideoEntityDifferences
{
VideoType = VideoTypeEnum.UperPost,
VideoTitleSimplify = simplifiedTitle
};
}
/// <summary>
/// 下载作者头像
/// </summary>
/// <param name="mainPath"></param>
/// <param name="item"></param>
/// <param name="avatarSavePath"></param>
/// <param name="cookie"></param>
/// <returns></returns>
private async Task DownAuthorAvatar(string mainPath, Aweme item, string avatarSavePath, string cookie)
{
if (item.Author == null) return;
var avatarUrl = item.Author.AvatarLarger?.UrlList != null && item.Author.AvatarLarger.UrlList.Any()
? item.Author.AvatarLarger.UrlList[0] : null;
if (string.IsNullOrWhiteSpace(avatarUrl)) return;
var path = Path.Combine(mainPath, "author");
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
if (!File.Exists(avatarSavePath))
{
// 头像下载前随机延迟
await _douyinService.DownloadAsync(avatarUrl, avatarSavePath, cookie);
}
}
/// <summary>
/// 创建目录
/// </summary>
/// <param name="cookie"></param>
/// <param name="uperName"></param>
/// <returns></returns>
private static string CreateSaveFolder(DyUserCookies cookie, string uperName)
{
var saveFolder = Path.Combine(cookie.UpSavePath, uperName);
// 创建文件夹（提前创建，避免下载时才操作）
if (!Directory.Exists(saveFolder))
{
Directory.CreateDirectory(saveFolder);
}
return saveFolder;
// 路径中避免特殊字符，用ID替代描述
//var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : SanitizePath(tag1);
//List<string> pathParts = new List<string> { cookie.UpSavePath, safeTag1 };
//return Path.Combine(pathParts[0], string.Join("-", pathParts.Skip(1)), SanitizePath(item.Desc) + "@" + item.AwemeId);
//if (string.IsNullOrWhiteSpace(tag2))
// return Path.Combine(pathParts[0], string.Join("-", pathParts.Skip(1)), SanitizePath(item.Desc) + "@" + item.AwemeId);
//else
// return Path.Combine(pathParts[0], string.Join("-", pathParts.Skip(1)), tag2 + "@" + item.AwemeId);
}
}
}
}
job/DouyinBaseSyncJob.cs
+546
View File
@@ -0,0 +1,546 @@
using ClockSnowFlake;
using dy.image;
using dy.net.dto;
using dy.net.model;
using dy.net.service;
using dy.net.utils;
using Quartz;
using Quartz.Util;
namespace dy.net.job
{
[DisallowConcurrentExecution]
public abstract class DouyinBaseSyncJob : IJob
{
protected readonly DouyinCookieService _dyCookieService;
protected readonly DouyinHttpClientService _douyinService;
protected readonly DouyinVideoService _douyinVideoService;
protected readonly DouyinCommonService _commonService;
protected readonly Random _random = new Random();
protected readonly IServiceProvider _serviceProvider;
protected readonly IWebHostEnvironment _environment;
protected string count = "18"; // 每页请求的视频数量
protected abstract string JobType { get; }
private bool _downImageVideo;
/// <summary>
/// 清除保存失败的数据
/// </summary>
/// <param name="dyCookieService"></param>
/// <param name="dyHttpClientService"></param>
/// <param name="dyCollectVideoService"></param>
/// <param name="commonService"></param>
/// <param name="serviceProvider"></param>
/// <param name="webHostEnvironment"></param>
protected DouyinBaseSyncJob(
DouyinCookieService dyCookieService,
DouyinHttpClientService dyHttpClientService,
DouyinVideoService dyCollectVideoService,
DouyinCommonService commonService, IServiceProvider serviceProvider, IWebHostEnvironment webHostEnvironment)
{
_dyCookieService = dyCookieService;
_douyinService = dyHttpClientService;
_douyinVideoService = dyCollectVideoService;
_commonService = commonService;
_serviceProvider = serviceProvider;
_environment = webHostEnvironment;
}
/// <summary>
///
/// </summary>
/// <param name="context"></param>
/// <returns></returns>
public async Task Execute(IJobExecutionContext context)
{
var config = _commonService.GetConfig();
if (config == null)
{
Serilog.Log.Debug($"{JobType}-请先在设置中初始化配置，再执行同步任务");
return;
}
if (config.BatchCount > 0)
count = config.BatchCount.ToString();
// 读取环境变量覆盖配置中是否下载图片视频配置
InitializeDownImageVideoSetting(config);
await BeforeProcessCookies();
var cookies = await GetValidCookies();
if (cookies == null || !cookies.Any())
{
Serilog.Log.Debug($"{JobType}-无可用Cookie，任务终止");
return;
}
Serilog.Log.Debug($"{JobType}-当前有{cookies.Count}个cookie开启了同步，即将开始同步");
foreach (var cookie in cookies)
{
await ProcessSyncUserCookie(cookie);
}
}
/// <summary>
/// 检查并初始化下载图片视频的设置
/// </summary>
/// <param name="config"></param>
private void InitializeDownImageVideoSetting(AppConfig config)
{
var downImageVideoConfig = Appsettings.Get("DOWN_IMGVIDEO");
if (!string.IsNullOrWhiteSpace(downImageVideoConfig))
{
downImageVideoConfig = downImageVideoConfig.ToLower();
_downImageVideo = config.DownImageVideo && (downImageVideoConfig == "1" || downImageVideoConfig == "y" || downImageVideoConfig == "t" || downImageVideoConfig == "true");
}
}
/// <summary>
/// 检查并处理Cookie
/// </summary>
/// <param name="cookie"></param>
/// <returns></returns>
protected async Task ProcessSyncUserCookie(DouyinUserCookie cookie)
{
try
{
if (!IsCookieValid(cookie))
{
Serilog.Log.Debug($"{JobType}-Cookie[{cookie.UserName}]无效，跳过");
return;
}
Serilog.Log.Debug($"{JobType}-开始同步 Cookie-[{cookie.UserName}]");
int syncCount = 0;
string cursor = "0";
bool hasMore = true;
while (hasMore)
{
var data = await FetchVideoData(cookie, cursor);
if (data == null)
{
Serilog.Log.Debug($"{JobType}-Cookie[{cookie.UserName}]获取数据失败");
break;
}
hasMore = ShouldContinueSync(cookie, data);
cursor = GetNextCursor(data);
if (data.AwemeList == null || !data.AwemeList.Any())
break;
var videos = await ProcessVideoList(cookie, data);
syncCount += await SaveVideos(videos);
await Task.Delay(_random.Next(5, 10) * 1000);
}
await HandleSyncCompletion(cookie, syncCount);
}
catch (Exception ex)
{
Serilog.Log.Error(ex, $"{JobType}-处理Cookie[{cookie.Id}]时出错");
}
}
/// <summary>
/// 处理接口返回的视频数据
/// </summary>
/// <param name="cookie"></param>
/// <param name="data"></param>
/// <returns></returns>
protected async Task<List<DouyinVideo>> ProcessVideoList(DouyinUserCookie cookie, DouyinVideoInfo data)
{
var videos = new List<DouyinVideo>();
foreach (var item in data.AwemeList)
{
var video = await ProcessSingleVideo(cookie, item, data);
if (video != null)
videos.Add(video);
//如果配置了下载图片视频，则处理图片集并合成视频
if (_downImageVideo)
{
var mergevideo = await ProcessImageSetAndMergeToVideo(cookie, item, data);
if (mergevideo != null)
videos.Add(mergevideo);
}
}
return videos;
}
/// <summary>
/// 处理单个视频数据
/// </summary>
/// <param name="cookie"></param>
/// <param name="item"></param>
/// <param name="data"></param>
/// <returns></returns>
protected async Task<DouyinVideo> ProcessSingleVideo(DouyinUserCookie cookie, Aweme item, DouyinVideoInfo data)
{
if (!IsAwemeValid(item)) return null;
var v = item.Video.BitRate.FirstOrDefault();
if (v == null) return null;
var videoUrl = v.PlayAddr.UrlList?.FirstOrDefault();
if (string.IsNullOrWhiteSpace(videoUrl)) return null;
var (tag1, tag2, tag3) = GetVideoTags(item);
var saveFolder = CreateSaveFolder(cookie, item, tag1, tag2);
var fileName = GetVideoFileName(cookie, item, v);
var savePath = Path.Combine(saveFolder, fileName);
if (File.Exists(savePath)) return null;
Serilog.Log.Debug($"{JobType}-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]开始下载");
await Task.Delay(_random.Next(1, 4) * 1000);
if (!await _douyinService.DownloadAsync(videoUrl, savePath, cookie.Cookies))
{
Serilog.Log.Error($"{JobType}-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]下载失败");
return null;
}
await DownVideoCover(item, saveFolder, cookie.Cookies);
var (avatarSavePath, avatarUrl) = await DownAuthorAvatar(cookie, item);
await GenerateNfoFile(saveFolder, item, tag1, tag2, tag3, avatarSavePath, avatarUrl);
return CreateVideoEntity(cookie, item, v, savePath, saveFolder, tag1, tag2, tag3, avatarSavePath, avatarUrl, data);
}
/// <summary>
/// 创建视频实体
/// </summary>
/// <param name="cookie"></param>
/// <param name="item"></param>
/// <param name="bitRate"></param>
/// <param name="savePath"></param>
/// <param name="saveFolder"></param>
/// <param name="tag1"></param>
/// <param name="tag2"></param>
/// <param name="tag3"></param>
/// <param name="avatarSavePath"></param>
/// <param name="avatarUrl"></param>
/// <param name="data"></param>
/// <returns></returns>
protected DouyinVideo CreateVideoEntity(
DouyinUserCookie cookie, Aweme item, VideoBitRate bitRate, string savePath, string saveFolder,
string tag1, string tag2, string tag3, string avatarSavePath, string avatarUrl, DouyinVideoInfo data)
{
var diffs = GetVideoEntityDifferences(cookie, item);
return new DouyinVideo
{
ViedoType = diffs.VideoType.GetHashCode().ToString(),
AwemeId = item.AwemeId,
Author = item.Author?.Nickname,
AuthorId = item.Author?.Uid,
AuthorAvatar = avatarSavePath,
AuthorAvatarUrl = avatarUrl,
CreateTime = DateTimeUtil.Convert10BitTimestamp(item.CreateTime),
VideoTitle = item.Desc,
VideoTitleSimplify = diffs.VideoTitleSimplify,
Id = IdGener.GetLong().ToString(),
Resolution = $"{bitRate.PlayAddr.Width}×{bitRate.PlayAddr.Height}",
FileSize = bitRate.PlayAddr.DataSize,
FileHash = bitRate.PlayAddr.FileHash,
Tag1 = tag1,
Tag2 = tag2,
Tag3 = tag3,
VideoUrl = bitRate.PlayAddr.UrlList?.FirstOrDefault(),
VideoCoverUrl = item.Video.Cover.UrlList?.FirstOrDefault(),
VideoSavePath = savePath,
VideoCoverSavePath = Path.Combine(saveFolder, "poster.jpg"),
SyncTime = DateTime.Now,
DyUserId = item.AuthorUserId == 0 ? item.Author?.Uid : item.AuthorUserId.ToString(),
CookieId = cookie.Id
};
}
/// <summary>
/// 保存视频信息到数据库
/// </summary>
/// <param name="videos"></param>
/// <returns></returns>
protected async Task<int> SaveVideos(List<DouyinVideo> videos)
{
if (!videos.Any()) return 0;
try
{
await _douyinVideoService.batchInsert(videos);
return videos.Count;
}
catch (Exception ex)
{
Serilog.Log.Error(ex, $"{JobType}-批量保存视频到数据库失败");
await CleanupFailedVideos(videos);
return 0;
}
}
/// <summary>
/// 清理保存失败的视频文件
/// </summary>
/// <param name="videos"></param>
/// <returns></returns>
protected async Task CleanupFailedVideos(List<DouyinVideo> videos)
{
foreach (var video in videos)
{
if (!string.IsNullOrWhiteSpace(video.VideoSavePath) && Directory.Exists(Path.GetDirectoryName(video.VideoSavePath)))
{
Directory.Delete(Path.GetDirectoryName(video.VideoSavePath), recursive: true);
}
}
Serilog.Log.Debug($"{JobType}-因数据库保存失败，已清理本次下载的视频目录");
}
/// <summary>
/// 下载视频封面
/// </summary>
/// <param name="item"></param>
/// <param name="saveFolder"></param>
/// <param name="cookie"></param>
/// <returns></returns>
protected async Task DownVideoCover(Aweme item, string saveFolder, string cookie)
{
var coverUrl = item.Video.Cover.UrlList?.FirstOrDefault();
if (string.IsNullOrWhiteSpace(coverUrl)) return;
var coverSavePath = Path.Combine(saveFolder, "poster.jpg");
if (File.Exists(coverSavePath)) return;
if (await _douyinService.DownloadAsync(coverUrl, coverSavePath, cookie))
{
File.Copy(coverSavePath, Path.Combine(saveFolder, "fanart.jpg"), true);
}
}
/// <summary>
/// 下载图片合成的视频封面
/// </summary>
/// <param name="coverUrl"></param>
/// <param name="saveFolder"></param>
/// <param name="cookie"></param>
/// <returns></returns>
private async Task DownVideoCover(string coverUrl, string saveFolder, string cookie)
{
if (string.IsNullOrWhiteSpace(coverUrl)) return;
var coverImgName = "poster.jpg";
var coverSavePath = Path.Combine(saveFolder, coverImgName);
if (!File.Exists(coverSavePath))
{
// 封面下载前随机延迟
var downRes = await _douyinService.DownloadAsync(coverUrl, coverSavePath, cookie);
if (downRes)
{
var copyPath = Path.Combine(saveFolder, "fanart.jpg");
File.Copy(coverSavePath, copyPath, true);
}
}
}
/// <summary>
/// 下载作者头像
/// </summary>
/// <param name="cookie"></param>
/// <param name="item"></param>
/// <returns></returns>
protected async Task<(string savePath, string url)> DownAuthorAvatar(DouyinUserCookie cookie, Aweme item)
{
if (item.Author == null) return (null, null);
var avatarUrl = item.Author.AvatarLarger?.UrlList?.FirstOrDefault() ?? item.Author.AvatarThumb?.UrlList?.FirstOrDefault();
if (string.IsNullOrWhiteSpace(avatarUrl)) return (null, null);
var avatarSavePath = Path.Combine(GetAuthorAvatarBasePath(cookie), $"{item.Author.Uid}.jpg");
var avatarDir = Path.GetDirectoryName(avatarSavePath);
if (!Directory.Exists(avatarDir)) Directory.CreateDirectory(avatarDir);
if (!File.Exists(avatarSavePath))
{
await _douyinService.DownloadAsync(avatarUrl, avatarSavePath, cookie.Cookies);
}
return (avatarSavePath, avatarUrl);
}
/// <summary>
/// 处理图片集并合成为视频
/// </summary>
/// <param name="cookie"></param>
/// <param name="item"></param>
/// <param name="data"></param>
/// <returns></returns>
protected async Task<DouyinVideo> ProcessImageSetAndMergeToVideo(DouyinUserCookie cookie, Aweme item, DouyinVideoInfo data)
{
try
{
List<string> imageUrls = item.Images?
.Where(img => img.UrlList != null && img.UrlList.Any())
.Select(img => img.UrlList.FirstOrDefault())
.Where(url => !string.IsNullOrWhiteSpace(url))
.ToList();
if (imageUrls == null || !imageUrls.Any())
{
return null;
}
if (string.IsNullOrWhiteSpace(cookie.ImgSavePath))
{
Serilog.Log.Error($"{JobType}-图片视频同步-没有配置图片存储路径");
return null;
}
var imageService = _serviceProvider.GetService<ImageMergeToVideoService>();
if (imageService == null)
{
Serilog.Log.Error($"{JobType} -图片视频同步-无法创建 ImageMergeToVideoService。");
return null;
}
var fileNamefolder = Path.Combine(cookie.ImgSavePath, TikTokFileNameHelper.GenerateFileName(item.Desc, item.AwemeId));
if (!Directory.Exists(fileNamefolder)) Directory.CreateDirectory(fileNamefolder);
var savePath = Path.Combine(fileNamefolder, $"{item.AwemeId}.mp4");
if (File.Exists(savePath)) return null;
var mp3Url = item.Music?.PlayUrl?.UrlList?.FirstOrDefault();
var reqParams = new MediaMergeRequest
{
ImageDurationPerSecond = 3,
OutputFormat = "mp4",
VideoFps = 30,
AudioUrls = string.IsNullOrWhiteSpace(mp3Url) ? new List<string>() : new List<string> { mp3Url },
ImageUrls = imageUrls,
VideoWidth = 1080,
VideoHeight = 1920,
};
var mergeResult = await imageService.MergeToVideo(AppContext.BaseDirectory, reqParams, savePath,fileNamefolder);
if (!mergeResult)
{
Serilog.Log.Error($"{JobType}-图片视频同步-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]合成失败");
return null;
}
if (!File.Exists(savePath) || new FileInfo(savePath).Length <= 0)
{
Serilog.Log.Error($"{JobType}-图片视频同步-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]合成失败");
if (Directory.Exists(fileNamefolder))
{
File.Delete(savePath);
Directory.Delete(fileNamefolder, true);
Serilog.Log.Error($"{JobType}-图片视频同步-已删除合成失败的视频文件和目录");
}
return null;
}
// 复用基类的方法
await DownVideoCover(imageUrls.FirstOrDefault(), fileNamefolder, cookie.Cookies);
var (avatarSavePath, avatarUrl) = await DownAuthorAvatar(cookie, item);
// 为合成的视频创建一个“虚拟”的BitRate对象，以便复用CreateVideoEntity
var virtualBitRate = new VideoBitRate
{
PlayAddr = new PlayAddr
{
Width = reqParams.VideoWidth,
Height = reqParams.VideoHeight,
DataSize = new FileInfo(savePath).Length
}
};
var (tag1, tag2, tag3) = GetVideoTags(item);
await GenerateNfoFile(fileNamefolder, item, tag1, tag2, tag3, avatarSavePath, avatarUrl);
var videoEntity = CreateVideoEntity(
cookie, item, virtualBitRate, savePath, fileNamefolder,
tag1, tag2, tag3, avatarSavePath, avatarUrl, data);
// 特殊处理合成视频的字段
videoEntity.FileHash = string.Empty;
videoEntity.VideoUrl = "/"; // 合成视频没有原始URL
videoEntity.ViedoType= VideoTypeEnum.ImageVideo.ToString();
return videoEntity;
}
catch (Exception ex)
{
Serilog.Log.Error(ex, $"{JobType}-图片视频同步-处理图片集并合成视频时出错");
return null;
}
}
/// <summary>
/// 生成NFO文件
/// </summary>
protected async Task GenerateNfoFile(string saveFolder, Aweme item, string tag1, string tag2, string tag3,
string avatarSavePath, string avatarUrl)
{
var nfoPath = Path.Combine(saveFolder, $"{item.AwemeId}.nfo");
NfoFileGenerator.GenerateNfoFile(new DouyinVideoNfo
{
Actors = new List<Actor>
{
new() {
Name = item.Author?.Nickname,
Role = "主演",
Thumb = avatarSavePath
}
},
Author = item.Author?.Nickname,
Poster = Path.Combine(saveFolder, "poster.jpg"),
Title = item.Desc,
Thumbnail = Path.Combine(saveFolder, "fanart.jpg"),
ReleaseDate = DateTimeUtil.Convert10BitTimestamp(item.CreateTime),
Genres = new List<string> { tag1, tag2, tag3 }.Where(t => !string.IsNullOrWhiteSpace(t)).ToList()
}, nfoPath);
}
/// <summary>
/// 检查视频数据是否有效
/// </summary>
/// <param name="item"></param>
/// <returns></returns>
protected bool IsAwemeValid(Aweme item) => item != null && item.Video != null && item.Video.BitRate != null;
/// <summary>
/// 获取视频标签信息
/// </summary>
/// <param name="item"></param>
/// <returns></returns>
protected (string tag1, string tag2, string tag3) GetVideoTags(Aweme item)
{
var tags = item.VideoTags;
return (
tags?.FirstOrDefault(x => x.Level == 1)?.TagName,
tags?.FirstOrDefault(x => x.Level == 2)?.TagName,
tags?.FirstOrDefault(x => x.Level == 3)?.TagName
);
}
/// <summary>
/// AOP
/// </summary>
/// <returns></returns>
protected virtual Task BeforeProcessCookies() => Task.CompletedTask;
protected abstract Task<List<DouyinUserCookie>> GetValidCookies();
protected abstract bool IsCookieValid(DouyinUserCookie cookie);
protected abstract Task<DouyinVideoInfo> FetchVideoData(DouyinUserCookie cookie, string cursor);
protected abstract bool ShouldContinueSync(DouyinUserCookie cookie, DouyinVideoInfo data);
protected abstract string GetNextCursor(DouyinVideoInfo data);
protected abstract string CreateSaveFolder(DouyinUserCookie cookie, Aweme item, string tag1, string tag2);
protected abstract string GetVideoFileName(DouyinUserCookie cookie, Aweme item, VideoBitRate bitRate);
protected abstract string GetAuthorAvatarBasePath(DouyinUserCookie cookie);
protected abstract Task HandleSyncCompletion(DouyinUserCookie cookie, int syncCount);
protected abstract VideoEntityDifferences GetVideoEntityDifferences(DouyinUserCookie cookie, Aweme item);
}
public class VideoEntityDifferences
{
public VideoTypeEnum VideoType { get; set; }
public string VideoTitleSimplify { get; set; } = string.Empty;
}
}