nanxun/douyin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

博主的视频 增加配置 用标题做文件名，但是要对标题进行处理后才可以做文件名。另外对于可能重复的标题进行处理

Browse Source
This commit is contained in:
jianzhichu
2025-11-18 22:38:25 +08:00
parent 8dcda7877e
commit 8048f7e1e6
8 changed files with 284 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
job/DouYinCollectSyncJob.cs
+5 -22
View File
@@ -129,17 +129,17 @@ namespace dy.net.job
if (!File.Exists(savePath))
{
Serilog.Log.Debug($"collect-视频[{SanitizePath(item.Desc)}]开始下载");
Serilog.Log.Debug($"collect-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]开始下载");
await Task.Delay(_random.Next(1, 4) * 1000);
var downVideo = await _douyinService.DownloadAsync(videoUrl, savePath, cookie.Cookies);
if (downVideo)
{
Serilog.Log.Debug($"collect-视频[{SanitizePath(item.Desc)}]下载{(downVideo ? "成功" : "失败")}");
Serilog.Log.Debug($"collect-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]下载{(downVideo ? "成功" : "失败")}");
}
else
{
Serilog.Log.Error($"collect-视频[{SanitizePath(item.Desc)}]下载{(downVideo ? "成功" : "失败")}");
Serilog.Log.Error($"collect-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]下载{(downVideo ? "成功" : "失败")}");
}
if (downVideo)
@@ -332,9 +332,9 @@ namespace dy.net.job
private static string CreateSaveFolder(DyUserCookies cookie, Aweme item, string? tag1, string? tag2)
{
// 路径中避免特殊字符，用ID替代描述
var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : SanitizePath(tag1);
var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : TikTokFileNameHelper.SanitizePath(tag1);
List<string> pathParts = new List<string> { cookie.SavePath, safeTag1 };
var saveFolder = Path.Combine(pathParts[0], string.Join("-", pathParts.Skip(1)), SanitizePath(item.Desc) + "@" + item.AwemeId);
var saveFolder = Path.Combine(pathParts[0], string.Join("-", pathParts.Skip(1)), TikTokFileNameHelper.SanitizePath(item.Desc) + "@" + item.AwemeId);
// 创建文件夹（提前创建，避免下载时才操作）
if (!Directory.Exists(saveFolder))
@@ -348,23 +348,6 @@ namespace dy.net.job
// return Path.Combine(pathParts[0], string.Join("-", pathParts.Skip(1)), tag2 + "@" + item.AwemeId);
}
// 清理路径中的特殊字符（避免创建文件夹失败）
private static string SanitizePath(string path)
{
if (string.IsNullOrWhiteSpace(path))
{
path = "其他";
}
foreach (var c in Path.GetInvalidFileNameChars())
{
path = path.Replace(c, '_');
}
if (path.Length > 50)
{
path = path.Substring(0, 50);
}
return path.Trim();
}
/// <summary>
/// 重置Cookie的同步状态为0（每天凌晨1点到1点半之间执行一次）
job/DouYinFavoritSyncJob.cs
+5 -22
View File
@@ -124,17 +124,17 @@ namespace dy.net.job
if (!File.Exists(savePath))
{
Serilog.Log.Debug($"favorite-视频[{SanitizePath(item.Desc)}]开始下载");
Serilog.Log.Debug($"favorite-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]开始下载");
await Task.Delay(_random.Next(1, 4) * 1000);
var downVideo = await _douyinService.DownloadAsync(videoUrl, savePath, cookie.Cookies);
if (downVideo)
{
Serilog.Log.Debug($"favorite-视频[{SanitizePath(item.Desc)}]下载{(downVideo ? "成功" : "失败")}");
Serilog.Log.Debug($"favorite-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]下载{(downVideo ? "成功" : "失败")}");
}
else
{
Serilog.Log.Error($"favorite-视频[{SanitizePath(item.Desc)}]下载{(downVideo ? "成功" : "失败")}");
Serilog.Log.Error($"favorite-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]下载{(downVideo ? "成功" : "失败")}");
}
if (downVideo)
{
@@ -329,9 +329,9 @@ namespace dy.net.job
private static string CreateSaveFolder(DyUserCookies cookie, Aweme item, string? tag1, string? tag2)
{
// 路径中避免特殊字符，用ID替代描述
var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : SanitizePath(tag1);
var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : TikTokFileNameHelper.SanitizePath(tag1);
List<string> pathParts = new List<string> { cookie.FavSavePath, safeTag1 };
var saveFolder= Path.Combine(pathParts[0], string.Join("-", pathParts.Skip(1)), SanitizePath(item.Desc) + "@" + item.AwemeId);
var saveFolder= Path.Combine(pathParts[0], string.Join("-", pathParts.Skip(1)), TikTokFileNameHelper.SanitizePath(item.Desc) + "@" + item.AwemeId);
// 创建文件夹（提前创建，避免下载时才操作）
if (!Directory.Exists(saveFolder))
@@ -345,22 +345,5 @@ namespace dy.net.job
// return Path.Combine(pathParts[0], string.Join("-", pathParts.Skip(1)), tag2 + "@" + item.AwemeId);
}
// 清理路径中的特殊字符（避免创建文件夹失败）
private static string SanitizePath(string path)
{
if (string.IsNullOrWhiteSpace(path))
{
path = "其他";
}
foreach (var c in Path.GetInvalidFileNameChars())
{
path = path.Replace(c, '_');
}
if (path.Length > 50)
{
path = path.Substring(0, 50);
}
return path.Trim();
}
}
}
job/DouYinUperPostSyncJob.cs
+41 -30
View File
@@ -5,6 +5,7 @@ using dy.net.service;
using dy.net.utils;
using Newtonsoft.Json;
using Quartz;
using SqlSugar.Extensions;
namespace dy.net.job
{
@@ -22,7 +23,7 @@ namespace dy.net.job
private readonly string httpDownName = "dy_down_uper";
public DouYinUperPostSyncJob(DyCookieService dyCookieService, DyHttpClientService dyHttpClientService, DyCollectVideoService dyCollectVideoService,CommonService commonService)
public DouYinUperPostSyncJob(DyCookieService dyCookieService, DyHttpClientService dyHttpClientService, DyCollectVideoService dyCollectVideoService, CommonService commonService)
{
_dyCookieService = dyCookieService;
_douyinService = dyHttpClientService;
@@ -44,7 +45,7 @@ namespace dy.net.job
}
var cookies = await _dyCookieService.GetAllCookies();
cookies = cookies.Where(x => !string.IsNullOrWhiteSpace(x.UpSecUserIds)&&!string.IsNullOrWhiteSpace(x.UpSavePath))?.ToList();
cookies = cookies.Where(x => !string.IsNullOrWhiteSpace(x.UpSecUserIds) && !string.IsNullOrWhiteSpace(x.UpSavePath))?.ToList();
if (!cookies.Any())
{
Serilog.Log.Debug("dyuploder-无可用cookie，任务终止");
@@ -55,7 +56,7 @@ namespace dy.net.job
//Serilog.Log.Debug($"dyuploder-当前有{cookies.Count}个cookie开启了同步,即将开始同步");
//return;
}
foreach (var cookie in cookies)
{
//if(string.IsNullOrWhiteSpace(cookie.UpSavePath))
@@ -63,7 +64,7 @@ namespace dy.net.job
// Serilog.Log.Debug($"dyuploder-Cookie[{cookie.UserName}]的UP主保存路径无效，跳过");
// continue;
//}
var ups=JsonConvert.DeserializeObject<List<DyUpSecUserIdDto>>(cookie.UpSecUserIds);
var ups = JsonConvert.DeserializeObject<List<DyUpSecUserIdDto>>(cookie.UpSecUserIds);
foreach (var uper in ups)
{
@@ -85,8 +86,8 @@ namespace dy.net.job
while (hasMore)
{
var data = await _douyinService.SyncUpderPostVideos(count, cursor,uper.uid, cookie.Cookies);
hasMore = data != null && data.HasMore == 1 && cookie.UperSyncd == 0 &&uper.syncAll;
var data = await _douyinService.SyncUpderPostVideos(count, cursor, uper.uid, cookie.Cookies);
hasMore = data != null && data.HasMore == 1 && cookie.UperSyncd == 0 && uper.syncAll;
if (cookie.UperSyncd == 1)
{
Serilog.Log.Debug($"dyuploder-Cookie[{cookie.UserName}]已完整同步过，后续只获取最新一页数据");
@@ -126,22 +127,41 @@ namespace dy.net.job
var tag3 = tags.FirstOrDefault(x => x.Level == 3)?.TagName;
string saveFolder = CreateSaveFolder(cookie, uper.uper);
var fileName = $"{item.AwemeId}.{v.Format}"; // 用ID做文件名，避免特殊字符
string samplePrefix = null;
string sampleName = null;
if (config.UperUseViedoTitle)// 使用视频标题作为文件名
{
sampleName = TikTokFileNameHelper.GenerateFileName(item.Desc, item.AwemeId);
var sampleViedo = await _douyinVideoService.GetUperLastViedoFileName(item.Author.Uid, sampleName);
if (string.IsNullOrWhiteSpace(sampleViedo.Item1))
{
fileName = $"{sampleName}.{v.Format}";
}
else
{
fileName = $"{sampleViedo.Item1}.{v.Format}";
samplePrefix = sampleViedo.Item2;
}
}
var savePath = Path.Combine(saveFolder, fileName);
if (!File.Exists(savePath))
{
Serilog.Log.Debug($"dyuploder-[{uper.uper}]-视频[{SanitizePath(item.Desc)}]开始下载");
Serilog.Log.Debug($"dyuploder-[{uper.uper}]-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]开始下载");
await Task.Delay(_random.Next(1, 4) * 1000);
var downVideo = await _douyinService.DownloadAsync(videoUrl, savePath, cookie.Cookies);
if(downVideo)
if (downVideo)
{
Serilog.Log.Debug($"dyuploder-[{uper.uper}]-视频[{SanitizePath(item.Desc)}]下载{(downVideo ? "成功" : "失败")}");
Serilog.Log.Debug($"dyuploder-[{uper.uper}]-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]下载{(downVideo ? "成功" : "失败")}");
}
else
{
Serilog.Log.Error($"dyuploder-[{uper.uper}]-视频[{SanitizePath(item.Desc)}]下载{(downVideo ? "成功" : "失败")}");
Serilog.Log.Error($"dyuploder-[{uper.uper}]-视频[{TikTokFileNameHelper.SanitizePath(item.Desc)}]下载{(downVideo ? "成功" : "失败")}");
}
if (downVideo)
{
@@ -186,8 +206,16 @@ namespace dy.net.job
VideoCoverSavePath = Path.Combine(saveFolder, "poster.jpg"),
SyncTime = DateTime.Now,
DyUserId = data.Uid,
CookieId = cookie.Id
CookieId = cookie.Id,
};
if (!string.IsNullOrEmpty(sampleName))
{
video.VideoTitleSimplify = sampleName;
if (!string.IsNullOrEmpty(samplePrefix))
video.VideoTitleSimplifyPrefix = samplePrefix;
}
videos.Add(video);
var nfoPath = Path.Combine(saveFolder, $"{item.AwemeId}.nfo");
@@ -245,7 +273,7 @@ namespace dy.net.job
if (syncCount > 0)
{
Serilog.Log.Debug($"dyuploder-[{uper.uper}],本次共同步成功{syncCount}条视频");
}
else
{
@@ -349,22 +377,5 @@ namespace dy.net.job
// return Path.Combine(pathParts[0], string.Join("-", pathParts.Skip(1)), tag2 + "@" + item.AwemeId);
}
// 清理路径中的特殊字符（避免创建文件夹失败）
private static string SanitizePath(string path)
{
if (string.IsNullOrWhiteSpace(path))
{
path = "其他";
}
foreach (var c in Path.GetInvalidFileNameChars())
{
path = path.Replace(c, '_');
}
if (path.Length > 50)
{
path = path.Substring(0, 50);
}
return path.Trim();
}
}
}
model/AppConfig.cs
+6
View File
@@ -20,5 +20,11 @@ namespace dy.net.model
/// 每次查询数量
/// </summary>
public int BatchCount { get; set; } = 10;
/// <summary>
/// 博主视频是否 直接用标题做文件名，不另外加一层文件夹
/// </summary>
public bool UperUseViedoTitle { get; set; }
}
}
model/DyCollectVideo.cs
+16
View File
@@ -33,6 +33,19 @@ namespace dy.net.model
/// </summary>
[SugarColumn(Length =2000,IsNullable =true)]
public string VideoTitle { get; set; }
/// <summary>
/// 如果配置文件 UperFileNameUseViedoTitle=true
/// 简化后的标题 默认空 只有关注的博主的 视频会存储该字段，实际UP主的视频文件名是 VideoTitleSimplify + VideoTitleSimplifyPrefix
/// </summary>
[SugarColumn(Length =500,IsNullable =true)]
public string VideoTitleSimplify { get; set; }
/// <summary>
/// 精简标题的前缀 默认空，其实就是序号，类似 001-，002-
/// </summary>
[SugarColumn(Length =50,IsNullable =true)]
public string VideoTitleSimplifyPrefix { get; set; }
/// <summary>
/// 标签（分类）
/// </summary>
@@ -118,6 +131,9 @@ namespace dy.net.model
[SugarColumn(IsIgnore = true)]
public string SyncTimeStr => SyncTime.ToString("yyyy-MM-dd HH:mm:ss");
/// <summary>
/// 1喜欢的，2收藏的，3关注的
/// </summary>
[SugarColumn(Length =200,IsNullable =true)]
public string ViedoType { get; set; }
repository/DyCollectVideoRepository.cs
+48 -1
View File
@@ -1,5 +1,8 @@
using dy.net.model;
using dy.net.dto;
using dy.net.model;
using Quartz.Util;
using SqlSugar;
using System.Linq;
namespace dy.net.repository
{
@@ -59,5 +62,49 @@ namespace dy.net.repository
return (list, totalCount);
}
/// <summary>
///
/// </summary>
/// <param name="AuthorId"></param>
/// <param name="ViedoNameSimplify"></param>
/// <returns></returns>
public async Task<(string, string)> GetUperLastViedoFileName(string AuthorId,string ViedoNameSimplify)
{
var video= await this.Db.Queryable<DyCollectVideo>().Where(x => x.AuthorId == AuthorId && x.ViedoType == "3")
.Where(x => x.VideoTitleSimplify == ViedoNameSimplify)
.OrderByDescending(x => x.CreateTime).FirstAsync();
if (video != null)
{
if (string.IsNullOrWhiteSpace(video.VideoTitleSimplifyPrefix))
{
return (ViedoNameSimplify, "001");
}
else
{
//VideoTitleSimplifyPrefix的规则是从001开始
var prefixNumber = video.VideoTitleSimplifyPrefix.TrimEnd('-');
if (int.TryParse(prefixNumber, out int number))
{
number += 1;
return (ViedoNameSimplify, number.ToString("D3"));
}
else
{
return (ViedoNameSimplify, "001");
}
}
}
else
{
return (ViedoNameSimplify, "");
}
}
}
}
service/DyCollectVideoService.cs
+13
View File
@@ -90,6 +90,19 @@ namespace dy.net.service
return await _dyCollectVideoRepository.GetPagedAsync(pageIndex, pageSize, tag, author, viedoType, dates);
}
/// <summary>
/// 关注的博主的视频如果配置为视频标题作为文件名，生成文件名
/// </summary>
/// <param name="AuthorId"></param>
/// <param name="ViedoNameSimplify"></param>
/// <returns></returns>
public async Task<(string, string)> GetUperLastViedoFileName(string AuthorId, string ViedoNameSimplify)
{
return await _dyCollectVideoRepository.GetUperLastViedoFileName(AuthorId, ViedoNameSimplify);
}
}
utils/TikTokFileNameHelper.cs
+150
View File
@@ -0,0 +1,150 @@
using ClockSnowFlake;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
namespace dy.net.utils
{
/// <summary>
/// 抖音标题转文件名工具类（兼容Windows/macOS/Linux）
/// </summary>
public static class TikTokFileNameHelper
{
#region 配置参数
/// <summary>
/// 建议120字节（约60个汉字），适配所有系统
/// </summary>
private const int MaxFileNameBytes = 120;
/// <summary>
/// 非法字符替换后的占位符（也可设为空字符串）
/// </summary>
private const string IllegalCharReplacement = "";
#endregion
#region 处理抖音视频文件名
/// <summary>
/// 生成抖音视频文件名
/// </summary>
/// <param name="originalTitle">抖音原始标题</param>
/// <param name="videoId">排序号--如果有重复标题，根据时间排序取最后一个的序号+1 从001开始</param>
/// <returns>安全可用的文件名</returns>
public static string GenerateFileName(string originalTitle,string videoId)
{
// 1. 基础容错（处理空值）
originalTitle = string.IsNullOrWhiteSpace(originalTitle) ? videoId : originalTitle;
// 2. 净化标题：移除非法内容
string purifiedTitle = PurifyTitle(originalTitle, videoId);
// 3. 长度控制：按UTF-8字节数截断（避免超系统限制）
string truncatedTitle = TruncateByByteLength(purifiedTitle, MaxFileNameBytes);
return truncatedTitle;
}
#endregion
#region 内部辅助方法：标题净化
/// <summary>
/// 净化标题（移除非法字符、表情、话题标签）
/// </summary>
private static string PurifyTitle(string title,string id)
{
if (string.IsNullOrEmpty(title))
{
title = id;
}
// 步骤1：移除话题标签（#xxx 或 #xxx#yyy）
title = Regex.Replace(title, @"#\S+", "", RegexOptions.Compiled);
// 步骤2：移除表情符号（匹配常见表情Unicode区块）
string emojiPattern = @"[\u1F600-\u1F64F\u1F300-\u1F5FF\u1F680-\u1F6FF\u1E000-\u1EFFF\u2600-\u2B55\u200D]";
title = Regex.Replace(title, emojiPattern, "", RegexOptions.Compiled);
// 步骤3：过滤系统非法字符（Windows/macOS/Linux通用禁止）
char[] illegalChars = new[] { '/', '\\', ':', '*', '?', '"', '<', '>', '|', '\0', '\t', '\n', '\r' };
foreach (char c in illegalChars)
{
title = title.Replace(c.ToString(), IllegalCharReplacement);
}
char Separator = '-';
foreach (var c in Path.GetInvalidFileNameChars())
{
title = title.Replace(c, '_');
}
// 步骤5：合并连续分隔符（避免多个---）
title = Regex.Replace(title, $"{Separator}+", Separator.ToString(), RegexOptions.Compiled);
// 步骤6：移除首尾无效字符（分隔符、点号）
title = title.Trim(Separator, '.');
// 容错：如果净化后为空，返回默认值
return string.IsNullOrWhiteSpace(title) ? id : title;
}
#endregion
#region 内部辅助方法：按字节数截断
/// <summary>
/// 按UTF-8字节数截断字符串（避免截断半个中文）
/// </summary>
/// <param name="str">要截断的字符串</param>
/// <param name="maxBytes">最大字节数</param>
/// <returns>截断后的字符串</returns>
private static string TruncateByByteLength(string str, int maxBytes)
{
if (string.IsNullOrEmpty(str)) return str;
byte[] bytes = Encoding.UTF8.GetBytes(str);
if (bytes.Length <= maxBytes) return str;
// 从后往前截断，直到字节数≤maxBytes（避免半个中文）
for (int i = str.Length - 1; i >= 0; i--)
{
string truncated = str.Substring(0, i + 1);
if (Encoding.UTF8.GetByteCount(truncated) <= maxBytes)
{
return truncated;
}
}
// 极端情况（单个字符就超字节数），返回空字符串
return "";
}
#endregion
// 清理路径中的特殊字符（避免创建文件夹失败）
public static string SanitizePath(string path)
{
if (string.IsNullOrWhiteSpace(path))
{
path = "其他";
}
// 步骤1：移除话题标签（#xxx 或 #xxx#yyy）
path = Regex.Replace(path, @"#\S+", "", RegexOptions.Compiled);
// 步骤2：移除表情符号（匹配常见表情Unicode区块）
string emojiPattern = @"[\u1F600-\u1F64F\u1F300-\u1F5FF\u1F680-\u1F6FF\u1E000-\u1EFFF\u2600-\u2B55\u200D]";
path = Regex.Replace(path, emojiPattern, "", RegexOptions.Compiled);
foreach (var c in Path.GetInvalidFileNameChars())
{
path = path.Replace(c, '_');
}
if (path.Length > 50)
{
path = path.Substring(0, 50);
}
return path.Trim();
}
}
}