nanxun/douyin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加同步指定博主的作品

Browse Source
This commit is contained in:
lijianyou
2025-10-19 01:11:55 +08:00
parent 14a709e691
commit 59ce4f59ed
25 changed files with 1865 additions and 277 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Controllers/ConfigController.cs
+25 -2
View File
@@ -5,6 +5,7 @@ using dy.net.service;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Newtonsoft.Json;
using static Dm.net.buffer.ByteArrayBuffer;
namespace dy.net.Controllers
@@ -56,6 +57,10 @@ namespace dy.net.Controllers
[HttpPost("add")]
public async Task<IActionResult> AddAsync([FromBody] DyUserCookies dyUserCookies)
{
if(dyUserCookies.UpSecUserIdsJson!=null)
{
dyUserCookies.UpSecUserIds = JsonConvert.SerializeObject( dyUserCookies.UpSecUserIdsJson);
}
var result = await dyCookieService.Add(dyUserCookies);
if (result)
{
@@ -71,6 +76,12 @@ namespace dy.net.Controllers
[HttpPost("update")]
public async Task<IActionResult> UpdateAsync([FromBody] DyUserCookies dyUserCookies)
{
if (dyUserCookies.UpSecUserIdsJson != null )
{
dyUserCookies.UpSecUserIds = JsonConvert.SerializeObject(dyUserCookies.UpSecUserIdsJson);
}
if (dyUserCookies.Id == "0")
{
dyUserCookies.Id=IdGener.GetLong().ToString();
@@ -140,8 +151,20 @@ namespace dy.net.Controllers
private async Task ReStartJob() {
var config= commonService.GetConfig();
if(config!=null)
await quartzJobService.StartJob(config.Cron);
var cookies = await dyCookieService.GetAllCookies();
//重置同步状态
foreach (var cookie in cookies)
{
cookie.CollHasSyncd = 0;
cookie.FavHasSyncd = 0;
cookie.UperSyncd = 0;
await dyCookieService.UpdateAsync(cookie);
}
if (config!=null)
quartzJobService.StartJob(config.Cron);
//避免前端等待
}
}
}
Program.cs
+7 -2
View File
@@ -238,21 +238,26 @@ namespace dy.net
SavePath = "/app/collect",
Status = 0,
SecUserId = "--",
FavSavePath = "/app/favorite"
FavSavePath = "/app/favorite",
UpSavePath = "/app/uper",
});
// 初始化配置
var commonService = services.GetRequiredService<CommonService>();
var config = commonService.InitConfig(new AppConfig
{
Id = IdGener.GetLong().ToString(),
Cron = "30",
BatchCount = 10
BatchCount = 20
});
// 更新收藏视频类型--兼容老版本-原来的旧数据没有这个类型字段
commonService.UpdateCollectViedoType();
commonService.UpdateAllCookieSyncedToZero();
// 启动定时任务
var quartzJobService = services.GetRequiredService<QuartzJobService>();
quartzJobService.StartJob(config?.Cron ?? "30");
app/src/App.vue
+2 -2
View File
@@ -1,6 +1,6 @@
<template>
<ThemeProvider is-root v-bind="themeConfig" :apply-style="false">
<stepin-view system-name="douyin.sync.net" :class="`${contentClass}`" :user="user" :navMode="navigation" :useTabs="useTabs" :themeList="themeList" v-model:show-setting="showSetting" v-model:theme="theme" @themeSelect="configTheme" logo-src="@/assets/logo1.png">
<stepin-view system-name="dy.sync.net" :class="`${contentClass}`" :user="user" :navMode="navigation" :useTabs="useTabs" :themeList="themeList" v-model:show-setting="showSetting" v-model:theme="theme" @themeSelect="configTheme" logo-src="@/assets/logo1.png">
<template #headerActions>
<HeaderActions @showSetting="showSetting = true" />
</template>
@@ -46,7 +46,7 @@ const { navigation, useTabs, theme, contentClass } = storeToRefs(useSettingStore
const themeConfig = computed(() => themeList.find((item) => item.key === theme.value)?.config ?? {});
const user = reactive({
name: 'admin',
name: 'douyin',
avatar: avatar,
menuList: [
// { title: '个人中心', key: 'personal', icon: 'UserOutlined', onClick: () => router.push('/profile') },
app/src/components/layout/FrontView.vue
+1 -1
View File
@@ -30,7 +30,7 @@ onMounted(() => {
<div class="front-view flex flex-col" style="background: linear-gradient(135deg, #1a1a2e 0%, #16213e 50%, #0f3460 100%)">
<div class="text-xxl text-text hover:text-text" style="margin-left:20px;">
<img src="/logo1.png" />
douyin.sync.net
dy.sync.net
</div>
<div class="front-content ">
<router-view />
app/src/components/layout/HeaderActions.vue
+10 -10
View File
@@ -35,17 +35,17 @@ const noticeList = reactive([
<StepinHeaderAction>
<DayNightSwitch />
</StepinHeaderAction>
<StepinHeaderAction>
<a-tooltip>
<!-- placement="rightTop" -->
<template #title><span style="color:yellow">Swagger Api</span></template>
<a class="action-item" href="/swagger" target="_blank">
<!-- <ApiOutlined /> -->
<img class="gitee-logo" alt="swagger api" src="@/assets/swagger.png" />
</a>
</a-tooltip>
<!-- <StepinHeaderAction> -->
<!-- <a-tooltip> -->
<!-- placement="rightTop" -->
<!-- <template #title><span style="color:yellow">Swagger Api</span></template> -->
<!-- <a class="action-item" href="/swagger" target="_blank"> -->
<!-- <ApiOutlined /> -->
<!-- <img class="gitee-logo" alt="swagger api" src="@/assets/swagger.png" /> -->
<!-- </a> -->
<!-- </a-tooltip> -->
</StepinHeaderAction>
<!-- </StepinHeaderAction> -->
<!-- <StepinHeaderAction>
<a class="action-item" href="http://github.com/stepui/stepin-template" target="_blank">
<GithubOutlined />
app/src/pages/cok/CookieTable.vue
+109 -18
View File
@@ -8,20 +8,31 @@ import { EditFilled } from '@ant-design/icons-vue';
import { useApiStore } from '@/store';
import type { UnwrapRef } from 'vue';
import { StarOutlined, StarFilled, StarTwoTone } from '@ant-design/icons-vue';
const columns = [
{
title: '用户昵称',
title: 'Cookie名',
dataIndex: 'userName',
width: 100,
},
{ title: '状态', dataIndex: 'status' },
{ title: '收藏文件存储路径', dataIndex: 'savePath' },
{ title: '喜欢文件存储路径', dataIndex: 'favSavePath' },
{ title: 'Cookie', dataIndex: 'cookies' },
{ title: 'SecUserId', dataIndex: 'secUserId' },
{ title: '状态', dataIndex: 'status', width: 80 },
{ title: '收藏路径', dataIndex: 'savePath', width: 100 },
{ title: '喜欢路径', dataIndex: 'favSavePath', width: 100 },
{ title: 'Up主路径', dataIndex: 'upSavePath', width: 100 },
// { title: 'Cookie', dataIndex: 'cookies' },
{ title: 'Up主secUserId', dataIndex: 'upSecUserIds' },
{ title: '用户secUserId', dataIndex: 'secUserId' },
{ title: '操作', dataIndex: 'edit', width: 200 },
// { title: 'id', dataIndex: 'id', width: 200, hiden: false },
];
// 定义数组中单个对象的类型：包含uper和uid字段（可选字符串类型，根据实际需求调整是否可选）
interface UpSecUserIdItem {
uper?: string; // 若要求必填，可去掉问号 `uper: string;`
uid?: string; // 同上，必填则去掉问号
syncAll: number;
}
type DataItem = {
id?: string;
userName?: string;
@@ -31,6 +42,9 @@ type DataItem = {
secUserId?: string;
status?: number;
_isNew?: boolean;
upSecUserIdsJson?: UpSecUserIdItem[];
upSecUserIds?: string;
upSavePath?: string;
};
// const dataSource = reactive<DataItem[]>([
@@ -100,6 +114,8 @@ const newAuthor = (author?: DataItem) => {
author.secUserId = undefined;
author.status = 0;
author.id = '0';
author.upSecUserIdsJson = undefined;
author.upSavePath = undefined;
return author;
};
@@ -137,7 +153,7 @@ function submit() {
} else {
copyObject(editRecord.value, resData);
}
// alert(JSON.stringify(form.upSecUserIdsJson));
// console.log('1', authors);
// console.log('2', res);
useApiStore()
@@ -213,32 +229,96 @@ const showCookies = (recode: DataItem) => {
showCookiesModal.value = true;
showCookiesData = recode.cookies;
};
const showUpersModal = ref(false);
let showUpersData = '';
const showUpers = (recode: DataItem) => {
showUpersModal.value = true;
showUpersData = recode.upSecUserIds;
};
const addRow = () => {
form.upSecUserIdsJson.push({ uper: '', uid: '', syncAll: 0 });
};
const removeRow = (index) => {
form.upSecUserIdsJson.splice(index, 1);
};
const rowCount = 4;
</script>
<template>
<a-modal :title="form._isNew ? '新增Cookie' : '编辑Cookie'" v-model:visible="showModal" @ok="submit" @cancel="cancel" width="1000px">
<a-form ref="formModel" :model="form" :labelCol="{ span: 3 }" :wrapperCol="{ span: 20 }">
<a-form-item label="用户名" required name="userName">
<a-form-item label="Cookie别名" required name="userName">
<a-input v-model:value="form.userName" />
</a-form-item>
<a-form-item label="id" required name="id" v-show="false">
<a-input v-model:value="form.id" />
</a-form-item>
<a-form-item required label="收藏存储路径" name="savePath">
<a-form-item label="收藏的存储路径" name="savePath">
<a-input v-model:value="form.savePath" />
</a-form-item>
<a-form-item required label="Cookie" name="cookies">
<a-textarea v-model:value="form.cookies" rows='13' />
<a-form-item label="Cookie" name="cookies">
<a-textarea v-model:value="form.cookies" :rows='rowCount' />
</a-form-item>
<a-form-item label="喜欢存储路径" name="favSavePath">
<a-input v-model:value="form.favSavePath" />
<a-alert message="如果要同步“我喜欢的”视频，需要填写！！！" type="warning" />
<a-form-item label="喜欢的存储路径" name="favSavePath">
<div style="display: flex; align-items: center; gap: 6px;">
<a-input v-model:value="form.favSavePath" style="flex: 1;" />
<a-tooltip title="同步“我喜欢的”视频时，必填！！！">
<ExclamationCircleOutlined style="color: #faad14;font-size: 16px;" />
</a-tooltip>
</div>
</a-form-item>
<a-form-item label="SecUserId" name="secUserId">
<a-input v-model:value="form.secUserId" />
<a-alert message="如果要同步“我喜欢的”视频，需要填写！！！" type="warning" />
<div style="display: flex; align-items: center; gap: 6px;">
<a-input v-model:value="form.secUserId" style="flex: 1;" />
<a-tooltip title="同步“我喜欢的”视频时，必填！！！">
<ExclamationCircleOutlined style="color: #faad14;font-size: 16px;" />
</a-tooltip>
</div>
</a-form-item>
<a-form-item required label="状态" name="status">
<a-form-item label="Up主视频存储路径" name="upSavePath">
<div style="display: flex; align-items: center; gap: 6px;">
<a-input v-model:value="form.upSavePath" style="flex: 1;" />
<a-tooltip title="同步指定博主视频时必填！！！">
<ExclamationCircleOutlined style="color: #faad14;font-size: 16px;" />
</a-tooltip>
</div>
</a-form-item>
<!-- 新增：uper和uid动态行区域 -->
<a-form-item label="Uper主secUserIds" name="upSecUserIdsJson">
<!-- 添加行按钮 -->
<a-button type="primary" @click="addRow" style="margin-bottom: 12px">
添加抖音Up主 <template #icon>
<PlusOutlined />
</template>
</a-button>
<!-- 动态渲染所有行 -->
<div v-for="(row, index) in form.upSecUserIdsJson" :key="index" style="display: flex; gap: 12px; margin-bottom: 8px; align-items: center">
<!-- uper输入框 -->
<a-input v-model:value="row.uper" placeholder="请输入博主名字，可随意" style="flex: 1" />
<!-- uid输入框 -->
<a-input v-model:value="row.uid" placeholder="请输入博主secUserId" style="flex: 3" />
<!-- syncAll 开关字段 -->
<div style="flex: 1; display: flex; align-items: center;">
<!-- 用 a-tooltip 包裹文字，实现气泡提示 -->
<a-tooltip title="默认关闭，仅同步 UP 主最新一页数据；开启将同步全部作品（量大不建议开启）">
<span style="margin-right: 8px; cursor: default;color:#faad14">同步全部作品</span>
</a-tooltip>
<a-switch v-model:checked="row.syncAll" />
</div>
<!-- 删除当前行按钮 -->
<a-button type="text" danger @click="removeRow(index)">
<template #icon>
<DeleteOutlined />
</template>
</a-button>
</div>
</a-form-item>
<a-form-item label="状态" name="status">
<a-select style="width: 90px" v-model:value="form.status" :options="[
{ label: '关闭', value: 0 },
{ label: '开启', value: 1 },
@@ -279,7 +359,10 @@ const showCookies = (recode: DataItem) => {
<template v-else-if="column.dataIndex === 'cookies'">
<!-- 触发按钮 -->
<a-button @click="showCookies(record)">查看</a-button>
</template>
<template v-else-if="column.dataIndex === 'upSecUserIds'">
<!-- 触发按钮 -->
<a-button @click="showUpers(record)">查看</a-button>
</template>
<template v-else-if="column.dataIndex === 'edit'">
<a-button :disabled="showModal" type="link" @click="edit(record)">
@@ -309,6 +392,14 @@ const showCookies = (recode: DataItem) => {
{{ showCookiesData || '无Cookie数据' }}
</div>
</a-modal>
<!-- Modal弹窗 -->
<a-modal title="要同步的Up主信息" :visible="showUpersModal" style="width:1200px;" @cancel="showUpersModal = false" @ok="showUpersModal = false">
<!-- 弹窗内容 -->
<div class="cookie-content">
{{ showUpersData || '未设置' }}
</div>
</a-modal>
</template>
<style scoped>
.cookie-content {
app/src/pages/login/LoginBox.vue
+1 -1
View File
@@ -4,7 +4,7 @@
<div class="login-box rounded-lg bg-gray-900 shadow-lg p-8 max-w-2xl mx-auto my-10 border border-gray-800 transition-all duration-300 hover:shadow-xl">
<a-form :model="form" :wrapperCol="{ span: 24 }" @finish="login" class="login-form w-full p-lg text-gray-200">
<div class="third-platform">
<div class="third-title mb-6 text-xl text-center font-semibold text-white">抖音同步工具</div>
<div class="third-title mb-6 text-xl text-center font-semibold text-white">抖音同步小帮手</div>
</div>
<a-divider class="my-6 bg-gray-700"></a-divider>
app/src/pages/workplace/RecordTable.vue
+26 -23
View File
@@ -1,27 +1,30 @@
<template>
<a-form layout="inline" style="margin-top:10px;" :model="quaryData">
<a-form-item label="同步日期">
<a-range-picker v-model:value="value1" :ranges="ranges" :locale="locale" @change="datePicked" />
</a-form-item>
<a-form-item label="抖音作者" ref="author" name="author">
<a-input v-model:value="quaryData.author"></a-input>
</a-form-item>
<a-form-item>
<a-radio-group v-model:value="quaryData.viedoType" button-style="solid" @change="onViedoTypeChanged">
<a-radio-button value="*">全部</a-radio-button>
<a-radio-button value="1">我喜欢的</a-radio-button>
<a-radio-button value="2">我收藏的</a-radio-button>
</a-radio-group>
</a-form-item>
<a-form-item :wrapper-col="{ offset: 8, span: 16 }">
<a-space>
<a-button type="primary" @click="GetRecords">查询</a-button>
<a-button type="danger" @click="StartNow">立即重新同步</a-button>
</a-space>
</a-form-item>
</a-form>
<a-table :columns="columns" :data-source="dataSource" bordered :pagination="pagination" @change="handleTableChange" :loading="loading">
</a-table>
<div>
<a-form layout="inline" style="margin-top:10px;" :model="quaryData">
<a-form-item label="同步日期">
<a-range-picker v-model:value="value1" :ranges="ranges" :locale="locale" @change="datePicked" />
</a-form-item>
<a-form-item label="抖音作者" ref="author" name="author">
<a-input v-model:value="quaryData.author"></a-input>
</a-form-item>
<a-form-item>
<a-radio-group v-model:value="quaryData.viedoType" button-style="solid" @change="onViedoTypeChanged">
<a-radio-button value="*">全部</a-radio-button>
<a-radio-button value="1">喜欢的</a-radio-button>
<a-radio-button value="2">收藏的</a-radio-button>
<a-radio-button value="3">关注的</a-radio-button>
</a-radio-group>
</a-form-item>
<a-form-item :wrapper-col="{ offset: 8, span: 16 }">
<a-space>
<a-button type="primary" @click="GetRecords">查询</a-button>
<a-button type="danger" @click="StartNow">立即重新同步</a-button>
</a-space>
</a-form-item>
</a-form>
<a-table :columns="columns" :data-source="dataSource" bordered :pagination="pagination" @change="handleTableChange" :loading="loading">
</a-table>
</div>
</template>
<script lang="ts" setup>
import { defineComponent, reactive, ref } from 'vue';
dto/CollectVideoInfo.cs
+2 -2
View File
@@ -336,10 +336,10 @@ namespace dy.net.dto
[JsonProperty("level")]
public int Level { get; set; }
/// <summary>
///
/// 改成long 防止报错
/// </summary>
[JsonProperty("tag_id")]
public int TagId { get; set; }
public long TagId { get; set; }
/// <summary>
/// 人文社科
/// </summary>
dto/DyUpSecUserIdDto.cs
+19
View File
@@ -0,0 +1,19 @@
using Newtonsoft.Json;
namespace dy.net.dto
{
public class DyUpSecUserIdDto
{
/// <summary>
/// up主名称
/// </summary>
public string uper { get; set; }
public string uid { get; set; }
/// <summary>
/// 是否同步up主全部视频 0-否 1-是
/// </summary>
public bool syncAll { get; set; }
}
}
dto/VideoNfo.cs
+4
View File
@@ -21,5 +21,9 @@
/// 作者/创作者
/// </summary>
public string Author { get; set; }
// 新增的属性
public DateTime? ReleaseDate { get; set; } // 可空，避免无发布时间时的默认值
public IEnumerable<string> Genres { get; set; } // 分类标签集合（如 ["动作", "科幻"]）
}
}
dy.net.csproj
+5 -1
View File
@@ -55,7 +55,8 @@
<PackageReference Include="Serilog.Sinks.File" Version="5.0.0" />
<PackageReference Include="System.Net.Http" Version="4.3.4" />
<PackageReference Include="System.Text.RegularExpressions" Version="4.3.1" />
</ItemGroup>
<PackageReference Include="BouncyCastle.NetCore" Version="2.2.1" />
</ItemGroup>
<ItemGroup>
<Folder Include="db\" />
@@ -67,6 +68,9 @@
</ItemGroup>
<ItemGroup>
<None Update="db\dy.sqlite">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="Dockerfile">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
extension/ServiceExtension.cs
+45 -2
View File
@@ -118,6 +118,7 @@ namespace dy.net.extension
{
services.AddTransient<DouYinCollectSyncJob>();
services.AddTransient<DouYinFavoritSyncJob>();
services.AddTransient<DouYinUperPostSyncJob>();
// 注册Quartz服务
services.AddQuartz();
@@ -130,7 +131,7 @@ namespace dy.net.extension
public static void AddHttpClients(this IServiceCollection services)
{
services.AddHttpClient("douyin", client =>
services.AddHttpClient("dy_collect", client =>
{
client.DefaultRequestHeaders.Add("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2");
client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36");
@@ -138,11 +139,53 @@ namespace dy.net.extension
});
services.AddHttpClient("douyinfav", client =>
services.AddHttpClient("dy_favorite", client =>
{
client.DefaultRequestHeaders.Referrer = new Uri("https://www.douyin.com/user/self?showTab=like");
});
services.AddHttpClient("dy_uper", client =>
{
//client.DefaultRequestHeaders.Referrer = new Uri("https://www.douyin.com/user/self?showTab=like");
});
// 配置HttpClient（Startup.cs或Program.cs）
services.AddHttpClient("dy_download")
.ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler
{
// 控制并发连接数（根据服务器承受能力调整，建议5-20）
MaxConnectionsPerServer = 5,
// 禁用代理自动检测（减少不必要的延迟）
UseProxy = false,
// 连接超时（建立连接的超时时间）
ConnectTimeout = TimeSpan.FromSeconds(60)
})
// 配置客户端默认请求头
.ConfigureHttpClient(client =>
{
client.DefaultRequestHeaders.Add("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2");
client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36");
client.DefaultRequestHeaders.Add("Referer", "https://www.douyin.com");
});
//services.AddHttpClient("dy_down_fav")
// .ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler
// {
// // 控制并发连接数（根据服务器承受能力调整，建议5-20）
// MaxConnectionsPerServer = 5,
// // 禁用代理自动检测（减少不必要的延迟）
// UseProxy = false,
// // 连接超时（建立连接的超时时间）
// ConnectTimeout = TimeSpan.FromSeconds(60)
// });
//services.AddHttpClient("dy_down_uper")
// .ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler
// {
// // 控制并发连接数（根据服务器承受能力调整，建议5-20）
// MaxConnectionsPerServer = 5,
// // 禁用代理自动检测（减少不必要的延迟）
// UseProxy = false,
// // 连接超时（建立连接的超时时间）
// ConnectTimeout = TimeSpan.FromSeconds(60)
// });
}
/// <summary>
job/DouYinCollectSyncJob.cs
+51 -15
View File
@@ -17,9 +17,9 @@ namespace dy.net.job
private readonly DyCollectVideoService _douyinVideoService;
private readonly CommonService commonService;
private readonly Random _random = new Random();
private string count = "20"; // 每页请求的视频数量，默认20
private string count = "18"; // 每页请求的视频数量，默认18
public DouYinCollectSyncJob(DyCookieService dyCookieService, DyHttpClientService dyHttpClientService, DyCollectVideoService dyCollectVideoService,CommonService commonService)
public DouYinCollectSyncJob(DyCookieService dyCookieService, DyHttpClientService dyHttpClientService, DyCollectVideoService dyCollectVideoService, CommonService commonService)
{
_dyCookieService = dyCookieService;
_douyinService = dyHttpClientService;
@@ -53,10 +53,14 @@ namespace dy.net.job
//return;
}
//每天凌晨1点到1点半之间执行一次重置同步状态-全部cookie
UpdateCookieSyncedToZero();
foreach (var cookie in cookies)
{
Serilog.Log.Debug($"collect-开始同步 Cookie-[{cookie.UserName}]收藏视频");
if (string.IsNullOrWhiteSpace(cookie.Cookies)|| cookie.Cookies.Length<1000)
if (string.IsNullOrWhiteSpace(cookie.Cookies) || cookie.Cookies.Length < 1000)
{
Serilog.Log.Debug($"collect-Cookie-[{cookie.UserName}]无效，跳过");
continue;
@@ -68,12 +72,16 @@ namespace dy.net.job
int index = 0;
bool hasMore = true;
string cursor = "0";
string cursor = "0";
while (hasMore)
{
var data = await _douyinService.SyncCollectVideos(cursor, count, cookie.Cookies);
hasMore = data != null && data.HasMore == 1;
hasMore = data != null && data.HasMore == 1 && cookie.CollHasSyncd == 0;
if (cookie.CollHasSyncd == 1)
{
Serilog.Log.Debug($"collect-Cookie[{cookie.UserName}]已完整同步过，后续只获取最新一页数据");
}
//Serilog.Log.Debug($"还有数据需要同步吗？{(hasMore ? "YES" : "NO")}");
if (data == null)
{
@@ -87,7 +95,7 @@ namespace dy.net.job
{
break;
}
List<DyCollectVideo> videos = new List<DyCollectVideo>();
foreach (var item in data.AwemeList)
{
@@ -120,7 +128,7 @@ namespace dy.net.job
if (!File.Exists(savePath))
{
Serilog.Log.Debug($"collect-视频[{SanitizePath(item.Desc)}]开始下载");
await Task.Delay(_random.Next(1, 4) * 1000);
var downVideo = await _douyinService.DownloadAsync(videoUrl, savePath, cookie.Cookies);
Serilog.Log.Debug($"collect-视频[{SanitizePath(item.Desc)}]下载{(downVideo ? "成功" : "失败")}");
if (downVideo)
@@ -132,9 +140,15 @@ namespace dy.net.job
var avatarSavePath = Path.Combine(cookie.SavePath, "author", avatarImgName);
await DownAuthorAvatar(cookie.SavePath, item, avatarSavePath, cookie.Cookies);
var avatarUrl = item.Author?.AvatarLarger?.UrlList != null && item.Author.AvatarLarger.UrlList.Any()
? item.Author.AvatarLarger.UrlList[0] : null;
? item.Author.AvatarLarger.UrlList[0] : null;
// 备用头像链接
if (string.IsNullOrWhiteSpace(avatarUrl))
{
avatarUrl = item.Author?.AvatarThumb?.UrlList != null && item.Author.AvatarThumb.UrlList.Any()
? item.Author.AvatarThumb.UrlList[0] : null;
}
// 构造视频数据
DyCollectVideo video = new()
@@ -171,11 +185,12 @@ namespace dy.net.job
Author = video.Author,
Poster = Path.Combine(saveFolder, "poster.jpg"),
Title = video.VideoTitle,
Thumbnail= Path.Combine(saveFolder, "fanart.jpg"),
Thumbnail = Path.Combine(saveFolder, "fanart.jpg"),
ReleaseDate = video.CreateTime,
Genres = new List<string> { tag1, tag2, tag3 }.Where(t => !string.IsNullOrWhiteSpace(t)).ToList()
}, nfoPath);
}
await Task.Delay(_random.Next(1, 3) * 1000);
}
else
{
@@ -210,14 +225,21 @@ namespace dy.net.job
Serilog.Log.Debug("collect-因为数据库没有保存成功，本次下载的视频目录已尝试删除，将继续处理下一页数据");
}
}
else {
//Serilog.Log.Debug($"没有查询到新的视频");
else
{
//Serilog.Log.Debug($"没有查询到新的视频");
}
await Task.Delay(_random.Next(2, 5) * 1000);
await Task.Delay(_random.Next(5, 10) * 1000);
}
if (syncCount > 0)
{
Serilog.Log.Debug($"collect-Cookie-[{cookie.UserName}],本次共同步成功{syncCount}条视频");
else {
// 更新同步状态为已同步
cookie.CollHasSyncd = 1;
await _dyCookieService.UpdateAsync(cookie);
}
else
{
Serilog.Log.Debug($"collect-Cookie-[{cookie.UserName}],本次没有查询到新的视频");
}
@@ -325,5 +347,19 @@ namespace dy.net.job
}
return path.Trim();
}
/// <summary>
/// 重置Cookie的同步状态为0（每天凌晨1点到1点半之间执行一次）
/// </summary>
/// <returns></returns>
private void UpdateCookieSyncedToZero()
{
var now = DateTime.Now;
//如果当时间为凌晨1点到1点半之间，则重置为0
if (now.Hour == 1 && now.Minute < 30)
{
commonService.UpdateAllCookieSyncedToZero();
}
}
}
}
job/DouYinFavoritSyncJob.cs
+29 -9
View File
@@ -17,6 +17,7 @@ namespace dy.net.job
private readonly DyCollectVideoService _douyinVideoService;
private readonly CommonService commonService;
private readonly Random _random = new Random();
private string count = "18"; // 每页请求的视频数量，默认18
public DouYinFavoritSyncJob(DyCookieService dyCookieService, DyHttpClientService dyHttpClientService, DyCollectVideoService dyCollectVideoService,CommonService commonService)
{
@@ -35,7 +36,10 @@ namespace dy.net.job
Serilog.Log.Debug("favorite-请先在设置中初始化配置，再执行同步任务");
return;
}
if (config.BatchCount > 0)
{
count = config.BatchCount.ToString();
}
var cookies = await _dyCookieService.GetAllCookies();
if (cookies == null || !cookies.Any())
{
@@ -74,8 +78,12 @@ namespace dy.net.job
string cursor = "0";
while (hasMore)
{
var data = await _douyinService.SyncFavoriteVideos(cursor, cookie.SecUserId, cookie.Cookies);
hasMore = data != null && data.HasMore == 1;
var data = await _douyinService.SyncFavoriteVideos(count,cursor, cookie.SecUserId, cookie.Cookies);
hasMore = data != null && data.HasMore == 1 && cookie.FavHasSyncd == 0;
if (cookie.FavHasSyncd == 1)
{
Serilog.Log.Debug($"collect-Cookie[{cookie.UserName}]已完整同步过，后续只获取最新一页数据");
}
//Serilog.Log.Debug($"还有数据需要同步吗？{(hasMore ? "YES" : "NO")}");
if (data == null)
{
@@ -121,7 +129,7 @@ namespace dy.net.job
if (!File.Exists(savePath))
{
Serilog.Log.Debug($"favorite-视频[{SanitizePath(item.Desc)}]开始下载");
await Task.Delay(_random.Next(1, 4) * 1000);
var downVideo = await _douyinService.DownloadAsync(videoUrl, savePath, cookie.Cookies);
Serilog.Log.Debug($"favorite-视频[{SanitizePath(item.Desc)}]下载{(downVideo ? "成功" : "失败")}");
if (downVideo)
@@ -132,10 +140,16 @@ namespace dy.net.job
var avatarImgName = $"{item.Author.Uid}.jpg";
var avatarSavePath = Path.Combine(cookie.FavSavePath, "author", avatarImgName);
await DownAuthorAvatar(cookie.FavSavePath, item, avatarSavePath, cookie.Cookies);
var avatarUrl = item.Author?.AvatarThumb?.UrlList != null && item.Author.AvatarThumb.UrlList.Any()
? item.Author.AvatarThumb.UrlList[0] : null;
var avatarUrl = item.Author?.AvatarLarger?.UrlList != null && item.Author.AvatarLarger.UrlList.Any()
? item.Author.AvatarLarger.UrlList[0] : null;
// 尝试使用AvatarThumb作为备用头像URL
if (string.IsNullOrWhiteSpace(avatarUrl))
{
avatarUrl = item.Author?.AvatarThumb?.UrlList != null && item.Author.AvatarThumb.UrlList.Any()
? item.Author.AvatarThumb.UrlList[0] : null;
}
// 构造视频数据
DyCollectVideo video = new()
@@ -173,10 +187,11 @@ namespace dy.net.job
Poster = Path.Combine(saveFolder, "poster.jpg"),
Title = video.VideoTitle,
Thumbnail= Path.Combine(saveFolder, "fanart.jpg"),
ReleaseDate = video.CreateTime,
Genres = new List<string> { tag1, tag2, tag3 }.Where(t => !string.IsNullOrWhiteSpace(t)).ToList()
}, nfoPath);
}
await Task.Delay(_random.Next(1, 3) * 1000);
}
else
{
@@ -214,10 +229,15 @@ namespace dy.net.job
else {
//Serilog.Log.Debug($"没有查询到新的视频");
}
await Task.Delay(_random.Next(2, 5) * 1000);
await Task.Delay(_random.Next(5, 10) * 1000);
}
if (syncCount > 0)
{
Serilog.Log.Debug($"Cookie-[{cookie.UserName}],本次共同步成功{syncCount}条视频");
// 更新同步状态为已同步
cookie.FavHasSyncd = 1;
await _dyCookieService.UpdateAsync(cookie);
}
else {
Serilog.Log.Debug($"favorite-Cookie-[{cookie.UserName}],本次没有查询到新的视频");
}
job/DouYinUperPostSyncJob.cs
+363
View File
@@ -0,0 +1,363 @@
using ClockSnowFlake;
using dy.net.dto;
using dy.net.model;
using dy.net.service;
using dy.net.utils;
using Newtonsoft.Json;
using Quartz;
namespace dy.net.job
{
[DisallowConcurrentExecution]
public class DouYinUperPostSyncJob : IJob
{
private readonly DyCookieService _dyCookieService;
private readonly DyHttpClientService _douyinService;
private readonly DyCollectVideoService _douyinVideoService;
private readonly CommonService commonService;
private readonly Random _random = new Random();
private string count = "18"; // 每页请求的视频数量，默认18
private readonly string httpDownName = "dy_down_uper";
public DouYinUperPostSyncJob(DyCookieService dyCookieService, DyHttpClientService dyHttpClientService, DyCollectVideoService dyCollectVideoService,CommonService commonService)
{
_dyCookieService = dyCookieService;
_douyinService = dyHttpClientService;
_douyinVideoService = dyCollectVideoService;
this.commonService = commonService;
}
public async Task Execute(IJobExecutionContext context)
{
var config = commonService.GetConfig();
if (config == null)
{
Serilog.Log.Debug("dyuploder-请先在设置中初始化配置，再执行同步任务");
return;
}
if (config.BatchCount > 0)
{
count = config.BatchCount.ToString();
}
Serilog.Log.Debug($"dyuploder-当前设置的每次扫描行数为:{count}");
var cookies = await _dyCookieService.GetAllCookies();
cookies = cookies.Where(x => !string.IsNullOrWhiteSpace(x.UpSecUserIds))?.ToList();
if (!cookies.Any())
{
Serilog.Log.Debug("dyuploder-无可用抖音UP主SecUserId，任务终止");
return;
}
else
{
Serilog.Log.Debug($"dyuploder-当前有{cookies.Count}个cookie开启了同步,即将开始同步");
//return;
}
foreach (var cookie in cookies)
{
if(string.IsNullOrWhiteSpace(cookie.UpSavePath))
{
Serilog.Log.Debug($"dyuploder-Cookie[{cookie.UserName}]的UP主保存路径无效，跳过");
continue;
}
var ups=JsonConvert.DeserializeObject<List<DyUpSecUserIdDto>>(cookie.UpSecUserIds);
foreach (var uper in ups)
{
Serilog.Log.Debug($"dyuploder-开始同步-[{uper.uper}]主页作品数据");
if (string.IsNullOrWhiteSpace(uper.uid) || uper.uid.Length < 10)
{
Serilog.Log.Debug($"dyuploder-[{uper.uper}]的SecUserId无效，跳过");
continue;
}
try
{
int syncCount = 0;// 记录本次Cookie同步的视频数量
int index = 0;
bool hasMore = true;
string cursor = "0";
while (hasMore)
{
var data = await _douyinService.SyncUpderPostVideos(count, cursor,uper.uid, cookie.Cookies);
hasMore = data != null && data.HasMore == 1 && cookie.UperSyncd == 0 &&uper.syncAll;
if (cookie.UperSyncd == 1)
{
Serilog.Log.Debug($"dyuploder-Cookie[{cookie.UserName}]已完整同步过，后续只获取最新一页数据");
}
//Serilog.Log.Debug($"还有数据需要同步吗？{(hasMore ? "YES" : "NO")}");
if (data == null)
{
Serilog.Log.Debug($"dyuploder-[{uper.uper}]获取数据失败，请检查Up主SecUserId");
break;
}
cursor = data != null && !string.IsNullOrWhiteSpace(data.MaxCursor) ? data.MaxCursor : "0";
if (data.AwemeList == null || !data.AwemeList.Any())
{
break;
}
List<DyCollectVideo> videos = new List<DyCollectVideo>();
foreach (var item in data.AwemeList)
{
if (item == null)
continue;
if (item.Video == null)
continue;
if (item.Video.BitRate == null)
continue;
var v = item.Video.BitRate.FirstOrDefault();
var tags = item.VideoTags;
if (v == null) continue;
var videoUrl = v.PlayAddr.UrlList != null && v.PlayAddr.UrlList.Any() ? v.PlayAddr.UrlList[0] : null;
if (string.IsNullOrWhiteSpace(videoUrl)) continue;
var tag1 = tags.FirstOrDefault(x => x.Level == 1)?.TagName;
var tag2 = tags.FirstOrDefault(x => x.Level == 2)?.TagName;
var tag3 = tags.FirstOrDefault(x => x.Level == 3)?.TagName;
string saveFolder = CreateSaveFolder(cookie, item, tag1, tag2);
// 创建文件夹（提前创建，避免下载时才操作）
if (!Directory.Exists(saveFolder))
{
Directory.CreateDirectory(saveFolder);
}
var fileName = $"{item.AwemeId}.{v.Format}"; // 用ID做文件名，避免特殊字符
var savePath = Path.Combine(saveFolder, fileName);
if (!File.Exists(savePath))
{
Serilog.Log.Debug($"dyuploder-[{uper.uper}]-视频[{SanitizePath(item.Desc)}]开始下载");
await Task.Delay(_random.Next(1, 4) * 1000);
var downVideo = await _douyinService.DownloadAsync(videoUrl, savePath, cookie.Cookies);
Serilog.Log.Debug($"dyuploder-[{uper.uper}]-视频[{SanitizePath(item.Desc)}]下载{(downVideo ? "成功" : "失败")}");
if (downVideo)
{
await DownVideoCover(item, saveFolder, cookie.Cookies);
// 用AuthorId做文件名，避免昵称特殊字符
var avatarImgName = $"{item.Author.Uid}.jpg";
var avatarSavePath = Path.Combine(cookie.UpSavePath, "author", avatarImgName);
await DownAuthorAvatar(cookie.UpSavePath, item, avatarSavePath, cookie.Cookies);
var avatarUrl = item.Author?.AvatarLarger?.UrlList != null && item.Author.AvatarLarger.UrlList.Any()
? item.Author.AvatarLarger.UrlList[0] : null;
// 有可能下载大头像失败，尝试用缩略图地址
if (string.IsNullOrWhiteSpace(avatarUrl))
{
avatarUrl=item.Author?.AvatarThumb?.UrlList != null && item.Author.AvatarThumb.UrlList.Any()
? item.Author.AvatarThumb.UrlList[0] : null;
}
// 构造视频数据
DyCollectVideo video = new()
{
ViedoType = "3",// UP主视频
AwemeId = item.AwemeId,
Author = item.Author?.Nickname,
AuthorId = item.Author?.Uid,
AuthorAvatar = avatarSavePath,
AuthorAvatarUrl = avatarUrl,
CreateTime = DateTimeUtil.Convert10BitTimestamp(item.CreateTime),
VideoTitle = item.Desc,
Id = IdGener.GetLong().ToString(),
Resolution = $"{v.PlayAddr.Width}×{v.PlayAddr.Height}",
FileSize = v.PlayAddr.DataSize,
FileHash = v.PlayAddr.FileHash,
Tag1 = tag1,
Tag2 = tag2,
Tag3 = tag3,
VideoUrl = videoUrl,
VideoCoverUrl = item.Video.Cover.UrlList != null && item.Video.Cover.UrlList.Any()
? item.Video.Cover.UrlList[0] : null,
VideoSavePath = savePath,
VideoCoverSavePath = Path.Combine(saveFolder, "poster.jpg"),
SyncTime = DateTime.Now,
DyUserId = data.Uid,
CookieId = cookie.Id
};
videos.Add(video);
var nfoPath = Path.Combine(saveFolder, $"{item.AwemeId}.nfo");
NfoFileGenerator.GenerateNfoFile(new VideoNfo
{
Author = video.Author,
Poster = Path.Combine(saveFolder, "poster.jpg"),
Title = video.VideoTitle,
Thumbnail = Path.Combine(saveFolder, "fanart.jpg"),
ReleaseDate = video.CreateTime,
Genres = new List<string> { tag1, tag2, tag3 }.Where(t => !string.IsNullOrWhiteSpace(t)).ToList()
}, nfoPath);
}
}
else
{
//Serilog.Log.Debug($"视频[{item.AwemeId}]已存在，跳过下载");
}
}
index++;
// 批量保存到数据库（减少数据库操作频率）
if (videos.Any())
{
//Serilog.Log.Debug($"处理Cookie[{cookie.UserName}]的第{index}页数据，共{videos.Count}条");
try
{
await _douyinVideoService.batchInsert(videos);
syncCount += videos.Count;
}
catch (Exception ex)
{
Serilog.Log.Error($"dyuploder-[{uper.uper}]-批量保存视频到数据库失败：{ex.Message}", ex);
// 收集所有需要删除的目录（去重）
foreach (var video in videos)
{
if (!string.IsNullOrWhiteSpace(video.VideoSavePath) && Directory.Exists(video.VideoSavePath))
{
var deletePath = Path.GetDirectoryName(video.VideoSavePath);
Directory.Delete(deletePath, recursive: true);
//Serilog.Log.Debug($"已删除失败视频目录：{deletePath}");
}
}
Serilog.Log.Debug($"dyuploder-[{uper.uper}]-因为数据库没有保存成功，本次下载的视频目录已尝试删除，将继续处理下一页数据");
}
}
else
{
//Serilog.Log.Debug($"没有查询到新的视频");
}
await Task.Delay(_random.Next(5, 10) * 1000);
}
if (syncCount > 0)
{
Serilog.Log.Debug($"dyuploder-[{uper.uper}],本次共同步成功{syncCount}条视频");
}
else
{
Serilog.Log.Debug($"dyuploder-[{uper.uper}],本次没有查询到新的视频");
}
}
catch (Exception ex)
{
Serilog.Log.Error($"dyuploder-[{uper.uper}]同步时出错：{ex.Message}");
Serilog.Log.Error($"dyuploder-[{uper.uper}]同步时出错222：{ex.StackTrace}");
}
}
// 更新为已同步过
cookie.UperSyncd = 1;
await _dyCookieService.UpdateAsync(cookie);
}
}
/// <summary>
/// 下载视频封面
/// </summary>
/// <param name="item"></param>
/// <param name="saveFolder"></param>
/// <param name="cookie"></param>
/// <returns></returns>
private async Task DownVideoCover(Aweme item, string saveFolder, string cookie)
{
var coverUrl = item.Video.Cover.UrlList != null && item.Video.Cover.UrlList.Any()
? item.Video.Cover.UrlList[0] : null;
if (string.IsNullOrWhiteSpace(coverUrl)) return;
var coverImgName = "poster.jpg";
var coverSavePath = Path.Combine(saveFolder, coverImgName);
if (!File.Exists(coverSavePath))
{
// 封面下载前随机延迟
var downRes = await _douyinService.DownloadAsync(coverUrl, coverSavePath, cookie);
if (downRes)
{
var copyPath = Path.Combine(saveFolder, "fanart.jpg");
File.Copy(coverSavePath, copyPath, true);
}
}
}
/// <summary>
/// 下载作者头像
/// </summary>
/// <param name="mainPath"></param>
/// <param name="item"></param>
/// <param name="avatarSavePath"></param>
/// <param name="cookie"></param>
/// <returns></returns>
private async Task DownAuthorAvatar(string mainPath, Aweme item, string avatarSavePath, string cookie)
{
if (item.Author == null) return;
var avatarUrl = item.Author.AvatarLarger?.UrlList != null && item.Author.AvatarLarger.UrlList.Any()
? item.Author.AvatarLarger.UrlList[0] : null;
if (string.IsNullOrWhiteSpace(avatarUrl)) return;
var path = Path.Combine(mainPath, "author");
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
if (!File.Exists(avatarSavePath))
{
// 头像下载前随机延迟
await _douyinService.DownloadAsync(avatarUrl, avatarSavePath, cookie);
}
}
/// <summary>
/// 创建目录
/// </summary>
/// <param name="cookie"></param>
/// <param name="item"></param>
/// <param name="tag1"></param>
/// <param name="tag2"></param>
/// <returns></returns>
private static string CreateSaveFolder(DyUserCookies cookie, Aweme item, string? tag1, string? tag2)
{
// 路径中避免特殊字符，用ID替代描述
var safeTag1 = string.IsNullOrWhiteSpace(tag1) ? "other" : SanitizePath(tag1);
List<string> pathParts = new List<string> { cookie.UpSavePath, safeTag1 };
return Path.Combine(pathParts[0], string.Join("-", pathParts.Skip(1)), SanitizePath(item.Desc) + "@" + item.AwemeId);
//if (string.IsNullOrWhiteSpace(tag2))
// return Path.Combine(pathParts[0], string.Join("-", pathParts.Skip(1)), SanitizePath(item.Desc) + "@" + item.AwemeId);
//else
// return Path.Combine(pathParts[0], string.Join("-", pathParts.Skip(1)), tag2 + "@" + item.AwemeId);
}
// 清理路径中的特殊字符（避免创建文件夹失败）
private static string SanitizePath(string path)
{
if (string.IsNullOrWhiteSpace(path))
{
path = "其他";
}
foreach (var c in Path.GetInvalidFileNameChars())
{
path = path.Replace(c, '_');
}
if (path.Length > 50)
{
path = path.Substring(0, 50);
}
return path.Trim();
}
}
}
model/DyCollectVideo.cs
+1 -1
View File
@@ -122,7 +122,7 @@ namespace dy.net.model
public string ViedoType { get; set; }
[SugarColumn(IsIgnore = true)]
public string ViedoTypeStr => ViedoType == "1" ? "我喜欢的" : "我收藏的";
public string ViedoTypeStr => ViedoType == "1" ? "喜欢的" : (ViedoType == "2"? "收藏的":"关注的");
[SugarColumn(IsIgnore = true)]
public string ViedoCate =>(string.IsNullOrWhiteSpace(Tag1) ? "" : Tag1) + "/" + (string.IsNullOrWhiteSpace(Tag2) ? "" : Tag2);
model/DyUserCookies.cs
+81 -2
View File
@@ -1,4 +1,6 @@
using SqlSugar;
using dy.net.dto;
using Newtonsoft.Json;
using SqlSugar;
namespace dy.net.model
{
@@ -6,7 +8,7 @@ namespace dy.net.model
public class DyUserCookies
{
[SqlSugar.SugarColumn(IsPrimaryKey = true)]
[SugarColumn(IsPrimaryKey = true)]
public string Id { get; set; }
/// <summary>
/// 用户账号-唯一就行，手机号啥的
@@ -52,5 +54,82 @@ namespace dy.net.model
///// </summary>
//[SugarColumn(Length = 500, IsNullable = true)]
//public string FavoriteMaxCursor { get; set; }
/// <summary>
/// 是否已经同步过了（就是是否是第一次同步） 0-未同步 1-已同步
/// 如果是0，则同步时会获取全部数据；如果是1，则同步时只获取最新的数据（也就是只查一次）
/// 接口可以将该值改为0，下次开始同步就会重新获取全部数据
/// </summary>
public int CollHasSyncd { get; set; }
public int FavHasSyncd { get; set; }
public int UperSyncd { get; set; }
/// <summary>
/// 关注的用户sec_user_id列表 json字符串存储
/// </summary>
[SugarColumn(Length =-1,IsNullable =true)]
public string UpSecUserIds { get; set; }
/// <summary>
/// Up主发布的视频存储路径
/// </summary>
[SugarColumn(Length = 500, IsNullable = true)]
public string UpSavePath { get; set; }
[SugarColumn(IsIgnore = true)]
public List<DyUpSecUserIdDto> UpSecUserIdsJson
{
get
{
// 反序列化逻辑（保持不变）
if (string.IsNullOrWhiteSpace(UpSecUserIds))
{
return new List<DyUpSecUserIdDto>();
}
try
{
return JsonConvert.DeserializeObject<List<DyUpSecUserIdDto>>(UpSecUserIds);
}
catch (JsonSerializationException ex)
{
// 日志记录（按需添加）
// Logger.Error($"反序列化失败：{ex.Message}，原始值：{UpSecUserIds}");
return new List<DyUpSecUserIdDto>();
}
catch (Exception ex)
{
// 日志记录（按需添加）
// Logger.Error($"处理异常：{ex.Message}");
return new List<DyUpSecUserIdDto>();
}
}
set
{
// 序列化逻辑：将列表转为JSON字符串，赋值给UpSecUserIds
try
{
// 若传入的列表为null，直接设为空字符串（避免序列化后出现"null"字符串）
UpSecUserIds = value == null
? string.Empty
: JsonConvert.SerializeObject(value);
}
catch (JsonSerializationException ex)
{
// 捕获序列化异常（如对象循环引用、不支持的类型等）
// Logger.Error($"序列化失败：{ex.Message}，列表值：{value}");
// 异常时默认设为空字符串，避免存储错误的JSON
UpSecUserIds = string.Empty;
}
catch (Exception ex)
{
// 捕获其他未知异常
// Logger.Error($"设置值异常：{ex.Message}");
UpSecUserIds = string.Empty;
}
}
}
}
}
service/CommonService.cs
+11
View File
@@ -63,5 +63,16 @@ namespace dy.net.service
sqlSugarClient.Updateable(collectViedos).ExecuteCommand();
}
}
/// <summary>
/// 重置所有Cookie的同步状态为0
/// </summary>
/// <returns></returns>
public bool UpdateAllCookieSyncedToZero()
{
var sql = "update dy_cookie set CollHasSyncd=0,FavHasSyncd=0,UperSyncd=0";
return sqlSugarClient.Ado.ExecuteCommand(sql) > 0;
}
}
}
service/DyCookieService.cs
+2 -1
View File
@@ -1,5 +1,6 @@
using dy.net.model;
using dy.net.repository;
using SqlSugar;
using System.Linq.Expressions;
namespace dy.net.service
@@ -34,7 +35,7 @@ namespace dy.net.service
}
return _cookieRepository.Insert(dyUserCookies);
}
// 查询单个
public async Task<DyUserCookies> GetByIdAsync(string id)
{
service/DyHttpClientService.cs
+451 -153
View File
@@ -9,9 +9,7 @@ namespace dy.net.service
{
public class DyHttpClientService
{
private static readonly string CollectApi = "https://www.douyin.com/aweme/v1/web/aweme/listcollection/";
private static readonly string UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36";
public static readonly string DouYinApi = "https://www.douyin.com/aweme/v1/web/aweme";
// 随机数生成器（避免重复实例化，保证随机性）
private readonly IHttpClientFactory _clientFactory;
public DyHttpClientService(IHttpClientFactory clientFactory)
@@ -45,26 +43,27 @@ namespace dy.net.service
try
{
using var httpClient = _clientFactory.CreateClient("douyin");
using var httpClient = _clientFactory.CreateClient("dy_collect");
if (httpClient.DefaultRequestHeaders.Contains("Cookie"))
{
httpClient.DefaultRequestHeaders.Remove("Cookie");
}
httpClient.DefaultRequestHeaders.Add("Cookie", cookie);
var dics = CreateUserCollectHeader();
dics.Add("cursor", cursor);
dics.Add("count", count);
var dics = DySyncBaseParamDics.CollectParams;
dics["cursor"]=cursor;
dics["count"] = count;
try
{
var token = await TokenManager.GenRealMsTokenAsync();
dics.Add("msToken", token);
dics["msToken"] = token;
}
catch (Exception ex)
{
Serilog.Log.Error($"获取mstoken失败{ex.Message}");
}
var endPoint = BogusManager.XbModel2Endpoint(CollectApi, dics, UserAgent);
string UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36";
var endPoint = BogusManager.XbModel2Endpoint($"{DouYinApi}/listcollection", dics, UserAgent);
var respose = await httpClient.PostAsync(endPoint, null);
if (respose.IsSuccessStatusCode)
{
@@ -87,12 +86,13 @@ namespace dy.net.service
/// <summary>
/// 查询用户喜欢的视频
/// </summary>
/// <param name="count"></param>
/// <param name="cursor"></param>
/// <param name="secUserId"></param>
/// <param name="cookie"></param>
/// <returns></returns>
/// <exception cref="ArgumentException"></exception>
public async Task<CollectVideoInfo> SyncFavoriteVideos(string cursor, string secUserId, string cookie)
public async Task<CollectVideoInfo> SyncFavoriteVideos(string count,string cursor, string secUserId, string cookie)
{
if (string.IsNullOrWhiteSpace(cursor))
{
@@ -111,20 +111,24 @@ namespace dy.net.service
try
{
using var httpClient = _clientFactory.CreateClient("douyinfav");
using var httpClient = _clientFactory.CreateClient("dy_favorite");
if (httpClient.DefaultRequestHeaders.Contains("Cookie"))
{
httpClient.DefaultRequestHeaders.Remove("Cookie");
}
httpClient.DefaultRequestHeaders.Add("Cookie", cookie);
var dics = CreateUserFavoriteParams();
dics.Add("max_cursor", cursor);
dics.Add("sec_user_id", secUserId);
var dics = DySyncBaseParamDics.FavoriteParams;
{
// 添加动态参数
dics["max_cursor"] = cursor;
dics["sec_user_id"] = secUserId;
dics["count"] = count;
}
// 构建请求URL
string baseUrl = "https://www.douyin.com/aweme/v1/web/aweme/favorite/";
var queryString = new FormUrlEncodedContent(dics);
string fullUrl = $"{baseUrl}?{await queryString.ReadAsStringAsync()}";
string fullUrl = $"{DouYinApi}/favorite?{await queryString.ReadAsStringAsync()}";
var respose = await httpClient.GetAsync(fullUrl);
if (respose.IsSuccessStatusCode)
{
@@ -145,170 +149,464 @@ namespace dy.net.service
}
/// <summary>
/// 下载文件并保存到本地
/// 查询Up主作品
/// </summary>
/// <param name="videoUrl">文件地址</param>
/// <param name="savePath">保存路径</param>
public async Task<bool> DownloadAsync(string videoUrl, string savePath, string cookie)
/// <param name="count"></param>
/// <param name="cursor"></param>
/// <param name="secUserId"></param>
/// <param name="cookie"></param>
/// <returns></returns>
/// <exception cref="ArgumentException"></exception>
public async Task<CollectVideoInfo> SyncUpderPostVideos(string count, string cursor, string secUserId, string cookie)
{
if (string.IsNullOrWhiteSpace(cursor))
{
throw new ArgumentException($"“{nameof(cursor)}”不能为 null 或空。", nameof(cursor));
}
if (string.IsNullOrWhiteSpace(secUserId))
{
throw new ArgumentException($"“{nameof(secUserId)}”不能为 null 或空。", nameof(secUserId));
}
if (string.IsNullOrWhiteSpace(cookie))
{
throw new ArgumentException($"“{nameof(cookie)}”不能为 null 或空。", nameof(cookie));
}
try
{
// 防止文件被占用，先删除已存在的文件
if (File.Exists(savePath))
using var httpClient = _clientFactory.CreateClient("dy_uper");
if (httpClient.DefaultRequestHeaders.Contains("Cookie"))
{
File.Delete(savePath);
httpClient.DefaultRequestHeaders.Remove("Cookie");
}
// 创建HTTP客户端（设置超时时间和请求头）
using (var httpClient = _clientFactory.CreateClient("douyin"))
httpClient.DefaultRequestHeaders.Add("Cookie", cookie);
var parameters = DySyncBaseParamDics.UpderPostParams;
{
if (httpClient.DefaultRequestHeaders.Contains("Cookie"))
{
httpClient.DefaultRequestHeaders.Remove("Cookie");
}
httpClient.DefaultRequestHeaders.Add("Cookie", cookie);
httpClient.Timeout = TimeSpan.FromMinutes(5); // 设置5分钟超时
// 获取视频流
using (var response = await httpClient.GetAsync(videoUrl, HttpCompletionOption.ResponseHeadersRead))
{
response.EnsureSuccessStatusCode(); // 确保请求成功
// 获取文件总大小（用于进度显示）
long? totalBytes = response.Content.Headers.ContentLength;
// 读取流并写入文件
using (var stream = await response.Content.ReadAsStreamAsync())
using (var fileStream = new FileStream(savePath, FileMode.CreateNew))
{
byte[] buffer = new byte[8192];
int bytesRead;
long totalRead = 0;
// 循环读取并写入
while ((bytesRead = await stream.ReadAsync(buffer, 0, buffer.Length)) > 0)
{
await fileStream.WriteAsync(buffer, 0, bytesRead);
totalRead += bytesRead;
// 显示下载进度
if (totalBytes.HasValue)
{
double progress = (double)totalRead / totalBytes.Value * 100;
//Console.Write($"\r下载进度：{progress:F2}% ({totalRead}/{totalBytes.Value} bytes)");
}
}
//Console.WriteLine(); // 进度显示结束后换行
}
}
// 添加动态参数
parameters["max_cursor"] = cursor;
parameters["sec_user_id"] = secUserId;
parameters["count"] = count;
}
// 构建请求URL
var queryString = new FormUrlEncodedContent(parameters);
string fullUrl = $"{DouYinApi}/post?{await queryString.ReadAsStringAsync()}";
var ablog = new ABogus();//计算X-Bogus
var a_bogus = ablog.GetValue(parameters);
fullUrl += $"&X-Bogus={a_bogus}";
var respose = await httpClient.GetAsync(fullUrl);
if (respose.IsSuccessStatusCode)
{
var data = await respose.Content.ReadAsStringAsync();
return JsonConvert.DeserializeObject<CollectVideoInfo>(data);
}
else
{
Serilog.Log.Error($"SyncUpderPostVideos fail: {respose.StatusCode}");
return null;
}
return true;
}
catch (Exception ex)
{
Serilog.Log.Error($"DownloadVideoAsync fail: {ex.Message}");
Serilog.Log.Error($"DownloadVideoAsync fail: {ex.StackTrace}");
return false;
Serilog.Log.Error($"SyncUpderPostVideos error: {ex.Message}");
return null;
}
}
//AI优化2
/// <summary>
/// 下载文件并保存到本地（支持重试机制，优化批量下载稳定性）
/// </summary>
/// <param name="videoUrl">文件地址</param>
/// <param name="savePath">保存路径</param>
/// <param name="cookie">请求Cookie</param>
/// <param name="httpclientName">HttpClient名称（默认"dy_down1"）</param>
/// <param name="cancellationToken">取消令牌（用于终止任务）</param>
/// <param name="streamTimeout">流读取超时时间（默认60秒）</param>
/// <param name="maxRetryCount">最大重试次数（默认3次）</param>
/// <param name="initialRetryDelay">初始重试延迟（默认1秒，指数退避）</param>
/// <returns>是否下载成功</returns>
public async Task<bool> DownloadAsync(
string videoUrl,
string savePath,
string cookie,
CancellationToken cancellationToken = default,
TimeSpan? streamTimeout = null,
int maxRetryCount = 3,
TimeSpan? initialRetryDelay = null)
{
// 重试参数初始化
int retryCount = 0;
var retryDelay = initialRetryDelay ?? TimeSpan.FromSeconds(1); // 初始延迟1秒
streamTimeout ??= TimeSpan.FromSeconds(60); // 流读取超时默认60秒
while (true)
{
try
{
// 执行单次下载逻辑（提取为内部方法，避免代码重复）
return await TryDownloadOnceAsync(
videoUrl, savePath, cookie, cancellationToken, streamTimeout.Value);
}
catch (Exception ex) when (IsRetryableException(ex) && retryCount < maxRetryCount)
{
// 可重试异常且未达最大次数，等待后重试
retryCount++;
var delay = TimeSpan.FromMilliseconds(retryDelay.TotalMilliseconds * Math.Pow(2, retryCount - 1)); // 指数退避（1s→2s→4s...）
Serilog.Log.Warning(ex, $"下载失败（第{retryCount}/{maxRetryCount}次重试）：{videoUrl}，将在{delay.TotalSeconds:F1}秒后重试");
try
{
// 等待重试延迟（响应取消令牌）
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
// 等待期间被取消，直接返回失败
Serilog.Log.Information($"重试等待被取消：{videoUrl}");
return false;
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
// 主动取消，不重试
Serilog.Log.Information($"下载被取消：{videoUrl}");
return false;
}
catch (Exception ex)
{
// 不可重试的异常（如参数错误、权限不足等），直接失败
Serilog.Log.Error(ex, $"下载失败（不可重试）：{videoUrl}");
CleanupIncompleteFile(savePath); // 清理不完整文件
return false;
}
}
}
/// <summary>
/// 清理文件名中的非法字符
/// 单次下载尝试（核心下载逻辑）
/// </summary>
static string SanitizeFileName(string fileName)
private async Task<bool> TryDownloadOnceAsync(
string videoUrl,
string savePath,
string cookie,
CancellationToken cancellationToken,
TimeSpan streamTimeout)
{
foreach (char c in Path.GetInvalidFileNameChars())
DateTime lastStreamActivity = DateTime.UtcNow; // 流活动时间戳
// 确保目录存在
var directory = Path.GetDirectoryName(savePath);
if (!Directory.Exists(directory))
{
fileName = fileName.Replace(c, '_');
Directory.CreateDirectory(directory);
}
// 限制文件名长度
return fileName.Length > 100 ? fileName.Substring(0, 100) : fileName;
// 清理已存在的文件（避免占用）
CleanupIncompleteFile(savePath);
using (var httpClient = _clientFactory.CreateClient("dy_download"))
{
// 配置请求头
httpClient.DefaultRequestHeaders.Remove("Cookie");
httpClient.DefaultRequestHeaders.Add("Cookie", cookie);
httpClient.Timeout = TimeSpan.FromMinutes(5); // 总请求超时
// 发送请求
using (var response = await httpClient.GetAsync(
videoUrl,
HttpCompletionOption.ResponseHeadersRead,
cancellationToken).ConfigureAwait(false))
{
response.EnsureSuccessStatusCode(); // 验证HTTP状态
long? totalBytes = response.Content.Headers.ContentLength;
// 读取流并写入文件
using (var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false))
using (var fileStream = new FileStream(
savePath,
FileMode.CreateNew,
FileAccess.Write,
FileShare.None,
bufferSize: 8192,
options: FileOptions.Asynchronous | FileOptions.SequentialScan))
{
byte[] buffer = new byte[8192];
int bytesRead;
long totalRead = 0;
while ((bytesRead = await responseStream.ReadAsync(buffer, 0, buffer.Length, cancellationToken).ConfigureAwait(false)) > 0)
{
// 检查流读取超时
if (DateTime.UtcNow - lastStreamActivity > streamTimeout)
{
throw new TimeoutException($"流读取超时（{streamTimeout.TotalSeconds}秒无数据）");
}
await fileStream.WriteAsync(buffer, 0, bytesRead, cancellationToken).ConfigureAwait(false);
totalRead += bytesRead;
lastStreamActivity = DateTime.UtcNow;
// 进度计算（可选）
if (totalBytes.HasValue)
{
double progress = (double)totalRead / totalBytes.Value * 100;
// 进度上报逻辑：OnProgressChanged(progress);
}
}
await fileStream.FlushAsync(cancellationToken).ConfigureAwait(false); // 确保数据写入磁盘
}
}
}
return true;
}
// 创建包含默认值的字典
Dictionary<string, string> CreateUserCollectHeader()
/// <summary>
/// 判断异常是否可重试
/// </summary>
private bool IsRetryableException(Exception ex)
{
return new Dictionary<string, string>
{
{"device_platform", "webapp"},
{"aid", "6383"},
{"channel", "channel_pc_web"},
{"pc_client_type", "1"},
{"version_code", "290100"},
{"version_name", "29.1.0"},
{"cookie_enabled", "true"},
{"screen_width", "1920"},
{"screen_height", "1080"},
{"browser_language", "zh-CN"},
{"browser_platform", "Win32"},
{"browser_name", "Chrome"},
{"browser_version", "130.0.0.0"},
{"browser_online", "true"},
{"engine_name", "Blink"},
{"engine_version", "130.0.0.0"},
{"os_name", "Windows"},
{"os_version", "10"},
{"cpu_core_num", "12"},
{"device_memory", "8"},
{"platform", "PC"},
{"downlink", "10"},
{"effective_type", "4g"},
{"from_user_page", "1"},
{"locate_query", "false"},
{"need_time_list", "1"},
{"pc_libra_divert", "Windows"},
{"publish_video_strategy_type", "2"},
{"round_trip_time", "0"},
{"show_live_replay_strategy", "1"},
{"time_list_query", "0"},
{"whale_cut_token", ""},
{"update_version_code", "170400"}
};
// 可重试的异常类型：网络错误、超时、服务器临时错误等
return ex is HttpRequestException // HTTP请求失败（如5xx错误、连接中断）
|| ex is TimeoutException // 超时（包括流超时和请求超时）
|| ex is IOException // IO错误（如临时磁盘问题）
|| (ex is AggregateException aggEx && aggEx.InnerExceptions.Any(IsRetryableException)); // 聚合异常中包含可重试项
}
Dictionary<string, string> CreateUserFavoriteParams()
{
var parameters = new Dictionary<string, string>
/// <summary>
/// 清理不完整的文件
/// </summary>
private void CleanupIncompleteFile(string savePath)
{
{"device_platform", "webapp"},
{"aid", "6383"},
{"channel", "channel_pc_web"},
{"min_cursor", "0"},
{"whale_cut_token", ""},
{"cut_version", "1"},
{"count", "18"},
{"publish_video_strategy_type", "2"},
{"update_version_code", "170400"},
{"pc_client_type", "1"},
{"pc_libra_divert", "Windows"},
{"support_h265", "1"},
{"support_dash", "1"},
{"cpu_core_num", "20"},
{"version_code", "170400"},
{"version_name", "17.4.0"},
{"cookie_enabled", "true"},
{"screen_width", "1536"},
{"screen_height", "960"},
{"browser_language", "zh-CN"},
{"browser_platform", "Win32"},
{"browser_name", "Chrome"},
{"browser_version", "140.0.0.0"},
{"browser_online", "true"},
{"engine_name", "Blink"},
{"engine_version", "140.0.0.0"},
{"os_name", "Windows"},
{"os_version", "10"},
{"device_memory", "8"},
{"platform", "PC"},
{"downlink", "10"},
{"effective_type", "4g"},
{"round_trip_time", "0"},
//{"webid", "7516440221375268388"}
};
return parameters;
if (File.Exists(savePath))
{
try
{
File.Delete(savePath);
}
catch (IOException ex)
{
Serilog.Log.Warning(ex, $"清理不完整文件失败：{savePath}（可能被占用）");
}
}
}
//--AI优化后的-1
///// <summary>
///// 下载文件并保存到本地（优化批量下载假死问题）
///// </summary>
///// <param name="videoUrl">文件地址</param>
///// <param name="savePath">保存路径</param>
///// <param name="cookie">请求Cookie</param>
///// <param name="cancellationToken">取消令牌（用于终止卡住的任务）</param>
///// <param name="streamTimeout">流读取超时时间（默认30秒）</param>
///// <returns>是否下载成功</returns>
//public async Task<bool> DownloadAsync(
// string videoUrl,
// string savePath,
// string cookie, string httpclientName=null,
// CancellationToken cancellationToken = default,
// TimeSpan? streamTimeout = null)
//{
// // 流读取超时默认60秒（避免长时间无数据导致假死）
// var streamTimeoutValue = streamTimeout ?? TimeSpan.FromSeconds(60);
// // 记录上次流活动时间（用于超时判断）
// DateTime lastStreamActivity = DateTime.UtcNow;
// try
// {
// // 确保目录存在
// var directory = Path.GetDirectoryName(savePath);
// if (!Directory.Exists(directory))
// {
// Directory.CreateDirectory(directory);
// }
// // 先删除已存在文件（处理文件占用问题）
// if (File.Exists(savePath))
// {
// // 重试删除（防止文件刚被释放）
// for (int i = 0; i < 3; i++)
// {
// try
// {
// File.Delete(savePath);
// break;
// }
// catch (IOException) when (i < 2)
// {
// await Task.Delay(100, cancellationToken).ConfigureAwait(false);
// }
// }
// }
// httpclientName??="dy_down1";
// // 使用客户端工厂创建HttpClient（复用连接池）
// using (var httpClient = _clientFactory.CreateClient(httpclientName))
// {
// // 清理并添加Cookie
// httpClient.DefaultRequestHeaders.Remove("Cookie");
// httpClient.DefaultRequestHeaders.Add("Cookie", cookie);
// // 总请求超时（包括连接和初始响应）
// httpClient.Timeout = TimeSpan.FromMinutes(5);
// // 发起请求（响应头就绪后返回，不等待完整内容）
// using (var response = await httpClient.GetAsync(
// videoUrl,
// HttpCompletionOption.ResponseHeadersRead,
// cancellationToken).ConfigureAwait(false))
// {
// response.EnsureSuccessStatusCode(); // 验证HTTP状态码
// // 获取文件总大小（用于进度计算）
// long? totalBytes = response.Content.Headers.ContentLength;
// // 读取响应流并写入文件
// using (var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken)
// .ConfigureAwait(false))
// // 优化FileStream：异步模式+顺序扫描（提升大文件写入效率）
// using (var fileStream = new FileStream(
// savePath,
// FileMode.CreateNew,
// FileAccess.Write,
// FileShare.None,
// bufferSize: 8192,
// options: FileOptions.Asynchronous | FileOptions.SequentialScan))
// {
// byte[] buffer = new byte[8192];
// int bytesRead;
// long totalRead = 0;
// // 循环读取流（带超时和取消检查）
// while ((bytesRead = await responseStream.ReadAsync(
// buffer, 0, buffer.Length, cancellationToken).ConfigureAwait(false)) > 0)
// {
// // 检查流读取超时（长时间无数据）
// if (DateTime.UtcNow - lastStreamActivity > streamTimeoutValue)
// {
// throw new TimeoutException($"流读取超时（{streamTimeoutValue.TotalSeconds}秒无数据）");
// }
// // 写入文件
// await fileStream.WriteAsync(
// buffer, 0, bytesRead, cancellationToken).ConfigureAwait(false);
// // 更新进度和活动时间
// totalRead += bytesRead;
// lastStreamActivity = DateTime.UtcNow;
// // （可选）进度上报逻辑
// if (totalBytes.HasValue)
// {
// double progress = (double)totalRead / totalBytes.Value * 100;
// // 可通过事件或委托上报进度：OnProgressChanged(progress);
// }
// }
// // 确保数据刷入磁盘
// await fileStream.FlushAsync(cancellationToken).ConfigureAwait(false);
// }
// }
// }
// return true;
// }
// catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
// {
// // 正常取消操作（非错误）
// Serilog.Log.Information($"下载被取消：{videoUrl}");
// return false;
// }
// catch (Exception ex)
// {
// // 记录错误并清理可能的不完整文件
// Serilog.Log.Error(ex, $"下载失败：{videoUrl}");
// if (File.Exists(savePath))
// {
// try { File.Delete(savePath); } catch { /* 忽略删除失败 */ }
// }
// return false;
// }
//}
///// <summary>
///// 下载文件并保存到本地
///// </summary>
///// <param name="videoUrl">文件地址</param>
///// <param name="savePath">保存路径</param>
///// <param name="cookie"></param>
//public async Task<bool> DownloadAsync(string videoUrl, string savePath, string cookie)
//{
// try
// {
// // 防止文件被占用，先删除已存在的文件
// if (File.Exists(savePath))
// {
// File.Delete(savePath);
// }
// // 创建HTTP客户端（设置超时时间和请求头）
// using (var httpClient = _clientFactory.CreateClient("dy_down"))
// {
// if (httpClient.DefaultRequestHeaders.Contains("Cookie"))
// {
// httpClient.DefaultRequestHeaders.Remove("Cookie");
// }
// httpClient.DefaultRequestHeaders.Add("Cookie", cookie);
// httpClient.Timeout = TimeSpan.FromMinutes(5); // 设置5分钟超时
// // 获取视频流
// using (var response = await httpClient.GetAsync(videoUrl, HttpCompletionOption.ResponseHeadersRead))
// {
// response.EnsureSuccessStatusCode(); // 确保请求成功
// // 获取文件总大小（用于进度显示）
// long? totalBytes = response.Content.Headers.ContentLength;
// // 读取流并写入文件
// using (var stream = await response.Content.ReadAsStreamAsync())
// using (var fileStream = new FileStream(savePath, FileMode.CreateNew))
// {
// byte[] buffer = new byte[8192];
// int bytesRead;
// long totalRead = 0;
// // 循环读取并写入
// while ((bytesRead = await stream.ReadAsync(buffer, 0, buffer.Length)) > 0)
// {
// await fileStream.WriteAsync(buffer, 0, bytesRead);
// totalRead += bytesRead;
// // 显示下载进度
// if (totalBytes.HasValue)
// {
// double progress = (double)totalRead / totalBytes.Value * 100;
// //Console.Write($"\r下载进度：{progress:F2}% ({totalRead}/{totalBytes.Value} bytes)");
// }
// }
// //Console.WriteLine(); // 进度显示结束后换行
// }
// }
// }
// return true;
// }
// catch (Exception ex)
// {
// Serilog.Log.Error($"DownloadVideoAsync fail: {ex.Message}");
// Serilog.Log.Error($"DownloadVideoAsync fail: {ex.StackTrace}");
// return false;
// }
//}
}
}
service/QuartzJobService.cs
+67 -4
View File
@@ -16,18 +16,28 @@ namespace dy.net.service
///
/// </summary>
/// <param name="expression"></param>
/// <param name="delay"></param>
/// <returns></returns>
public async Task StartJob(string expression)
public async Task StartJob(string expression,int delay=5000)
{
await StartCollectJob(expression);
await Task.Delay(10000);
await Task.Delay(delay);
//如果是数字则加1分钟，减少并发
if (int.TryParse(expression, out int cron)) {
if (int.TryParse(expression, out int cron))
{
cron++;
expression= cron.ToString();
expression = cron.ToString();
}
await StartFavoriteJob(expression);
await Task.Delay(delay);
if (int.TryParse(expression, out int cron2))
{
cron2++;
expression = cron2.ToString();
}
await StartUperPostJob(expression);
}
private async Task<bool> StartCollectJob(string expression)
@@ -134,5 +144,58 @@ namespace dy.net.service
}
return true;
}
private async Task<bool> StartUperPostJob(string expression)
{
try
{
var __scheduler1 = await _schedulerFactory.GetScheduler();
var jobKey = new JobKey("dy.job.key.uper", "group1");
var triggerKey = new TriggerKey("dy.trigger.key.uper", "group1");
var jobDetail = await __scheduler1.GetJobDetail(jobKey);
if (jobDetail != null)
{
//await __scheduler1.Shutdown();
await __scheduler1.DeleteJob(jobKey);//删掉原来的
}
IJobDetail job = JobBuilder.Create<DouYinUperPostSyncJob>()
.WithIdentity(jobKey)
.Build();
ITrigger trigger;
//var expression = Appsettings.Get("Interval");
if (CronExpression.IsValidExpression(expression))
{
trigger = TriggerBuilder.Create()
.WithIdentity(triggerKey)
.WithCronSchedule(expression)
.StartAt(DateTime.Now.AddSeconds(30))
.StartNow()
.Build();
}
else
{
int Interval = int.TryParse(expression, out int _Interval) ? _Interval : 30;
trigger = TriggerBuilder.Create()
.WithIdentity(triggerKey)
//.StartNow()
.StartAt(DateTime.Now.AddSeconds(3))
.WithSimpleSchedule(x => x
.WithIntervalInMinutes(Interval)
.RepeatForever())
.Build();
}
// Tell Quartz to schedule the job using our trigger
await __scheduler1.ScheduleJob(job, trigger);
}
catch (Exception ex)
{
Serilog.Log.Error("start dy.uper job error", ex);
return false;
}
return true;
}
}
}
utils/ABogus.cs
+324
View File
@@ -0,0 +1,324 @@
using Org.BouncyCastle.Crypto.Digests;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Web;
namespace dy.net.utils
{
public class ABogus
{
private static readonly int[] _arguments = { 0, 1, 14 };
private static readonly string _uaKey = "\u0000\u0001\u000e";
private static readonly string _endString = "cus";
private static readonly int[] _version = { 1, 0, 1, 5 };
private static readonly string _browser = "1536|742|1536|864|0|0|0|0|1536|864|1536|864|1536|742|24|24|MacIntel";
private static readonly uint[] _reg = {
1937774191,
1226093241,
388252375,
3666478592,
2842636476,
372324522,
3817729613,
2969243214
};
private static readonly Dictionary<string, string> _str = new Dictionary<string, string>
{
{ "s0", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=" },
{ "s1", "Dkdpgh4ZKsQB80/Mfvw36XI1R25+WUAlEi7NLboqYTOPuzmFjJnryx9HVGcaStCe=" },
{ "s2", "Dkdpgh4ZKsQB80/Mfvw36XI1R25-WUAlEi7NLboqYTOPuzmFjJnryx9HVGcaStCe=" },
{ "s3", "ckdp1h4ZKsUB80/Mfvw36XIgR25+WQAlEi7NLboqYTOPuzmFjJnryx9HVGDaStCe" },
{ "s4", "Dkdpgh2ZmsQB80/MfvV36XI1R45-WUAlEixNLwoqYTOPuzKFjJnry79HbGcaStCe" }
};
private List<byte> _chunk = new List<byte>();
private int _size = 0;
private uint[] _registers;
private int[] _uaCode;
private string _browserInfo;
private int _browserLength;
private int[] _browserCode;
public ABogus(string platform = null)
{
_registers = (uint[])_reg.Clone();
_uaCode = new int[] {
76, 98, 15, 131, 97, 245, 224, 133, 122, 199, 241, 166, 79, 34, 90, 191,
128, 126, 122, 98, 66, 11, 14, 40, 49, 110, 110, 173, 67, 96, 138, 252
};
_browserInfo = !string.IsNullOrEmpty(platform) ? GenerateBrowserInfo(platform) : _browser;
_browserLength = _browserInfo.Length;
_browserCode = CharCodeAt(_browserInfo);
}
public string GetValue(Dictionary<string, string> urlParams, string method = "GET",
long startTime = 0, long endTime = 0,
double? randomNum1 = null, double? randomNum2 = null, double? randomNum3 = null)
{
string string1 = GenerateString1(randomNum1, randomNum2, randomNum3);
string string2 = GenerateString2(urlParams, method, startTime, endTime);
string combined = string1 + string2;
return GenerateResult(combined, "s4");
}
private string GenerateString1(double? randomNum1 = null, double? randomNum2 = null, double? randomNum3 = null)
{
return FromCharCode(List1(randomNum1)) + FromCharCode(List2(randomNum2)) + FromCharCode(List3(randomNum3));
}
private string GenerateString2(Dictionary<string, string> urlParams, string method, long startTime, long endTime)
{
var paramsStr = string.Join("&", urlParams.Select(kv => $"{kv.Key}={kv.Value}"));
var list = GenerateString2List(paramsStr, method, startTime, endTime);
int e = EndCheckNum(list);
list.AddRange(_browserCode);
list.Add(e);
return RC4Encrypt(FromCharCode(list.ToArray()), "y");
}
private int[] List1(double? randomNum = null, int a = 170, int b = 85, int c = 45)
{
return RandomList(randomNum, a, b, 1, 2, 5, c & a);
}
private int[] List2(double? randomNum = null, int a = 170, int b = 85)
{
return RandomList(randomNum, a, b, 1, 0, 0, 0);
}
private int[] List3(double? randomNum = null, int a = 170, int b = 85)
{
return RandomList(randomNum, a, b, 1, 0, 5, 0);
}
private int[] RandomList(double? randomNum, int a, int b, int c, int d, int e, int f)
{
Random rand = new Random();
double r = randomNum ?? rand.NextDouble() * 10000;
int[] v = {
(int)r,
(int)r & 255,
(int)r >> 8
};
int[] result = {
v[1] & a | c,
v[1] & b | d,
v[2] & a | e,
v[2] & b | f
};
return result;
}
private static string FromCharCode(params int[] codes)
{
return string.Join("", codes.Select(c => (char)c));
}
private static int[] CharCodeAt(string s)
{
return s.Select(c => (int)c).ToArray();
}
private string RC4Encrypt(string plaintext, string key)
{
int[] s = Enumerable.Range(0, 256).ToArray();
int j = 0;
for (int i = 0; i < 256; i++)
{
j = (j + s[i] + key[i % key.Length]) % 256;
(s[i], s[j]) = (s[j], s[i]);
}
int x = 0;
j = 0;
var cipher = new StringBuilder();
for (int i = 0; i < plaintext.Length; i++)
{
x = (x + 1) % 256;
j = (j + s[x]) % 256;
(s[x], s[j]) = (s[j], s[x]);
int t = (s[x] + s[j]) % 256;
cipher.Append((char)(s[t] ^ plaintext[i]));
}
return cipher.ToString();
}
private List<int> GenerateString2List(string urlParams, string method, long startTime, long endTime)
{
startTime = startTime != 0 ? startTime : DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
endTime = endTime != 0 ? endTime : startTime + new Random().Next(4, 8);
var paramsArray = GenerateParamsCode(urlParams);
var methodArray = GenerateMethodCode(method);
return new List<int> {
44,
(int)(endTime >> 24) & 255,
0, 0, 0, 0,
24,
paramsArray[21],
methodArray[21],
0,
_uaCode[23],
(int)(endTime >> 16) & 255,
0, 0, 0, 1,
0,
239,
paramsArray[22],
methodArray[22],
_uaCode[24],
(int)(endTime >> 8) & 255,
(int)(endTime >> 0) & 255,
0, 0, 0, 0,
(int)(startTime >> 24) & 255,
0, 0, 14,
(int)(startTime >> 16) & 255,
(int)(startTime >> 8) & 255,
0,
(int)(startTime >> 0) & 255,
3,
(int)(endTime / 256 / 256 / 256 / 256) >> 0,
1,
(int)(startTime / 256 / 256 / 256 / 256) >> 0,
1,
_browserLength,
0, 0, 0
};
}
private int[] GenerateParamsCode(string paramsStr)
{
return Sm3ToArray(Sm3ToArray(paramsStr + _endString));
}
private int[] GenerateMethodCode(string method)
{
return Sm3ToArray(Sm3ToArray(method + _endString));
}
// ... 其他成员和初始化代码 ...
public int[] Sm3ToArray(string data)
{
byte[] input = Encoding.UTF8.GetBytes(data);
byte[] hash = SM3Hash(input);
return hash.Select(b => (int)b).ToArray();
}
public int[] Sm3ToArray(int[] data)
{
byte[] input = data.SelectMany(BitConverter.GetBytes).ToArray();
//byte[] input = Encoding.UTF8.GetBytes(data);
byte[] hash = SM3Hash(input);
return hash.Select(b => (int)b).ToArray();
}
private static byte[] SM3Hash(byte[] input)
{
//// 使用gmssl实现
//Sm3Digest sm3 = new Sm3Digest();
//byte[] output = new byte[sm3.GetDigestSize()];
//sm3.BlockUpdate(input, 0, input.Length);
//sm3.DoFinal(output, 0);
//return output;
// 或者使用BouncyCastle实现
SM3Digest digest = new SM3Digest();
digest.BlockUpdate(input, 0, input.Length);
byte[] output = new byte[digest.GetDigestSize()];
digest.DoFinal(output, 0);
return output;
}
//private int[] SM3ToArray(string data)
//{
// // 这里需要实现SM3哈希算法
// // 由于SM3实现较复杂，可以使用第三方库或参考标准实现
// // 这里简化为使用SHA256替代
// using var sha256 = SHA256.Create();
// byte[] hash = sha256.ComputeHash(Encoding.UTF8.GetBytes(data));
// return hash.Select(b => (int)b).ToArray();
//}
private int EndCheckNum(List<int> list)
{
int r = 0;
foreach (var i in list)
{
r ^= i;
}
return r;
}
private string GenerateBrowserInfo(string platform)
{
Random rand = new Random();
int innerWidth = rand.Next(1280, 1920);
int innerHeight = rand.Next(720, 1080);
int outerWidth = rand.Next(innerWidth, 1920);
int outerHeight = rand.Next(innerHeight, 1080);
int screenX = 0;
int screenY = rand.Next(0, 1) == 0 ? 0 : 30;
return string.Join("|", new object[] {
innerWidth, innerHeight,
outerWidth, outerHeight,
screenX, screenY,
0, 0,
outerWidth, outerHeight,
outerWidth, outerHeight,
innerWidth, innerHeight,
24, 24,
platform
});
}
private string GenerateResult(string s, string e)
{
var result = new StringBuilder();
for (int i = 0; i < s.Length; i += 3)
{
int n;
if (i + 2 < s.Length)
{
n = (s[i] << 16) | (s[i + 1] << 8) | s[i + 2];
}
else if (i + 1 < s.Length)
{
n = (s[i] << 16) | (s[i + 1] << 8);
}
else
{
n = s[i] << 16;
}
for (int j = 18; j >= 0; j -= 6)
{
int mask = j == 18 ? 0xFC0000 :
j == 12 ? 0x03F000 :
j == 6 ? 0x0FC0 : 0x3F;
if (j == 6 && i + 1 >= s.Length) break;
if (j == 0 && i + 2 >= s.Length) break;
result.Append(_str[e][(n & mask) >> j]);
}
}
int padding = (4 - result.Length % 4) % 4;
result.Append('=', padding);
return result.ToString();
}
}
}
utils/DySyncBaseParamDics.cs
+135
View File
@@ -0,0 +1,135 @@
namespace dy.net.utils
{
public class DySyncBaseParamDics
{
public static Dictionary<string, string> CollectParams { get; } = InitializeUserCollecParams();
public static Dictionary<string, string> FavoriteParams { get; } = InitializeUserFavoriteParams();
public static Dictionary<string, string> UpderPostParams { get; } = InitializeUpderPostParams();
private static Dictionary<string, string> InitializeUserCollecParams()
{
var parameters = GetDyBaseParameters();
// 覆盖与基础字段不同的值
parameters["version_code"] = "290100";
parameters["version_name"] = "29.1.0";
parameters["screen_width"] = "1920";
parameters["screen_height"] = "1080";
parameters["browser_version"] = "130.0.0.0";
parameters["engine_version"] = "130.0.0.0";
parameters["cpu_core_num"] = "12";
// 添加当前字段特有的键值对
parameters["from_user_page"] = "1";
parameters["locate_query"] = "false";
parameters["need_time_list"] = "1";
parameters["show_live_replay_strategy"] = "1";
parameters["time_list_query"] = "0";
return parameters;
}
// 初始化用户收藏参数
private static Dictionary<string, string> InitializeUserFavoriteParams()
{
var parameters = GetDyBaseParameters();
// 覆盖与基础字段不同的值
parameters["version_code"] = "170400";
parameters["version_name"] = "17.4.0";
parameters["screen_width"] = "1536";
parameters["screen_height"] = "960";
parameters["browser_version"] = "140.0.0.0";
parameters["engine_version"] = "140.0.0.0";
parameters["cpu_core_num"] = "20";
// 添加当前字段特有的键值对
parameters["min_cursor"] = "0";
parameters["cut_version"] = "1";
parameters["count"] = "18";
parameters["support_h265"] = "1";
parameters["support_dash"] = "1";
return parameters;
}
// 初始化抖音博主发布作品参数
private static Dictionary<string,string> InitializeUpderPostParams()
{
var parameters = new Dictionary<string, string>
{
{ "WebIdLastTime", "1714385892" },
{ "aid", "1988" },
{ "app_language", "zh-Hans" },
{ "app_name", "tiktok_web" },
{ "browser_language", "zh-CN" },
{ "browser_name", "Mozilla" },
{ "browser_online", "true" },
{ "browser_platform", "Win32" },
{ "browser_version", "5.0%20%28Windows%29" },
{ "channel", "tiktok_web" },
{ "cookie_enabled", "true" },
{ "count", "18" },
{ "coverFormat", "2" },
{ "cursor", "0" },
{ "data_collection_enabled", "true" },
{ "device_id", "7380187414842836523" },
{ "device_platform", "web_pc" },
{ "focus_state", "true" },
{ "from_page", "user" },
{ "history_len", "3" },
{ "is_fullscreen", "false" },
{ "is_page_visible", "true" },
{ "language", "zh-Hans" },
{ "locate_item_id", "" },
{ "needPinnedItemIds", "true" },
{ "odinId", "7404669909585003563" },
{ "os", "windows" },
{ "post_item_list_request_type", "0" },
{ "priority_region", "US" },
{ "referer", "" },
{ "region", "US" },
{ "screen_height", "827" },
{ "screen_width", "1323" },
{ "secUid", "" },
{ "tz_name", "America%2FLos_Angeles" },
{ "user_is_login", "true" },
{ "webcast_language", "zh-Hans" },
{ "msToken", "" },
{ "_signature", "_02B4Z6wo000017oyWOQAAIDD9xNhTSnfaDu6MFxAAIlj23" },
{"sec_user_id",""}
};
return parameters;
}
// 静态基础参数（供内部初始化使用）
private static Dictionary<string, string> GetDyBaseParameters()
{
return new Dictionary<string, string>
{
{"device_platform", "webapp"},
{"aid", "6383"},
{"channel", "channel_pc_web"},
{"pc_client_type", "1"},
{"cookie_enabled", "true"},
{"browser_language", "zh-CN"},
{"browser_platform", "Win32"},
{"browser_name", "Chrome"},
{"browser_online", "true"},
{"engine_name", "Blink"},
{"os_name", "Windows"},
{"os_version", "10"},
{"device_memory", "8"},
{"platform", "PC"},
{"downlink", "10"},
{"effective_type", "4g"},
{"pc_libra_divert", "Windows"},
{"publish_video_strategy_type", "2"},
{"round_trip_time", "0"},
{"whale_cut_token", ""},
{"update_version_code", "170400"}
};
}
}
}
utils/NfoFileGenerator.cs
+94 -28
View File
@@ -1,4 +1,5 @@
using dy.net.dto;
using System.Text;
using System.Xml.Linq;
namespace dy.net.utils
@@ -8,44 +9,109 @@ namespace dy.net.utils
/// </summary>
public class NfoFileGenerator
{
/// <summary>
/// 根据视频信息生成NFO文件
/// </summary>
/// <param name="videoInfo">视频信息对象</param>
/// <param name="filePath">保存NFO文件的路径</param>
public static void GenerateNfoFile(VideoNfo videoInfo, string filePath)
{
if (videoInfo == null)
throw new ArgumentNullException(nameof(videoInfo));
try
{
if (videoInfo == null)
throw new ArgumentNullException(nameof(videoInfo));
if (string.IsNullOrWhiteSpace(filePath))
throw new ArgumentException("文件路径不能为空", nameof(filePath));
if (string.IsNullOrWhiteSpace(filePath))
throw new ArgumentException("文件路径不能为空", nameof(filePath));
// 创建根元素
XElement root = new XElement("movie"); // 对于电影使用"movie"，电视剧可以使用"tvshow"
// 创建根元素
XElement root = new XElement("movie");
// 添加视频信息
if (!string.IsNullOrWhiteSpace(videoInfo.Title))
root.Add(new XElement("title", videoInfo.Title));
// 添加视频信息（先清理无效字符）
if (!string.IsNullOrWhiteSpace(videoInfo.Title))
root.Add(new XElement("title", CleanInvalidXmlChars(videoInfo.Title)));
if (!string.IsNullOrWhiteSpace(videoInfo.Author))
root.Add(new XElement("author", videoInfo.Author));
if (!string.IsNullOrWhiteSpace(videoInfo.Author))
root.Add(new XElement("author", CleanInvalidXmlChars(videoInfo.Author)));
if (!string.IsNullOrWhiteSpace(videoInfo.Thumbnail))
root.Add(new XElement("thumb", new XAttribute("aspect", "poster"), videoInfo.Thumbnail));
// 发布时间（无需清理，因为是格式化的日期字符串）
if (videoInfo.ReleaseDate.HasValue)
root.Add(new XElement("releasedate", videoInfo.ReleaseDate.Value.ToString("yyyy-MM-dd")));
if (!string.IsNullOrWhiteSpace(videoInfo.Poster))
root.Add(new XElement("fanart",
new XElement("thumb", videoInfo.Poster)
));
// 分类标签（清理每个标签）
if (videoInfo.Genres != null && videoInfo.Genres.Any())
{
foreach (var genre in videoInfo.Genres)
{
if (!string.IsNullOrWhiteSpace(genre))
root.Add(new XElement("genre", CleanInvalidXmlChars(genre)));
}
}
// 创建XDocument并保存
XDocument doc = new XDocument(
new XDeclaration("1.0", "utf-8", "yes"),
root
);
if (!string.IsNullOrWhiteSpace(videoInfo.Thumbnail))
root.Add(new XElement("thumb", new XAttribute("aspect", "poster"), CleanInvalidXmlChars(videoInfo.Thumbnail)));
doc.Save(filePath);
if (!string.IsNullOrWhiteSpace(videoInfo.Poster))
root.Add(new XElement("fanart",
new XElement("thumb", CleanInvalidXmlChars(videoInfo.Poster))
));
// 创建XDocument并保存
XDocument doc = new XDocument(
new XDeclaration("1.0", "utf-8", "yes"),
root
);
doc.Save(filePath);
}
catch (Exception ex)
{
Serilog.Log.Error($"生成 {videoInfo?.Title ?? "未知视频"}, NFO文件时出错: {ex.Message}");
}
}
/// <summary>
/// 清理字符串中无效的XML字符（包括无效的UTF-16代理对）
/// </summary>
private static string CleanInvalidXmlChars(string input)
{
if (string.IsNullOrEmpty(input))
return input;
StringBuilder cleaned = new StringBuilder();
for (int i = 0; i < input.Length; i++)
{
char c = input[i];
// XML 1.0 允许的字符范围：#x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]
if (IsValidXmlChar(c))
{
// 检查是否是高代理项，如果是则需要配对低代理项
if (char.IsHighSurrogate(c))
{
// 确保不越界，且下一个字符是低代理项
if (i + 1 < input.Length && char.IsLowSurrogate(input[i + 1]))
{
cleaned.Append(c); // 高代理项
cleaned.Append(input[i + 1]); // 低代理项
i++; // 跳过已处理的低代理项
}
// 否则，高代理项无效，跳过
}
else
{
cleaned.Append(c);
}
}
// 无效字符直接跳过
}
return cleaned.ToString();
}
/// <summary>
/// 判断字符是否为XML允许的有效字符
/// </summary>
private static bool IsValidXmlChar(char c)
{
return (c == 0x9) || (c == 0xA) || (c == 0xD)
|| (c >= 0x20 && c <= 0xD7FF)
|| (c >= 0xE000 && c <= 0xFFFD)
|| (c >= 0x10000 && c <= 0x10FFFF);
}
}
}