Files
douyin/service/DouyinHttpClientService.cs
T

1056 lines
51 KiB
C#

using dy.net.model.dto;
using dy.net.model.entity;
using dy.net.model.response;
using dy.net.utils;
using Microsoft.AspNetCore.WebUtilities;
using Microsoft.Extensions.Primitives;
using Newtonsoft.Json;
using Serilog;
using System.IO;
using System.Net;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using dy.net.storage;
namespace dy.net.service
{
public class DouyinHttpClientService : IDisposable
{
private readonly IHttpClientFactory _clientFactory;
private readonly MediaStorageRouter _storageRouter;
private readonly Func<TimeSpan, CancellationToken, Task> _delayAsync;
private const int TransientRequestRetryCount = 2;
private const int MediaNetworkRetryCount = 2;
private readonly JsonSerializerSettings _jsonSettings=new JsonSerializerSettings
{
NullValueHandling = NullValueHandling.Ignore,
MissingMemberHandling = MissingMemberHandling.Ignore
};
private bool _disposedValue;
public DouyinHttpClientService(IHttpClientFactory clientFactory, MediaStorageRouter storageRouter)
: this(clientFactory, storageRouter, Task.Delay)
{
}
internal DouyinHttpClientService(
IHttpClientFactory clientFactory,
MediaStorageRouter storageRouter,
Func<TimeSpan, CancellationToken, Task> delayAsync)
{
_clientFactory = clientFactory;
_storageRouter = storageRouter;
_delayAsync = delayAsync ?? Task.Delay;
}
/// <summary>
/// 异步获取HTTP响应消息(优化版)
/// </summary>
private async Task<HttpResponseMessage> GetHttpResponseMessage(
HttpMethod httpMethod,
string requestUrl,
Dictionary<string, string> requestParameters,
string refererValue,
string cookie)
{
string fullUrl = requestUrl;
if (requestParameters != null && requestParameters.Count > 0)
{
fullUrl = QueryHelpers.AddQueryString(
requestUrl,
requestParameters.ToDictionary(
kv => kv.Key,
kv => new StringValues(kv.Value)
)
);
}
var httpClient = _clientFactory.CreateClient(DouyinRequestParamManager.DY_HTTP_CLIENT);
for (var attempt = 0; ; attempt++)
{
using var requestMessage = new HttpRequestMessage(httpMethod, fullUrl);
if (!string.IsNullOrEmpty(refererValue) && Uri.IsWellFormedUriString(refererValue, UriKind.Absolute))
requestMessage.Headers.Referrer = new Uri(refererValue);
if (!string.IsNullOrEmpty(cookie))
requestMessage.Headers.TryAddWithoutValidation("Cookie", cookie);
try
{
var response = await httpClient.SendAsync(requestMessage);
if (!IsTransientStatus(response.StatusCode) || attempt >= TransientRequestRetryCount)
return response;
var status = (int)response.StatusCode;
response.Dispose();
await DelayTransientRetryAsync(attempt + 1, CancellationToken.None);
Log.Warning("抖音接口暂时返回 HTTP {Status},正在进行第 {Retry}/{MaxRetry} 次有限重试",
status, attempt + 1, TransientRequestRetryCount);
}
catch (Exception ex) when (IsTransientRequestFailure(ex) && attempt < TransientRequestRetryCount)
{
Log.Warning("抖音接口连接失败,正在进行第 {Retry}/{MaxRetry} 次有限重试:{Reason}",
attempt + 1, TransientRequestRetryCount, ex.GetBaseException().Message);
await DelayTransientRetryAsync(attempt + 1, CancellationToken.None);
}
}
}
private static bool IsTransientStatus(HttpStatusCode status) =>
status == HttpStatusCode.RequestTimeout || (int)status >= 500;
private static bool IsTransientRequestFailure(Exception error) =>
error is HttpRequestException or TimeoutException or TaskCanceledException;
private Task DelayTransientRetryAsync(int retryNumber, CancellationToken cancellationToken)
{
var exponential = retryNumber <= 1 ? 1 : 3;
var jitter = Random.Shared.Next(0, 401);
return _delayAsync(TimeSpan.FromMilliseconds(exponential * 1000 + jitter), cancellationToken);
}
#region 收藏夹相关方法(保留原有逻辑,仅优化资源释放)
public async Task<DouyinVideoInfoResponse> SyncCollectVideos(string cursor, string count, string cookie)
{
if (string.IsNullOrWhiteSpace(cursor))
throw new ArgumentException($"“{nameof(cursor)}”不能为 null 或空。", nameof(cursor));
if (string.IsNullOrWhiteSpace(count))
throw new ArgumentException($"“{nameof(count)}”不能为 null 或空。", nameof(count));
if (string.IsNullOrWhiteSpace(cookie))
throw new ArgumentException($"“{nameof(cookie)}”不能为 null 或空。", nameof(cookie));
try
{
var requestUrl = "/aweme/v1/web/aweme/listcollection";
var refererValue = "https://www.douyin.com";
var requestParameters = DouyinRequestParamManager.DouyinCollectParams;
requestParameters["cursor"] = cursor;
requestParameters["count"] = count;
using var response = await GetHttpResponseMessage(HttpMethod.Post, requestUrl, requestParameters, refererValue, cookie);
if (response.IsSuccessStatusCode)
{
using var stream = await response.Content.ReadAsStreamAsync();
using var reader = new StreamReader(stream);
using var jsonReader = new JsonTextReader(reader);
var model = JsonSerializer.Create(_jsonSettings).Deserialize<DouyinVideoInfoResponse>(jsonReader);
if (model == null)
Log.Error($"SyncCollectVideos fail, data is null");
return model;
}
else
{
Log.Error($"SyncCollectVideos fail: {response.StatusCode}");
return null;
}
}
catch (Exception ex)
{
Log.Error($"SyncCollectVideos error: {ex.Message}", ex);
return null;
}
}
public async Task<DouyinCollectListResponse> SyncCollectFolderList(string cookie, string cursor)
{
if (string.IsNullOrWhiteSpace(cookie))
{
Log.Error("cookie为空,无法获取收藏夹列表 ");
return null;
}
try
{
var requestUrl = "/aweme/v1/web/collects/list";
var refererValue = "https://www.douyin.com/user/self?from_tab_name=main&showSubTab=favorite_folder&showTab=favorite_collection";
var requestParameters = DouyinRequestParamManager.DouyinCollectListParams;
requestParameters["count"] = "10";
requestParameters["cursor"] = cursor;
using var response = await GetHttpResponseMessage(HttpMethod.Get, requestUrl, requestParameters, refererValue, cookie);
if (response.IsSuccessStatusCode)
{
using var stream = await response.Content.ReadAsStreamAsync();
using var reader = new StreamReader(stream);
using var jsonReader = new JsonTextReader(reader);
var model = JsonSerializer.Create(_jsonSettings).Deserialize<DouyinCollectListResponse>(jsonReader);
if (model == null)
Log.Error($"SyncCollectFolderList fail, data is null");
return model;
}
else
{
Log.Error("SyncCollectFolderList ,{StatusCode}", response.StatusCode);
return null;
}
}
catch (Exception ex)
{
Log.Error("SyncCollectFolderList ,{error}", ex);
return null;
}
}
public async Task<DouyinVideoInfoResponse> SyncCollectVideosByCollectId(string cursor, string count, string cookie, string collectsId)
{
if (string.IsNullOrWhiteSpace(collectsId))
throw new ArgumentException($"“{nameof(collectsId)}”不能为 null 或空。", nameof(collectsId));
if (string.IsNullOrWhiteSpace(cursor))
throw new ArgumentException($"“{nameof(cursor)}”不能为 null 或空。", nameof(cursor));
if (string.IsNullOrWhiteSpace(count))
throw new ArgumentException($"“{nameof(count)}”不能为 null 或空。", nameof(count));
if (string.IsNullOrWhiteSpace(cookie))
throw new ArgumentException($"“{nameof(cookie)}”不能为 null 或空。", nameof(cookie));
try
{
var requestUrl = "/aweme/v1/web/collects/video/list";
var refererValue = "https://www.douyin.com/user/self?from_tab_name=main&showSubTab=favorite_folder&showTab=favorite_collection";
var requestParameters = DouyinRequestParamManager.DouyinFolderCollectParams;
requestParameters["cursor"] = cursor;
requestParameters["count"] = "15";
requestParameters["collects_id"] = collectsId;
using var response = await GetHttpResponseMessage(HttpMethod.Get, requestUrl, requestParameters, refererValue, cookie);
if (response.IsSuccessStatusCode)
{
using var stream = await response.Content.ReadAsStreamAsync();
using var reader = new StreamReader(stream);
using var jsonReader = new JsonTextReader(reader);
var model = JsonSerializer.Create(_jsonSettings).Deserialize<DouyinVideoInfoResponse>(jsonReader);
if (model == null)
Log.Error($"SyncCollectVideosByCollectId fail: data is null");
return model;
}
else
{
Log.Error($"SyncCollectVideosByCollectId fail: {response.StatusCode}");
return null;
}
}
catch (Exception ex)
{
Log.Error($"SyncCollectVideosByCollectId error: {ex.Message}", ex);
return null;
}
}
#endregion
#region 合集相关方法
public async Task<DouyinMixListResponse> SyncMixList(string cookie, string cursor)
{
if (string.IsNullOrWhiteSpace(cookie))
{
Log.Error("cookie为空,无法获取收藏夹列表 ");
return null;
}
try
{
var requestUrl = "/aweme/v1/web/mix/listcollection";
var refererValue = "https://www.douyin.com/user/self?";
var requestParameters = DouyinRequestParamManager.DouyinMixListParams;
requestParameters["count"] = "10";
requestParameters["cursor"] = cursor;
using var response = await GetHttpResponseMessage(HttpMethod.Get, requestUrl, requestParameters, refererValue, cookie);
if (response.IsSuccessStatusCode)
{
using var stream = await response.Content.ReadAsStreamAsync();
using var reader = new StreamReader(stream);
using var jsonReader = new JsonTextReader(reader);
var model = JsonSerializer.Create(_jsonSettings).Deserialize<DouyinMixListResponse>(jsonReader);
if (model == null)
Log.Error($"SyncMixList fail, data is null");
return model;
}
else
{
Log.Error($"SyncMixList : {response.StatusCode}");
return null;
}
}
catch (Exception ex)
{
Log.Error("SyncMixList ,{error}", ex);
return null;
}
}
public async Task<DouyinVideoInfoResponse> SyncMixViedosByMixId(string cursor, string count, string cookie, string mixId)
{
if (string.IsNullOrWhiteSpace(mixId))
throw new ArgumentException($"“{nameof(mixId)}”不能为 null 或空。", nameof(mixId));
if (string.IsNullOrWhiteSpace(cursor))
throw new ArgumentException($"“{nameof(cursor)}”不能为 null 或空。", nameof(cursor));
if (string.IsNullOrWhiteSpace(count))
throw new ArgumentException($"“{nameof(count)}”不能为 null 或空。", nameof(count));
if (string.IsNullOrWhiteSpace(cookie))
throw new ArgumentException($"“{nameof(cookie)}”不能为 null 或空。", nameof(cookie));
try
{
var requestUrl = "/aweme/v1/web/mix/aweme";
var refererValue = "https://www.douyin.com/user/self?";
var requestParameters = DouyinRequestParamManager.DouyinMixVideoParams;
requestParameters["cursor"] = cursor;
requestParameters["count"] = "15";
requestParameters["mix_id"] = mixId;
using var response = await GetHttpResponseMessage(HttpMethod.Get, requestUrl, requestParameters, refererValue, cookie);
if (response.IsSuccessStatusCode)
{
using var stream = await response.Content.ReadAsStreamAsync();
using var reader = new StreamReader(stream);
using var jsonReader = new JsonTextReader(reader);
var model = JsonSerializer.Create(_jsonSettings).Deserialize<DouyinVideoInfoResponse>(jsonReader);
if (model == null)
Log.Error($"SyncMixViedosByMixId fail:data is null");
return model;
}
else
{
Log.Error($"SyncMixViedosByMixId fail: {response.StatusCode}");
return null;
}
}
catch (Exception ex)
{
Log.Error($"SyncMixViedosByMixId error: {ex.Message}", ex);
return null;
}
}
#endregion
#region 短剧相关方法
public async Task<DouyinSeriesListResponse> SyncSeriesList(string cookie, string cursor)
{
if (string.IsNullOrWhiteSpace(cookie))
{
Log.Error("cookie为空,无法获取收藏夹列表 ");
return null;
}
try
{
var requestUrl = "/aweme/v1/web/series/collections";
var refererValue = "https://www.douyin.com/user/self?";
var requestParameters = DouyinRequestParamManager.DouyinSeriesListParams;
requestParameters["count"] = "15";
requestParameters["cursor"] = cursor;
using var response = await GetHttpResponseMessage(HttpMethod.Get, requestUrl, requestParameters, refererValue, cookie);
if (response.IsSuccessStatusCode)
{
using var stream = await response.Content.ReadAsStreamAsync();
using var reader = new StreamReader(stream);
using var jsonReader = new JsonTextReader(reader);
var model = JsonSerializer.Create(_jsonSettings).Deserialize<DouyinSeriesListResponse>(jsonReader);
if (model == null)
Log.Error($"SyncShortList fail, data is null");
return model;
}
else
{
Log.Error($"SyncShortList fail: {response.StatusCode}");
return null;
}
}
catch (Exception ex)
{
Log.Error("SyncShortList ,{error}", ex);
return null;
}
}
public async Task<DouyinVideoInfoResponse> SyncSeriesViedosByMSeriesId(string cursor, string count, string cookie, string seriesId)
{
if (string.IsNullOrWhiteSpace(seriesId))
throw new ArgumentException($"“{nameof(seriesId)}”不能为 null 或空。", nameof(seriesId));
if (string.IsNullOrWhiteSpace(cursor))
throw new ArgumentException($"“{nameof(cursor)}”不能为 null 或空。", nameof(cursor));
if (string.IsNullOrWhiteSpace(count))
throw new ArgumentException($"“{nameof(count)}”不能为 null 或空。", nameof(count));
if (string.IsNullOrWhiteSpace(cookie))
throw new ArgumentException($"“{nameof(cookie)}”不能为 null 或空。", nameof(cookie));
try
{
var requestUrl = "/aweme/v1/web/series/aweme";
var refererValue = "https://www.douyin.com/user/self?";
var requestParameters = DouyinRequestParamManager.DouyinSeriesVideosParams;
requestParameters["cursor"] = cursor;
requestParameters["count"] = count;
requestParameters["series_id"] = seriesId;
using var response = await GetHttpResponseMessage(HttpMethod.Get, requestUrl, requestParameters, refererValue, cookie);
if (response.IsSuccessStatusCode)
{
using var stream = await response.Content.ReadAsStreamAsync();
using var reader = new StreamReader(stream);
using var jsonReader = new JsonTextReader(reader);
var model = JsonSerializer.Create(_jsonSettings).Deserialize<DouyinVideoInfoResponse>(jsonReader);
if (model == null)
Log.Error($"SyncSeriesViedosByMSeriesId fail:data is null");
return model;
}
else
{
Log.Error($"SyncSeriesViedosByMSeriesId fail: {response.StatusCode}");
return null;
}
}
catch (Exception ex)
{
Log.Error($"SyncSeriesViedosByMSeriesId error: {ex.Message}", ex);
return null;
}
}
#endregion
#region 喜欢/博主/关注相关方法
public async Task<DouyinVideoInfoResponse> SyncFavoriteVideos(string count, string cursor, string secUserId, string cookie)
{
if (string.IsNullOrWhiteSpace(cursor))
throw new ArgumentException($"“{nameof(cursor)}”不能为 null 或空。", nameof(cursor));
if (string.IsNullOrWhiteSpace(count))
throw new ArgumentException($"“{nameof(count)}”不能为 null 或空。", nameof(count));
if (string.IsNullOrWhiteSpace(secUserId))
throw new ArgumentException($"“{nameof(secUserId)}”不能为 null 或空。", nameof(secUserId));
if (string.IsNullOrWhiteSpace(cookie))
throw new ArgumentException($"“{nameof(cookie)}”不能为 null 或空。", nameof(cookie));
try
{
var requestUrl = "/aweme/v1/web/aweme/favorite";
var refererValue = "https://www.douyin.com/user/self?showTab=like";
var requestParameters = DouyinRequestParamManager.DouyinFavoriteParams;
requestParameters["max_cursor"] = cursor;
requestParameters["sec_user_id"] = secUserId;
requestParameters["count"] = count;
using var response = await GetHttpResponseMessage(HttpMethod.Get, requestUrl, requestParameters, refererValue, cookie);
if (response.IsSuccessStatusCode)
{
using var stream = await response.Content.ReadAsStreamAsync();
using var reader = new StreamReader(stream);
using var jsonReader = new JsonTextReader(reader);
var model = JsonSerializer.Create(_jsonSettings).Deserialize<DouyinVideoInfoResponse>(jsonReader);
if (model == null)
Log.Error($"SyncFavoriteVideos fail, data is null");
return model;
}
else
{
Log.Error($"SyncFavoriteVideos fail: {response.StatusCode}");
return null;
}
}
catch (Exception ex)
{
Log.Error($"SyncFavoriteVideos error: {ex.Message}", ex);
return null;
}
}
public async Task<DouyinVideoInfoResponse> SyncUpderPostVideos(string count, string cursor, string secUserId, string cookie)
{
if (string.IsNullOrWhiteSpace(cursor))
throw new ArgumentException($"“{nameof(cursor)}”不能为 null 或空。", nameof(cursor));
if (string.IsNullOrWhiteSpace(count))
throw new ArgumentException($"“{nameof(count)}”不能为 null 或空。", nameof(count));
if (string.IsNullOrWhiteSpace(secUserId))
throw new ArgumentException($"“{nameof(secUserId)}”不能为 null 或空。", nameof(secUserId));
if (string.IsNullOrWhiteSpace(cookie))
throw new ArgumentException($"“{nameof(cookie)}”不能为 null 或空。", nameof(cookie));
try
{
var requestUrl = "/aweme/v1/web/aweme/post";
var refererValue = "https://www.douyin.com/user/";
var requestParameters = DouyinRequestParamManager.DouyinUpderPostParams;
requestParameters["max_cursor"] = cursor;
requestParameters["sec_user_id"] = secUserId;
requestParameters["count"] = count;
using var response = await GetHttpResponseMessage(HttpMethod.Get, requestUrl, requestParameters, refererValue, cookie);
if (response.IsSuccessStatusCode)
{
using var stream = await response.Content.ReadAsStreamAsync();
using var reader = new StreamReader(stream);
using var jsonReader = new JsonTextReader(reader);
var model = JsonSerializer.Create(_jsonSettings).Deserialize<DouyinVideoInfoResponse>(jsonReader);
if (model == null)
throw new InvalidDataException("获取博主作品列表失败:响应内容为空或无法解析");
return model;
}
else
{
throw new HttpRequestException(
$"获取博主作品列表失败:HTTP {(int)response.StatusCode} {response.StatusCode}",
null,
response.StatusCode);
}
}
catch (Exception ex)
{
Log.Error($"SyncUpderPostVideos error: {ex.Message}", ex);
throw;
}
}
public async Task<DouyinFollowInfoResponse> SyncMyFollows(string count, string offset, string secUserId, string cookie)
{
if (string.IsNullOrWhiteSpace(offset))
throw new ArgumentException($"“{nameof(offset)}”不能为 null 或空。", nameof(offset));
if (string.IsNullOrWhiteSpace(count))
throw new ArgumentException($"“{nameof(count)}”不能为 null 或空。", nameof(count));
if (string.IsNullOrWhiteSpace(secUserId))
throw new ArgumentException($"“{nameof(secUserId)}”不能为 null 或空。", nameof(secUserId));
if (string.IsNullOrWhiteSpace(cookie))
throw new ArgumentException($"“{nameof(cookie)}”不能为 null 或空。", nameof(cookie));
try
{
var requestUrl = "/aweme/v1/web/user/following/list";
var refererValue = "https://www.douyin.com/user/self?showTab=like";
var requestParameters = DouyinRequestParamManager.DouyinMyFollowParams;
requestParameters["sec_user_id"] = secUserId;
requestParameters["count"] = count;
requestParameters["offset"] = offset;
using var response = await GetHttpResponseMessage(HttpMethod.Get, requestUrl, requestParameters, refererValue, cookie);
if (response.IsSuccessStatusCode)
{
using var stream = await response.Content.ReadAsStreamAsync();
using var reader = new StreamReader(stream);
using var jsonReader = new JsonTextReader(reader);
var model = JsonSerializer.Create(_jsonSettings).Deserialize<DouyinFollowInfoResponse>(jsonReader);
return model;
}
else
{
Log.Error($"SyncMyFollows fail: {response.StatusCode}");
return null;
}
}
catch (Exception ex)
{
Log.Error($"SyncMyFollows error: {ex.Message}", ex);
// 优化:抛出前确保资源释放,使用包装异常保留原始堆栈
throw new InvalidOperationException("获取关注列表失败", ex);
}
}
#endregion
#region Cookie检查
public async Task<bool> CheckCookie(DouyinCookie douyinCookie)
{
if (douyinCookie == null || string.IsNullOrWhiteSpace(douyinCookie.Cookies))
return false;
try
{
if (!string.IsNullOrWhiteSpace(douyinCookie.SecUserId))
{
var res = await SyncMyFollows("1", "10", douyinCookie.SecUserId, douyinCookie.Cookies);
return res != null && res.StatusCode == 0;
}
else
{
var res = await SyncCollectVideos("0", "10", douyinCookie.Cookies);
return res != null && res.StatusCode == 0;
}
}
catch (Exception ex)
{
Log.Error(ex, "检查Cookie有效性失败");
return false;
}
}
#endregion
#region 下载相关
public Task<MediaDownloadResult> DownloadAsync(
string videoUrl,
string savePath,
string cookie,
List<string> otherUrls = null,
CancellationToken cancellationToken = default,
TimeSpan? streamTimeout = null,
int maxRetryCount = 12,
TimeSpan? initialRetryDelay = null)
{
var urls = BuildCandidateUrls(videoUrl, otherUrls, maxRetryCount);
return DownloadCandidatesAsync(
urls,
savePath,
cookie,
cancellationToken,
streamTimeout ?? TimeSpan.FromSeconds(60),
initialRetryDelay ?? TimeSpan.FromSeconds(1),
async (response, actualPath, timeout, token) =>
{
await WriteLocalAsync(response, actualPath, timeout, token);
});
}
/// <summary>
/// 将抖音媒体写入指定存储。远端存储负责安全中转和最终提交,
/// 不会先落入永久本地媒体目录。
/// </summary>
public async Task<MediaDownloadResult> DownloadToStorageAsync(
StorageType storageType,
string mediaUrl,
string savePath,
string cookie,
List<string> otherUrls = null,
CancellationToken cancellationToken = default,
int maxRetryCount = 12,
IMediaStorage storageOverride = null)
{
if (storageType == StorageType.Local)
return await DownloadAsync(mediaUrl, savePath, cookie, otherUrls, cancellationToken, maxRetryCount: maxRetryCount);
var storage = storageOverride ?? _storageRouter.Resolve(storageType);
if (storage.StorageType != storageType)
throw new InvalidOperationException("指定的媒体存储与目标存储类型不一致");
if (storage is ICanonicalMediaStorage canonicalStorage)
savePath = await canonicalStorage.CanonicalizePathAsync(savePath, false, cancellationToken);
var urls = BuildCandidateUrls(mediaUrl, otherUrls, maxRetryCount);
return await DownloadCandidatesAsync(
urls,
savePath,
cookie,
cancellationToken,
TimeSpan.FromSeconds(60),
TimeSpan.FromSeconds(1),
async (response, actualPath, _, token) =>
{
try
{
await using var responseStream = await response.Content.ReadAsStreamAsync(token);
await storage.WriteAsync(
actualPath,
responseStream,
response.Content.Headers.ContentLength,
response.Content.Headers.ContentType?.MediaType,
token);
}
catch (OperationCanceledException) when (token.IsCancellationRequested) { throw; }
catch (MediaIntegrityException) { throw; }
catch (Exception ex)
{
throw new MediaStorageException($"{storageType} 媒体写入失败:{actualPath}", ex);
}
});
}
private async Task<MediaDownloadResult> DownloadCandidatesAsync(
IReadOnlyList<Uri> urls,
string savePath,
string cookie,
CancellationToken cancellationToken,
TimeSpan streamTimeout,
TimeSpan retryDelay,
Func<HttpResponseMessage, string, TimeSpan, CancellationToken, Task> writeAsync)
{
if (urls.Count == 0)
return Failure(MediaDownloadFailureKind.SourceUnavailable, savePath, null, null, 0, null, "没有有效的媒体下载地址。");
var client = _clientFactory.CreateClient(DouyinRequestParamManager.DY_HTTP_CLIENT_DOWN);
MediaDownloadResult lastFailure = null;
var forbiddenCount = 0;
var attempted = 0;
foreach (var source in urls)
{
attempted++;
var networkRetry = 0;
var rateLimitRetry = 0;
while (true)
{
HttpResponseMessage response = null;
Uri finalUri = source;
try
{
(response, finalUri) = await SendWithSafeRedirectsAsync(client, source, cookie, cancellationToken);
var status = (int)response.StatusCode;
var host = SafeHost(finalUri);
if (response.StatusCode == HttpStatusCode.Unauthorized)
return Failure(MediaDownloadFailureKind.SourceUnauthorized, savePath, status, host, attempted, null,
"抖音媒体请求返回 401,需要重新授权。");
if (response.StatusCode == HttpStatusCode.Forbidden)
{
forbiddenCount++;
lastFailure = Failure(MediaDownloadFailureKind.SourceForbidden, savePath, status, host, attempted, null,
"抖音媒体来源拒绝访问(403)。");
break;
}
if (response.StatusCode is HttpStatusCode.NotFound or HttpStatusCode.Gone)
{
lastFailure = Failure(MediaDownloadFailureKind.SourceNotFound, savePath, status, host, attempted, null,
$"媒体来源返回 {status},作品可能已下架或链接已失效。");
break;
}
if ((int)response.StatusCode == 429)
{
var retryAt = GetRetryAfter(response);
if (rateLimitRetry++ == 0)
{
var wait = retryAt.HasValue ? retryAt.Value - DateTime.UtcNow : TimeSpan.FromSeconds(5);
if (wait < TimeSpan.Zero) wait = TimeSpan.Zero;
if (wait > TimeSpan.FromSeconds(60)) wait = TimeSpan.FromSeconds(60);
await _delayAsync(wait, cancellationToken);
continue;
}
return Failure(MediaDownloadFailureKind.SourceRateLimited, savePath, status, host, attempted, retryAt,
"抖音媒体来源请求过于频繁(429),已进入冷却等待。");
}
if ((int)response.StatusCode >= 500)
{
if (networkRetry < MediaNetworkRetryCount)
{
networkRetry++;
await DelayMediaRetryAsync(retryDelay, networkRetry, cancellationToken);
continue;
}
lastFailure = Failure(MediaDownloadFailureKind.SourceUnavailable, savePath, status, host, attempted, null,
$"媒体来源暂时不可用({status})。");
break;
}
if (!response.IsSuccessStatusCode)
{
lastFailure = Failure(MediaDownloadFailureKind.SourceUnavailable, savePath, status, host, attempted, null,
$"媒体来源请求失败({status})。");
break;
}
var mediaType = response.Content.Headers.ContentType?.MediaType;
if (IsObviousNonMedia(mediaType))
{
lastFailure = Failure(MediaDownloadFailureKind.SourceInvalidContent, savePath, status, host, attempted, null,
$"媒体来源返回了非媒体内容({mediaType})。");
break;
}
if (response.Content.Headers.ContentLength == 0)
{
lastFailure = Failure(MediaDownloadFailureKind.IntegrityCheckFailed, savePath, status, host, attempted, null,
"媒体来源返回空内容。");
break;
}
var actualPath = DetectActualSavePath(savePath, mediaType);
await writeAsync(response, actualPath, streamTimeout, cancellationToken);
return MediaDownloadResult.Succeeded(actualPath, host, attempted);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
CleanupIncompleteFile(savePath);
return Failure(MediaDownloadFailureKind.Cancelled, savePath, null, SafeHost(finalUri), attempted, null, "媒体下载已取消。");
}
catch (MediaStorageException ex)
{
CleanupIncompleteFile(savePath);
Log.Error(ex, "媒体存储写入失败:{Path}", savePath);
return Failure(MediaDownloadFailureKind.StorageUnavailable, savePath, null, SafeHost(finalUri), attempted, null,
DescribeStorageFailure(ex));
}
catch (MediaIntegrityException ex)
{
CleanupIncompleteFile(savePath);
lastFailure = Failure(MediaDownloadFailureKind.IntegrityCheckFailed, savePath, null, SafeHost(finalUri), attempted, null, ex.Message);
break;
}
catch (MediaSourceException ex)
{
lastFailure = Failure(ex.FailureKind, savePath, ex.StatusCode, ex.SourceHost, attempted, ex.RetryAfter, ex.Message);
break;
}
catch (Exception ex) when (ex is HttpRequestException or TimeoutException or TaskCanceledException or IOException)
{
if (networkRetry < MediaNetworkRetryCount)
{
networkRetry++;
await DelayMediaRetryAsync(retryDelay, networkRetry, cancellationToken);
continue;
}
lastFailure = Failure(MediaDownloadFailureKind.SourceUnavailable, savePath, null, SafeHost(finalUri), attempted, null,
"连接媒体来源失败。");
break;
}
finally
{
response?.Dispose();
}
}
}
if (forbiddenCount == attempted && attempted > 0)
lastFailure = Failure(MediaDownloadFailureKind.SourceForbidden, savePath, 403, lastFailure?.SourceHost,
attempted, null, $"全部 {attempted} 个候选媒体地址均返回 403。");
else if (lastFailure?.FailureKind == MediaDownloadFailureKind.SourceForbidden)
lastFailure = Failure(MediaDownloadFailureKind.SourceUnavailable, savePath, null, lastFailure.SourceHost,
attempted, null, "候选媒体地址返回了多种来源错误。");
lastFailure ??= Failure(MediaDownloadFailureKind.SourceUnavailable, savePath, null, null, attempted, null, "所有候选媒体地址均不可用。");
if (lastFailure.FailureKind == MediaDownloadFailureKind.SourceForbidden)
Log.Warning("抖音媒体来源被拒绝:Host={SourceHost}, Status={Status}, Candidates={Candidates}",
lastFailure.SourceHost, lastFailure.HttpStatusCode, lastFailure.AttemptedUrlCount);
else
Log.Warning("媒体来源下载失败:Host={SourceHost}, Status={Status}, Kind={Kind}, Candidates={Candidates}",
lastFailure.SourceHost, lastFailure.HttpStatusCode, lastFailure.FailureKind, lastFailure.AttemptedUrlCount);
return lastFailure;
}
private Task DelayMediaRetryAsync(TimeSpan baseDelay, int retryNumber, CancellationToken cancellationToken)
{
var multiplier = Math.Pow(2, Math.Max(0, retryNumber - 1));
var milliseconds = Math.Min(5000, Math.Max(100, baseDelay.TotalMilliseconds * multiplier));
milliseconds += Random.Shared.Next(0, 251);
return _delayAsync(TimeSpan.FromMilliseconds(milliseconds), cancellationToken);
}
private static List<Uri> BuildCandidateUrls(string primary, IEnumerable<string> alternatives, int limit)
{
var result = new List<Uri>();
var seen = new HashSet<string>(StringComparer.Ordinal);
foreach (var value in new[] { primary }.Concat(alternatives ?? Enumerable.Empty<string>()))
{
if (result.Count >= Math.Clamp(limit, 1, 12)) break;
if (!Uri.TryCreate(value, UriKind.Absolute, out var uri) || (uri.Scheme != Uri.UriSchemeHttp && uri.Scheme != Uri.UriSchemeHttps)) continue;
if (seen.Add(uri.AbsoluteUri)) result.Add(uri);
}
return result;
}
internal static string DescribeStorageFailure(MediaStorageException error)
{
if (error == null) return "媒体存储写入失败。";
var message = error.Message?.Trim();
var reason = error.GetBaseException()?.Message?.Trim();
if (string.IsNullOrWhiteSpace(message)) message = "媒体存储写入失败。";
if (string.IsNullOrWhiteSpace(reason) || string.Equals(message, reason, StringComparison.Ordinal))
return message;
return $"{message};原因:{reason}";
}
private static async Task<(HttpResponseMessage Response, Uri FinalUri)> SendWithSafeRedirectsAsync(
HttpClient client,
Uri initialUri,
string cookie,
CancellationToken cancellationToken)
{
var current = initialUri;
for (var redirect = 0; redirect <= 5; redirect++)
{
using var request = new HttpRequestMessage(HttpMethod.Get, current);
if (!string.IsNullOrWhiteSpace(cookie) && IsDouyinHost(current.Host))
request.Headers.TryAddWithoutValidation("Cookie", cookie);
var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
if (!IsRedirect(response.StatusCode)) return (response, current);
var location = response.Headers.Location;
response.Dispose();
if (location == null)
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable, "媒体来源返回了无目标地址的跳转。",
(int)response.StatusCode, SafeHost(current));
if (redirect == 5)
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable, "媒体来源跳转次数超过 5 次。",
(int)response.StatusCode, SafeHost(current));
current = location.IsAbsoluteUri ? location : new Uri(current, location);
if (current.Scheme != Uri.UriSchemeHttp && current.Scheme != Uri.UriSchemeHttps)
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable, "媒体来源返回了不安全的跳转协议。",
sourceHost: SafeHost(current));
}
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable, "媒体来源跳转失败。", sourceHost: SafeHost(current));
}
private static async Task WriteLocalAsync(
HttpResponseMessage response,
string actualSavePath,
TimeSpan streamTimeout,
CancellationToken cancellationToken)
{
var directory = Path.GetDirectoryName(actualSavePath);
FileStream fileStream = null;
try
{
try
{
if (!string.IsNullOrWhiteSpace(directory)) Directory.CreateDirectory(directory);
CleanupIncompleteFile(actualSavePath);
fileStream = new FileStream(actualSavePath, FileMode.CreateNew, FileAccess.Write, FileShare.None,
81920, FileOptions.Asynchronous | FileOptions.SequentialScan);
}
catch (Exception ex) when (ex is UnauthorizedAccessException or IOException)
{
throw new MediaStorageException($"本地媒体写入失败:{actualSavePath}", ex);
}
Stream responseStream;
try { responseStream = await response.Content.ReadAsStreamAsync(cancellationToken); }
catch (Exception ex) when (ex is HttpRequestException or IOException)
{
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable, "读取媒体来源响应失败。",
sourceHost: SafeHost(response.RequestMessage?.RequestUri), innerException: ex);
}
await using var ownedResponseStream = responseStream;
var buffer = System.Buffers.ArrayPool<byte>.Shared.Rent(81920);
long totalRead = 0;
var lastActivity = DateTime.UtcNow;
try
{
int read;
while (true)
{
try { read = await responseStream.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken); }
catch (Exception ex) when (ex is HttpRequestException or IOException)
{
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable, "读取媒体来源响应失败。",
sourceHost: SafeHost(response.RequestMessage?.RequestUri), innerException: ex);
}
if (read <= 0) break;
if (DateTime.UtcNow - lastActivity > streamTimeout)
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable,
$"媒体流超过 {streamTimeout.TotalSeconds:0} 秒没有数据。");
try { await fileStream.WriteAsync(buffer.AsMemory(0, read), cancellationToken); }
catch (Exception ex) when (ex is UnauthorizedAccessException or IOException)
{
throw new MediaStorageException($"本地媒体写入失败:{actualSavePath}", ex);
}
totalRead += read;
lastActivity = DateTime.UtcNow;
}
try { await fileStream.FlushAsync(cancellationToken); }
catch (Exception ex) when (ex is UnauthorizedAccessException or IOException)
{
throw new MediaStorageException($"本地媒体写入失败:{actualSavePath}", ex);
}
}
finally
{
System.Buffers.ArrayPool<byte>.Shared.Return(buffer);
}
if (totalRead <= 0) throw new MediaIntegrityException("下载内容为空。");
var expected = response.Content.Headers.ContentLength;
if (expected.HasValue && expected.Value != totalRead)
throw new MediaIntegrityException($"下载长度不一致:期望 {expected.Value},实际 {totalRead}。");
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
catch (MediaSourceException) { CleanupIncompleteFile(actualSavePath); throw; }
catch (MediaIntegrityException) { CleanupIncompleteFile(actualSavePath); throw; }
catch (MediaStorageException) { CleanupIncompleteFile(actualSavePath); throw; }
finally { if (fileStream != null) await fileStream.DisposeAsync(); }
}
private static string DetectActualSavePath(string savePath, string mediaType)
{
if (!Path.GetExtension(savePath).Equals(".mp3", StringComparison.OrdinalIgnoreCase)) return savePath;
if (mediaType?.Contains("audio/mp4", StringComparison.OrdinalIgnoreCase) == true
|| mediaType?.Contains("audio/m4a", StringComparison.OrdinalIgnoreCase) == true)
return Path.ChangeExtension(savePath, ".m4a");
if (mediaType?.Contains("video/mp4", StringComparison.OrdinalIgnoreCase) == true)
return Path.ChangeExtension(savePath, ".mp4");
return savePath;
}
private static bool IsObviousNonMedia(string mediaType) =>
!string.IsNullOrWhiteSpace(mediaType)
&& (mediaType.StartsWith("text/html", StringComparison.OrdinalIgnoreCase)
|| mediaType.Contains("json", StringComparison.OrdinalIgnoreCase));
private static bool IsDouyinHost(string host) =>
string.Equals(host, "douyin.com", StringComparison.OrdinalIgnoreCase)
|| host.EndsWith(".douyin.com", StringComparison.OrdinalIgnoreCase);
private static bool IsRedirect(HttpStatusCode code) => code is
HttpStatusCode.MovedPermanently or HttpStatusCode.Redirect or HttpStatusCode.RedirectMethod
or HttpStatusCode.TemporaryRedirect or HttpStatusCode.PermanentRedirect;
private static string SafeHost(Uri uri) => uri?.IsAbsoluteUri == true ? uri.IdnHost : null;
private static DateTime? GetRetryAfter(HttpResponseMessage response)
{
var value = response.Headers.RetryAfter;
if (value?.Date.HasValue == true) return value.Date.Value.UtcDateTime;
if (value?.Delta.HasValue == true) return DateTime.UtcNow.Add(value.Delta.Value);
return DateTime.UtcNow.AddMinutes(15);
}
private static MediaDownloadResult Failure(
MediaDownloadFailureKind kind,
string savePath,
int? status,
string host,
int attempted,
DateTime? retryAfter,
string message) => new()
{
Success = false,
ActualSavePath = savePath,
FailureKind = kind,
HttpStatusCode = status,
SourceHost = host,
AttemptedUrlCount = attempted,
RetryAfter = retryAfter,
Message = message
};
private static void CleanupIncompleteFile(string savePath)
{
if (File.Exists(savePath))
{
try
{
File.Delete(savePath);
}
catch (IOException ex)
{
Log.Error(ex, $"清理无效文件失败:{savePath}(可能被占用)");
}
}
}
#endregion
#region IDisposable 实现(优化版)
protected virtual void Dispose(bool disposing)
{
if (!_disposedValue)
{
if (disposing)
{
// 释放托管资源:仅清理本类创建的托管资源(此处无)
}
// 释放非托管资源:本类无非托管资源
// 优化:移除强制GC调用,让GC自动管理
_disposedValue = true;
}
}
public void Dispose()
{
Dispose(disposing: true);
GC.SuppressFinalize(this);
}
#endregion
}
}