feat: add fnOS packaging, storage workflows and release pipeline
This commit is contained in:
+457
-162
@@ -11,12 +11,17 @@ using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using dy.net.storage;
|
||||
|
||||
namespace dy.net.service
|
||||
{
|
||||
public class DouyinHttpClientService : IDisposable
|
||||
{
|
||||
private readonly IHttpClientFactory _clientFactory;
|
||||
private readonly MediaStorageRouter _storageRouter;
|
||||
private readonly Func<TimeSpan, CancellationToken, Task> _delayAsync;
|
||||
private const int TransientRequestRetryCount = 2;
|
||||
private const int MediaNetworkRetryCount = 2;
|
||||
private readonly JsonSerializerSettings _jsonSettings=new JsonSerializerSettings
|
||||
{
|
||||
NullValueHandling = NullValueHandling.Ignore,
|
||||
@@ -24,9 +29,19 @@ namespace dy.net.service
|
||||
};
|
||||
private bool _disposedValue;
|
||||
|
||||
public DouyinHttpClientService(IHttpClientFactory clientFactory)
|
||||
public DouyinHttpClientService(IHttpClientFactory clientFactory, MediaStorageRouter storageRouter)
|
||||
: this(clientFactory, storageRouter, Task.Delay)
|
||||
{
|
||||
}
|
||||
|
||||
internal DouyinHttpClientService(
|
||||
IHttpClientFactory clientFactory,
|
||||
MediaStorageRouter storageRouter,
|
||||
Func<TimeSpan, CancellationToken, Task> delayAsync)
|
||||
{
|
||||
_clientFactory = clientFactory;
|
||||
_storageRouter = storageRouter;
|
||||
_delayAsync = delayAsync ?? Task.Delay;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -51,21 +66,46 @@ namespace dy.net.service
|
||||
);
|
||||
}
|
||||
|
||||
using var requestMessage = new HttpRequestMessage(httpMethod, fullUrl);
|
||||
|
||||
if (!string.IsNullOrEmpty(refererValue) && Uri.IsWellFormedUriString(refererValue, UriKind.Absolute))
|
||||
{
|
||||
requestMessage.Headers.Referrer = new Uri(refererValue);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(cookie))
|
||||
{
|
||||
requestMessage.Headers.TryAddWithoutValidation("Cookie", cookie);
|
||||
}
|
||||
|
||||
// 优化：移除using，由IHttpClientFactory管理生命周期
|
||||
var httpClient = _clientFactory.CreateClient(DouyinRequestParamManager.DY_HTTP_CLIENT);
|
||||
return await httpClient.SendAsync(requestMessage);
|
||||
for (var attempt = 0; ; attempt++)
|
||||
{
|
||||
using var requestMessage = new HttpRequestMessage(httpMethod, fullUrl);
|
||||
if (!string.IsNullOrEmpty(refererValue) && Uri.IsWellFormedUriString(refererValue, UriKind.Absolute))
|
||||
requestMessage.Headers.Referrer = new Uri(refererValue);
|
||||
if (!string.IsNullOrEmpty(cookie))
|
||||
requestMessage.Headers.TryAddWithoutValidation("Cookie", cookie);
|
||||
|
||||
try
|
||||
{
|
||||
var response = await httpClient.SendAsync(requestMessage);
|
||||
if (!IsTransientStatus(response.StatusCode) || attempt >= TransientRequestRetryCount)
|
||||
return response;
|
||||
var status = (int)response.StatusCode;
|
||||
response.Dispose();
|
||||
await DelayTransientRetryAsync(attempt + 1, CancellationToken.None);
|
||||
Log.Warning("抖音接口暂时返回 HTTP {Status}，正在进行第 {Retry}/{MaxRetry} 次有限重试",
|
||||
status, attempt + 1, TransientRequestRetryCount);
|
||||
}
|
||||
catch (Exception ex) when (IsTransientRequestFailure(ex) && attempt < TransientRequestRetryCount)
|
||||
{
|
||||
Log.Warning("抖音接口连接失败，正在进行第 {Retry}/{MaxRetry} 次有限重试：{Reason}",
|
||||
attempt + 1, TransientRequestRetryCount, ex.GetBaseException().Message);
|
||||
await DelayTransientRetryAsync(attempt + 1, CancellationToken.None);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsTransientStatus(HttpStatusCode status) =>
|
||||
status == HttpStatusCode.RequestTimeout || (int)status >= 500;
|
||||
|
||||
private static bool IsTransientRequestFailure(Exception error) =>
|
||||
error is HttpRequestException or TimeoutException or TaskCanceledException;
|
||||
|
||||
private Task DelayTransientRetryAsync(int retryNumber, CancellationToken cancellationToken)
|
||||
{
|
||||
var exponential = retryNumber <= 1 ? 1 : 3;
|
||||
var jitter = Random.Shared.Next(0, 401);
|
||||
return _delayAsync(TimeSpan.FromMilliseconds(exponential * 1000 + jitter), cancellationToken);
|
||||
}
|
||||
|
||||
#region 收藏夹相关方法（保留原有逻辑，仅优化资源释放）
|
||||
@@ -450,19 +490,21 @@ namespace dy.net.service
|
||||
using var jsonReader = new JsonTextReader(reader);
|
||||
var model = JsonSerializer.Create(_jsonSettings).Deserialize<DouyinVideoInfoResponse>(jsonReader);
|
||||
if (model == null)
|
||||
Log.Error($"SyncUpderPostVideos fail, data is null");
|
||||
throw new InvalidDataException("获取博主作品列表失败：响应内容为空或无法解析");
|
||||
return model;
|
||||
}
|
||||
else
|
||||
{
|
||||
Log.Error($"SyncUpderPostVideos StatusCode fail: {response.StatusCode}");
|
||||
return null;
|
||||
throw new HttpRequestException(
|
||||
$"获取博主作品列表失败：HTTP {(int)response.StatusCode} {response.StatusCode}",
|
||||
null,
|
||||
response.StatusCode);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Error($"SyncUpderPostVideos error: {ex.Message}", ex);
|
||||
return null;
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -538,185 +580,438 @@ namespace dy.net.service
|
||||
}
|
||||
#endregion
|
||||
|
||||
#region 下载相关（优化内存管理）
|
||||
public async Task<(bool Success, string ActualSavePath)> DownloadAsync(
|
||||
#region 下载相关
|
||||
public Task<MediaDownloadResult> DownloadAsync(
|
||||
string videoUrl,
|
||||
string savePath,
|
||||
string cookie,
|
||||
List<string> otherUrls = null,
|
||||
CancellationToken cancellationToken = default,
|
||||
TimeSpan? streamTimeout = null,
|
||||
int maxRetryCount = 3,
|
||||
int maxRetryCount = 12,
|
||||
TimeSpan? initialRetryDelay = null)
|
||||
{
|
||||
// 优化1：隔离外部列表，避免引用泄漏
|
||||
var retryUrls = new List<string> { videoUrl };
|
||||
if (otherUrls != null && otherUrls.Any())
|
||||
{
|
||||
retryUrls.AddRange(otherUrls);
|
||||
maxRetryCount = Math.Min(maxRetryCount, retryUrls.Count); // 防止溢出
|
||||
}
|
||||
|
||||
int retryCount = 0;
|
||||
var retryDelay = initialRetryDelay ?? TimeSpan.FromSeconds(1);
|
||||
streamTimeout ??= TimeSpan.FromSeconds(60);
|
||||
|
||||
while (retryCount < maxRetryCount)
|
||||
{
|
||||
try
|
||||
var urls = BuildCandidateUrls(videoUrl, otherUrls, maxRetryCount);
|
||||
return DownloadCandidatesAsync(
|
||||
urls,
|
||||
savePath,
|
||||
cookie,
|
||||
cancellationToken,
|
||||
streamTimeout ?? TimeSpan.FromSeconds(60),
|
||||
initialRetryDelay ?? TimeSpan.FromSeconds(1),
|
||||
async (response, actualPath, timeout, token) =>
|
||||
{
|
||||
string currentUrl = retryUrls[retryCount];
|
||||
return await TryDownloadOnceAsync(currentUrl, savePath, cookie, cancellationToken, streamTimeout.Value);
|
||||
}
|
||||
catch (Exception ex) when (IsRetryableException(ex) && retryCount < maxRetryCount - 1)
|
||||
{
|
||||
retryCount++;
|
||||
var delay = TimeSpan.FromMilliseconds(retryDelay.TotalMilliseconds * Math.Pow(2, retryCount - 1));
|
||||
Log.Warning(ex, $"下载失败（第{retryCount}/{maxRetryCount}次重试）：{videoUrl}，将在{delay.TotalSeconds:F1}秒后重试");
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
Log.Information($"重试等待被取消：{videoUrl}");
|
||||
return (false, savePath);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
Log.Information($"下载被取消：{videoUrl}");
|
||||
return (false, savePath);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Error(ex, $"下载失败（不可重试）：{videoUrl}");
|
||||
CleanupIncompleteFile(savePath);
|
||||
return (false, savePath);
|
||||
}
|
||||
}
|
||||
|
||||
Log.Error($"下载失败：已达最大重试次数 {maxRetryCount}，URL={videoUrl}");
|
||||
return (false, savePath);
|
||||
await WriteLocalAsync(response, actualPath, timeout, token);
|
||||
});
|
||||
}
|
||||
|
||||
private async Task<(bool Success, string ActualSavePath)> TryDownloadOnceAsync(
|
||||
string videoUrl,
|
||||
/// <summary>
|
||||
/// 将抖音媒体写入指定存储。远端存储负责安全中转和最终提交，
|
||||
/// 不会先落入永久本地媒体目录。
|
||||
/// </summary>
|
||||
public async Task<MediaDownloadResult> DownloadToStorageAsync(
|
||||
StorageType storageType,
|
||||
string mediaUrl,
|
||||
string savePath,
|
||||
string cookie,
|
||||
List<string> otherUrls = null,
|
||||
CancellationToken cancellationToken = default,
|
||||
int maxRetryCount = 12,
|
||||
IMediaStorage storageOverride = null)
|
||||
{
|
||||
if (storageType == StorageType.Local)
|
||||
return await DownloadAsync(mediaUrl, savePath, cookie, otherUrls, cancellationToken, maxRetryCount: maxRetryCount);
|
||||
var storage = storageOverride ?? _storageRouter.Resolve(storageType);
|
||||
if (storage.StorageType != storageType)
|
||||
throw new InvalidOperationException("指定的媒体存储与目标存储类型不一致");
|
||||
if (storage is ICanonicalMediaStorage canonicalStorage)
|
||||
savePath = await canonicalStorage.CanonicalizePathAsync(savePath, false, cancellationToken);
|
||||
var urls = BuildCandidateUrls(mediaUrl, otherUrls, maxRetryCount);
|
||||
return await DownloadCandidatesAsync(
|
||||
urls,
|
||||
savePath,
|
||||
cookie,
|
||||
cancellationToken,
|
||||
TimeSpan.FromSeconds(60),
|
||||
TimeSpan.FromSeconds(1),
|
||||
async (response, actualPath, _, token) =>
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var responseStream = await response.Content.ReadAsStreamAsync(token);
|
||||
await storage.WriteAsync(
|
||||
actualPath,
|
||||
responseStream,
|
||||
response.Content.Headers.ContentLength,
|
||||
response.Content.Headers.ContentType?.MediaType,
|
||||
token);
|
||||
}
|
||||
catch (OperationCanceledException) when (token.IsCancellationRequested) { throw; }
|
||||
catch (MediaIntegrityException) { throw; }
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new MediaStorageException($"{storageType} 媒体写入失败：{actualPath}", ex);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private async Task<MediaDownloadResult> DownloadCandidatesAsync(
|
||||
IReadOnlyList<Uri> urls,
|
||||
string savePath,
|
||||
string cookie,
|
||||
CancellationToken cancellationToken,
|
||||
TimeSpan streamTimeout)
|
||||
TimeSpan streamTimeout,
|
||||
TimeSpan retryDelay,
|
||||
Func<HttpResponseMessage, string, TimeSpan, CancellationToken, Task> writeAsync)
|
||||
{
|
||||
DateTime lastStreamActivity = DateTime.UtcNow;
|
||||
string actualSavePath = savePath;
|
||||
string detectedExtension = string.Empty;
|
||||
if (urls.Count == 0)
|
||||
return Failure(MediaDownloadFailureKind.SourceUnavailable, savePath, null, null, 0, null, "没有有效的媒体下载地址。");
|
||||
|
||||
// 确保目录存在
|
||||
var directory = Path.GetDirectoryName(savePath);
|
||||
if (!Directory.Exists(directory))
|
||||
var client = _clientFactory.CreateClient(DouyinRequestParamManager.DY_HTTP_CLIENT_DOWN);
|
||||
MediaDownloadResult lastFailure = null;
|
||||
var forbiddenCount = 0;
|
||||
var attempted = 0;
|
||||
|
||||
foreach (var source in urls)
|
||||
{
|
||||
Directory.CreateDirectory(directory);
|
||||
attempted++;
|
||||
var networkRetry = 0;
|
||||
var rateLimitRetry = 0;
|
||||
while (true)
|
||||
{
|
||||
HttpResponseMessage response = null;
|
||||
Uri finalUri = source;
|
||||
try
|
||||
{
|
||||
(response, finalUri) = await SendWithSafeRedirectsAsync(client, source, cookie, cancellationToken);
|
||||
var status = (int)response.StatusCode;
|
||||
var host = SafeHost(finalUri);
|
||||
|
||||
if (response.StatusCode == HttpStatusCode.Unauthorized)
|
||||
return Failure(MediaDownloadFailureKind.SourceUnauthorized, savePath, status, host, attempted, null,
|
||||
"抖音媒体请求返回 401，需要重新授权。");
|
||||
|
||||
if (response.StatusCode == HttpStatusCode.Forbidden)
|
||||
{
|
||||
forbiddenCount++;
|
||||
lastFailure = Failure(MediaDownloadFailureKind.SourceForbidden, savePath, status, host, attempted, null,
|
||||
"抖音媒体来源拒绝访问（403）。");
|
||||
break;
|
||||
}
|
||||
|
||||
if (response.StatusCode is HttpStatusCode.NotFound or HttpStatusCode.Gone)
|
||||
{
|
||||
lastFailure = Failure(MediaDownloadFailureKind.SourceNotFound, savePath, status, host, attempted, null,
|
||||
$"媒体来源返回 {status}，作品可能已下架或链接已失效。");
|
||||
break;
|
||||
}
|
||||
|
||||
if ((int)response.StatusCode == 429)
|
||||
{
|
||||
var retryAt = GetRetryAfter(response);
|
||||
if (rateLimitRetry++ == 0)
|
||||
{
|
||||
var wait = retryAt.HasValue ? retryAt.Value - DateTime.UtcNow : TimeSpan.FromSeconds(5);
|
||||
if (wait < TimeSpan.Zero) wait = TimeSpan.Zero;
|
||||
if (wait > TimeSpan.FromSeconds(60)) wait = TimeSpan.FromSeconds(60);
|
||||
await _delayAsync(wait, cancellationToken);
|
||||
continue;
|
||||
}
|
||||
return Failure(MediaDownloadFailureKind.SourceRateLimited, savePath, status, host, attempted, retryAt,
|
||||
"抖音媒体来源请求过于频繁（429），已进入冷却等待。");
|
||||
}
|
||||
|
||||
if ((int)response.StatusCode >= 500)
|
||||
{
|
||||
if (networkRetry < MediaNetworkRetryCount)
|
||||
{
|
||||
networkRetry++;
|
||||
await DelayMediaRetryAsync(retryDelay, networkRetry, cancellationToken);
|
||||
continue;
|
||||
}
|
||||
lastFailure = Failure(MediaDownloadFailureKind.SourceUnavailable, savePath, status, host, attempted, null,
|
||||
$"媒体来源暂时不可用（{status}）。");
|
||||
break;
|
||||
}
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
lastFailure = Failure(MediaDownloadFailureKind.SourceUnavailable, savePath, status, host, attempted, null,
|
||||
$"媒体来源请求失败（{status}）。");
|
||||
break;
|
||||
}
|
||||
|
||||
var mediaType = response.Content.Headers.ContentType?.MediaType;
|
||||
if (IsObviousNonMedia(mediaType))
|
||||
{
|
||||
lastFailure = Failure(MediaDownloadFailureKind.SourceInvalidContent, savePath, status, host, attempted, null,
|
||||
$"媒体来源返回了非媒体内容（{mediaType}）。");
|
||||
break;
|
||||
}
|
||||
if (response.Content.Headers.ContentLength == 0)
|
||||
{
|
||||
lastFailure = Failure(MediaDownloadFailureKind.IntegrityCheckFailed, savePath, status, host, attempted, null,
|
||||
"媒体来源返回空内容。");
|
||||
break;
|
||||
}
|
||||
|
||||
var actualPath = DetectActualSavePath(savePath, mediaType);
|
||||
await writeAsync(response, actualPath, streamTimeout, cancellationToken);
|
||||
return MediaDownloadResult.Succeeded(actualPath, host, attempted);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
CleanupIncompleteFile(savePath);
|
||||
return Failure(MediaDownloadFailureKind.Cancelled, savePath, null, SafeHost(finalUri), attempted, null, "媒体下载已取消。");
|
||||
}
|
||||
catch (MediaStorageException ex)
|
||||
{
|
||||
CleanupIncompleteFile(savePath);
|
||||
Log.Error(ex, "媒体存储写入失败：{Path}", savePath);
|
||||
return Failure(MediaDownloadFailureKind.StorageUnavailable, savePath, null, SafeHost(finalUri), attempted, null,
|
||||
DescribeStorageFailure(ex));
|
||||
}
|
||||
catch (MediaIntegrityException ex)
|
||||
{
|
||||
CleanupIncompleteFile(savePath);
|
||||
lastFailure = Failure(MediaDownloadFailureKind.IntegrityCheckFailed, savePath, null, SafeHost(finalUri), attempted, null, ex.Message);
|
||||
break;
|
||||
}
|
||||
catch (MediaSourceException ex)
|
||||
{
|
||||
lastFailure = Failure(ex.FailureKind, savePath, ex.StatusCode, ex.SourceHost, attempted, ex.RetryAfter, ex.Message);
|
||||
break;
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or TimeoutException or TaskCanceledException or IOException)
|
||||
{
|
||||
if (networkRetry < MediaNetworkRetryCount)
|
||||
{
|
||||
networkRetry++;
|
||||
await DelayMediaRetryAsync(retryDelay, networkRetry, cancellationToken);
|
||||
continue;
|
||||
}
|
||||
lastFailure = Failure(MediaDownloadFailureKind.SourceUnavailable, savePath, null, SafeHost(finalUri), attempted, null,
|
||||
"连接媒体来源失败。");
|
||||
break;
|
||||
}
|
||||
finally
|
||||
{
|
||||
response?.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 清理已存在的文件
|
||||
CleanupIncompleteFile(savePath);
|
||||
if (forbiddenCount == attempted && attempted > 0)
|
||||
lastFailure = Failure(MediaDownloadFailureKind.SourceForbidden, savePath, 403, lastFailure?.SourceHost,
|
||||
attempted, null, $"全部 {attempted} 个候选媒体地址均返回 403。");
|
||||
else if (lastFailure?.FailureKind == MediaDownloadFailureKind.SourceForbidden)
|
||||
lastFailure = Failure(MediaDownloadFailureKind.SourceUnavailable, savePath, null, lastFailure.SourceHost,
|
||||
attempted, null, "候选媒体地址返回了多种来源错误。");
|
||||
|
||||
// 优化2：使用using包裹下载专用HttpClient（工厂创建的Client仍可using，释放内部handler）
|
||||
using var httpClient = _clientFactory.CreateClient(DouyinRequestParamManager.DY_HTTP_CLIENT_DOWN);
|
||||
lastFailure ??= Failure(MediaDownloadFailureKind.SourceUnavailable, savePath, null, null, attempted, null, "所有候选媒体地址均不可用。");
|
||||
if (lastFailure.FailureKind == MediaDownloadFailureKind.SourceForbidden)
|
||||
Log.Warning("抖音媒体来源被拒绝：Host={SourceHost}, Status={Status}, Candidates={Candidates}",
|
||||
lastFailure.SourceHost, lastFailure.HttpStatusCode, lastFailure.AttemptedUrlCount);
|
||||
else
|
||||
Log.Warning("媒体来源下载失败：Host={SourceHost}, Status={Status}, Kind={Kind}, Candidates={Candidates}",
|
||||
lastFailure.SourceHost, lastFailure.HttpStatusCode, lastFailure.FailureKind, lastFailure.AttemptedUrlCount);
|
||||
return lastFailure;
|
||||
}
|
||||
|
||||
private Task DelayMediaRetryAsync(TimeSpan baseDelay, int retryNumber, CancellationToken cancellationToken)
|
||||
{
|
||||
var multiplier = Math.Pow(2, Math.Max(0, retryNumber - 1));
|
||||
var milliseconds = Math.Min(5000, Math.Max(100, baseDelay.TotalMilliseconds * multiplier));
|
||||
milliseconds += Random.Shared.Next(0, 251);
|
||||
return _delayAsync(TimeSpan.FromMilliseconds(milliseconds), cancellationToken);
|
||||
}
|
||||
|
||||
private static List<Uri> BuildCandidateUrls(string primary, IEnumerable<string> alternatives, int limit)
|
||||
{
|
||||
var result = new List<Uri>();
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
foreach (var value in new[] { primary }.Concat(alternatives ?? Enumerable.Empty<string>()))
|
||||
{
|
||||
if (result.Count >= Math.Clamp(limit, 1, 12)) break;
|
||||
if (!Uri.TryCreate(value, UriKind.Absolute, out var uri) || (uri.Scheme != Uri.UriSchemeHttp && uri.Scheme != Uri.UriSchemeHttps)) continue;
|
||||
if (seen.Add(uri.AbsoluteUri)) result.Add(uri);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
internal static string DescribeStorageFailure(MediaStorageException error)
|
||||
{
|
||||
if (error == null) return "媒体存储写入失败。";
|
||||
var message = error.Message?.Trim();
|
||||
var reason = error.GetBaseException()?.Message?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(message)) message = "媒体存储写入失败。";
|
||||
if (string.IsNullOrWhiteSpace(reason) || string.Equals(message, reason, StringComparison.Ordinal))
|
||||
return message;
|
||||
return $"{message}；原因：{reason}";
|
||||
}
|
||||
|
||||
private static async Task<(HttpResponseMessage Response, Uri FinalUri)> SendWithSafeRedirectsAsync(
|
||||
HttpClient client,
|
||||
Uri initialUri,
|
||||
string cookie,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var current = initialUri;
|
||||
for (var redirect = 0; redirect <= 5; redirect++)
|
||||
{
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, current);
|
||||
if (!string.IsNullOrWhiteSpace(cookie) && IsDouyinHost(current.Host))
|
||||
request.Headers.TryAddWithoutValidation("Cookie", cookie);
|
||||
var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
|
||||
if (!IsRedirect(response.StatusCode)) return (response, current);
|
||||
|
||||
var location = response.Headers.Location;
|
||||
response.Dispose();
|
||||
if (location == null)
|
||||
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable, "媒体来源返回了无目标地址的跳转。",
|
||||
(int)response.StatusCode, SafeHost(current));
|
||||
if (redirect == 5)
|
||||
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable, "媒体来源跳转次数超过 5 次。",
|
||||
(int)response.StatusCode, SafeHost(current));
|
||||
current = location.IsAbsoluteUri ? location : new Uri(current, location);
|
||||
if (current.Scheme != Uri.UriSchemeHttp && current.Scheme != Uri.UriSchemeHttps)
|
||||
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable, "媒体来源返回了不安全的跳转协议。",
|
||||
sourceHost: SafeHost(current));
|
||||
}
|
||||
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable, "媒体来源跳转失败。", sourceHost: SafeHost(current));
|
||||
}
|
||||
|
||||
private static async Task WriteLocalAsync(
|
||||
HttpResponseMessage response,
|
||||
string actualSavePath,
|
||||
TimeSpan streamTimeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var directory = Path.GetDirectoryName(actualSavePath);
|
||||
FileStream fileStream = null;
|
||||
try
|
||||
{
|
||||
// 配置请求头
|
||||
httpClient.DefaultRequestHeaders.Remove("Cookie"); // 先移除再添加，避免重复
|
||||
httpClient.DefaultRequestHeaders.Add("Cookie", cookie);
|
||||
httpClient.Timeout = TimeSpan.FromMinutes(5);
|
||||
|
||||
// 优化3：使用HttpCompletionOption.ResponseHeadersRead，不缓存整个响应
|
||||
using var response = await httpClient.GetAsync(
|
||||
videoUrl,
|
||||
HttpCompletionOption.ResponseHeadersRead,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
long? totalBytes = response.Content.Headers.ContentLength;
|
||||
|
||||
// 检测文件类型
|
||||
var ext = Path.GetExtension(savePath);
|
||||
if (ext == ".mp3")
|
||||
{
|
||||
var contentType = response.Content.Headers.ContentType?.MediaType;
|
||||
if (!string.IsNullOrEmpty(contentType))
|
||||
{
|
||||
if (contentType.Contains("audio/mp4") || contentType.Contains("audio/m4a"))
|
||||
detectedExtension = "m4a";
|
||||
else if (contentType.Contains("audio/mpeg") || contentType.Contains("audio/mp3"))
|
||||
detectedExtension = "mp3";
|
||||
else if (contentType.Contains("video/mp4"))
|
||||
detectedExtension = "mp4";
|
||||
}
|
||||
}
|
||||
|
||||
// 修正保存路径
|
||||
if (!string.IsNullOrEmpty(detectedExtension))
|
||||
{
|
||||
actualSavePath = Path.ChangeExtension(actualSavePath, detectedExtension.Trim('.'));
|
||||
CleanupIncompleteFile(actualSavePath);
|
||||
}
|
||||
|
||||
// 优化4：流式写入文件，且使用ArrayPool复用缓冲区（减少大对象分配）
|
||||
using var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
using var fileStream = new FileStream(
|
||||
actualSavePath,
|
||||
FileMode.CreateNew,
|
||||
FileAccess.Write,
|
||||
FileShare.None,
|
||||
bufferSize: 8192,
|
||||
FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
|
||||
var buffer = System.Buffers.ArrayPool<byte>.Shared.Rent(8192); // 复用缓冲区
|
||||
try
|
||||
{
|
||||
int bytesRead;
|
||||
long totalRead = 0;
|
||||
if (!string.IsNullOrWhiteSpace(directory)) Directory.CreateDirectory(directory);
|
||||
CleanupIncompleteFile(actualSavePath);
|
||||
fileStream = new FileStream(actualSavePath, FileMode.CreateNew, FileAccess.Write, FileShare.None,
|
||||
81920, FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
}
|
||||
catch (Exception ex) when (ex is UnauthorizedAccessException or IOException)
|
||||
{
|
||||
throw new MediaStorageException($"本地媒体写入失败：{actualSavePath}", ex);
|
||||
}
|
||||
|
||||
while ((bytesRead = await responseStream.ReadAsync(buffer, 0, buffer.Length, cancellationToken).ConfigureAwait(false)) > 0)
|
||||
Stream responseStream;
|
||||
try { responseStream = await response.Content.ReadAsStreamAsync(cancellationToken); }
|
||||
catch (Exception ex) when (ex is HttpRequestException or IOException)
|
||||
{
|
||||
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable, "读取媒体来源响应失败。",
|
||||
sourceHost: SafeHost(response.RequestMessage?.RequestUri), innerException: ex);
|
||||
}
|
||||
await using var ownedResponseStream = responseStream;
|
||||
var buffer = System.Buffers.ArrayPool<byte>.Shared.Rent(81920);
|
||||
long totalRead = 0;
|
||||
var lastActivity = DateTime.UtcNow;
|
||||
try
|
||||
{
|
||||
int read;
|
||||
while (true)
|
||||
{
|
||||
if (DateTime.UtcNow - lastStreamActivity > streamTimeout)
|
||||
throw new TimeoutException($"流读取超时（{streamTimeout.TotalSeconds}秒无数据）");
|
||||
|
||||
await fileStream.WriteAsync(buffer, 0, bytesRead, cancellationToken).ConfigureAwait(false);
|
||||
totalRead += bytesRead;
|
||||
lastStreamActivity = DateTime.UtcNow;
|
||||
try { read = await responseStream.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken); }
|
||||
catch (Exception ex) when (ex is HttpRequestException or IOException)
|
||||
{
|
||||
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable, "读取媒体来源响应失败。",
|
||||
sourceHost: SafeHost(response.RequestMessage?.RequestUri), innerException: ex);
|
||||
}
|
||||
if (read <= 0) break;
|
||||
if (DateTime.UtcNow - lastActivity > streamTimeout)
|
||||
throw new MediaSourceException(MediaDownloadFailureKind.SourceUnavailable,
|
||||
$"媒体流超过 {streamTimeout.TotalSeconds:0} 秒没有数据。");
|
||||
try { await fileStream.WriteAsync(buffer.AsMemory(0, read), cancellationToken); }
|
||||
catch (Exception ex) when (ex is UnauthorizedAccessException or IOException)
|
||||
{
|
||||
throw new MediaStorageException($"本地媒体写入失败：{actualSavePath}", ex);
|
||||
}
|
||||
totalRead += read;
|
||||
lastActivity = DateTime.UtcNow;
|
||||
}
|
||||
try { await fileStream.FlushAsync(cancellationToken); }
|
||||
catch (Exception ex) when (ex is UnauthorizedAccessException or IOException)
|
||||
{
|
||||
throw new MediaStorageException($"本地媒体写入失败：{actualSavePath}", ex);
|
||||
}
|
||||
|
||||
await fileStream.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
System.Buffers.ArrayPool<byte>.Shared.Return(buffer); // 归还缓冲区
|
||||
System.Buffers.ArrayPool<byte>.Shared.Return(buffer);
|
||||
}
|
||||
|
||||
return (true, actualSavePath);
|
||||
}
|
||||
catch
|
||||
{
|
||||
CleanupIncompleteFile(actualSavePath);
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
//httpClient.DefaultRequestHeaders.Clear(); // 清空请求头，帮助GC
|
||||
if (totalRead <= 0) throw new MediaIntegrityException("下载内容为空。");
|
||||
var expected = response.Content.Headers.ContentLength;
|
||||
if (expected.HasValue && expected.Value != totalRead)
|
||||
throw new MediaIntegrityException($"下载长度不一致：期望 {expected.Value}，实际 {totalRead}。");
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
|
||||
catch (MediaSourceException) { CleanupIncompleteFile(actualSavePath); throw; }
|
||||
catch (MediaIntegrityException) { CleanupIncompleteFile(actualSavePath); throw; }
|
||||
catch (MediaStorageException) { CleanupIncompleteFile(actualSavePath); throw; }
|
||||
finally { if (fileStream != null) await fileStream.DisposeAsync(); }
|
||||
}
|
||||
|
||||
private bool IsRetryableException(Exception ex)
|
||||
private static string DetectActualSavePath(string savePath, string mediaType)
|
||||
{
|
||||
return ex is HttpRequestException
|
||||
|| ex is TimeoutException
|
||||
|| ex is IOException
|
||||
|| (ex is AggregateException aggEx && aggEx.InnerExceptions.Any(IsRetryableException));
|
||||
if (!Path.GetExtension(savePath).Equals(".mp3", StringComparison.OrdinalIgnoreCase)) return savePath;
|
||||
if (mediaType?.Contains("audio/mp4", StringComparison.OrdinalIgnoreCase) == true
|
||||
|| mediaType?.Contains("audio/m4a", StringComparison.OrdinalIgnoreCase) == true)
|
||||
return Path.ChangeExtension(savePath, ".m4a");
|
||||
if (mediaType?.Contains("video/mp4", StringComparison.OrdinalIgnoreCase) == true)
|
||||
return Path.ChangeExtension(savePath, ".mp4");
|
||||
return savePath;
|
||||
}
|
||||
|
||||
private static bool IsObviousNonMedia(string mediaType) =>
|
||||
!string.IsNullOrWhiteSpace(mediaType)
|
||||
&& (mediaType.StartsWith("text/html", StringComparison.OrdinalIgnoreCase)
|
||||
|| mediaType.Contains("json", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
private static bool IsDouyinHost(string host) =>
|
||||
string.Equals(host, "douyin.com", StringComparison.OrdinalIgnoreCase)
|
||||
|| host.EndsWith(".douyin.com", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static bool IsRedirect(HttpStatusCode code) => code is
|
||||
HttpStatusCode.MovedPermanently or HttpStatusCode.Redirect or HttpStatusCode.RedirectMethod
|
||||
or HttpStatusCode.TemporaryRedirect or HttpStatusCode.PermanentRedirect;
|
||||
|
||||
private static string SafeHost(Uri uri) => uri?.IsAbsoluteUri == true ? uri.IdnHost : null;
|
||||
|
||||
private static DateTime? GetRetryAfter(HttpResponseMessage response)
|
||||
{
|
||||
var value = response.Headers.RetryAfter;
|
||||
if (value?.Date.HasValue == true) return value.Date.Value.UtcDateTime;
|
||||
if (value?.Delta.HasValue == true) return DateTime.UtcNow.Add(value.Delta.Value);
|
||||
return DateTime.UtcNow.AddMinutes(15);
|
||||
}
|
||||
|
||||
private static MediaDownloadResult Failure(
|
||||
MediaDownloadFailureKind kind,
|
||||
string savePath,
|
||||
int? status,
|
||||
string host,
|
||||
int attempted,
|
||||
DateTime? retryAfter,
|
||||
string message) => new()
|
||||
{
|
||||
Success = false,
|
||||
ActualSavePath = savePath,
|
||||
FailureKind = kind,
|
||||
HttpStatusCode = status,
|
||||
SourceHost = host,
|
||||
AttemptedUrlCount = attempted,
|
||||
RetryAfter = retryAfter,
|
||||
Message = message
|
||||
};
|
||||
|
||||
private static void CleanupIncompleteFile(string savePath)
|
||||
{
|
||||
if (File.Exists(savePath))
|
||||
@@ -757,4 +1052,4 @@ namespace dy.net.service
|
||||
}
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user