fix: use curl pipe to bypass FFmpeg HTTP header buffer limit
The previous -max_alloc approach does not fix the 'overlong headers' error because FFmpeg 5.1 (Debian bookworm) uses a compile-time stack-allocated buffer (MAX_URL_SIZE=4096) for HTTP response headers, which -max_alloc cannot change. Instead, when the stream URL is HTTP/HTTPS, launch curl to handle the HTTP connection and pipe its stdout to FFmpeg via stdin (pipe:0). Curl does not have the 4096-byte header limit, so it handles oversized CDN response headers from Douyin without error. Additional changes: - RequestStopAsync kills curl first (instead of sending 'q' to FFmpeg), causing the pipe to close and FFmpeg to exit gracefully on EOF. - SessionProcessRuntime tracks the curl process for cleanup.
This commit is contained in:
@@ -1905,6 +1905,7 @@ public sealed partial class FfmpegService
|
|||||||
public bool HasRetriedWithRefreshedStream { get; }
|
public bool HasRetriedWithRefreshedStream { get; }
|
||||||
public bool HasRetriedWithAlternateProtocol { get; set; }
|
public bool HasRetriedWithAlternateProtocol { get; set; }
|
||||||
public Process? Process { get; private set; }
|
public Process? Process { get; private set; }
|
||||||
|
public Process? CurlProcess { get; private set; }
|
||||||
public int ProcessId => Process?.Id ?? 0;
|
public int ProcessId => Process?.Id ?? 0;
|
||||||
public bool CompletionRequested { get; private set; }
|
public bool CompletionRequested { get; private set; }
|
||||||
public bool StopRequested { get; private set; }
|
public bool StopRequested { get; private set; }
|
||||||
@@ -1977,6 +1978,7 @@ public sealed partial class FfmpegService
|
|||||||
private bool RuntimeSourceFailureVerificationInProgress { get; set; }
|
private bool RuntimeSourceFailureVerificationInProgress { get; set; }
|
||||||
|
|
||||||
public void AttachProcess(Process process) => Process = process;
|
public void AttachProcess(Process process) => Process = process;
|
||||||
|
public void AttachCurlProcess(Process curlProcess) => CurlProcess = curlProcess;
|
||||||
|
|
||||||
public void MarkStopRequested(bool markAsCompletedOnExit)
|
public void MarkStopRequested(bool markAsCompletedOnExit)
|
||||||
{
|
{
|
||||||
@@ -2094,6 +2096,11 @@ public sealed partial class FfmpegService
|
|||||||
|
|
||||||
public void Dispose()
|
public void Dispose()
|
||||||
{
|
{
|
||||||
|
if (CurlProcess is not null)
|
||||||
|
{
|
||||||
|
try { if (!CurlProcess.HasExited) CurlProcess.Kill(true); } catch { }
|
||||||
|
CurlProcess.Dispose();
|
||||||
|
}
|
||||||
DanmakuCancellation.Dispose();
|
DanmakuCancellation.Dispose();
|
||||||
Gate.Dispose();
|
Gate.Dispose();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -659,28 +659,14 @@ public sealed partial class FfmpegService
|
|||||||
var arguments = new List<string> { "-hide_banner", "-y", "-progress", "pipe:1" };
|
var arguments = new List<string> { "-hide_banner", "-y", "-progress", "pipe:1" };
|
||||||
var useIntermediateTransportStream = ShouldUseIntermediateTransportStream(outputFilePath, outputFormat, saveMode);
|
var useIntermediateTransportStream = ShouldUseIntermediateTransportStream(outputFilePath, outputFormat, saveMode);
|
||||||
|
|
||||||
|
var effectiveStreamUrl = streamUrl;
|
||||||
if (IsHttpInput(streamUrl))
|
if (IsHttpInput(streamUrl))
|
||||||
{
|
{
|
||||||
if (!string.IsNullOrWhiteSpace(inputHeaders?.UserAgent))
|
// When the input is HTTP, use pipe:0 so that curl handles the HTTP connection.
|
||||||
{
|
// This avoids FFmpeg's built-in HTTP handler which has a hard-coded 4096-byte
|
||||||
arguments.AddRange(["-user_agent", inputHeaders.UserAgent]);
|
// response header buffer (MAX_URL_SIZE) that triggers "overlong headers" errors
|
||||||
}
|
// when CDNs like Douyin return oversized headers.
|
||||||
|
effectiveStreamUrl = "pipe:0";
|
||||||
if (!string.IsNullOrWhiteSpace(inputHeaders?.Referer))
|
|
||||||
{
|
|
||||||
arguments.AddRange(["-referer", inputHeaders.Referer]);
|
|
||||||
}
|
|
||||||
|
|
||||||
var customHeaders = BuildCustomHeaderArgument(inputHeaders);
|
|
||||||
if (!string.IsNullOrWhiteSpace(customHeaders))
|
|
||||||
{
|
|
||||||
arguments.AddRange(["-headers", customHeaders]);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Increase the max allocation size to avoid "overlong headers" errors
|
|
||||||
// when CDN (e.g. Douyin) returns HTTP response headers exceeding FFmpeg's
|
|
||||||
// default internal buffer (4096 bytes).
|
|
||||||
arguments.AddRange(["-max_alloc", "100000000"]);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (enableReconnect &&
|
if (enableReconnect &&
|
||||||
@@ -696,7 +682,7 @@ public sealed partial class FfmpegService
|
|||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
|
|
||||||
arguments.AddRange(["-rw_timeout", readWriteTimeoutMilliseconds.ToString(), "-fflags", "+discardcorrupt+genpts", "-i", streamUrl]);
|
arguments.AddRange(["-rw_timeout", readWriteTimeoutMilliseconds.ToString(), "-fflags", "+discardcorrupt+genpts", "-i", effectiveStreamUrl]);
|
||||||
arguments.AddRange(BuildCodecArguments(recordingTemplate));
|
arguments.AddRange(BuildCodecArguments(recordingTemplate));
|
||||||
|
|
||||||
var writesTransportStream = useIntermediateTransportStream || outputFormat == RecordOutputFormat.Ts;
|
var writesTransportStream = useIntermediateTransportStream || outputFormat == RecordOutputFormat.Ts;
|
||||||
@@ -988,6 +974,44 @@ public sealed partial class FfmpegService
|
|||||||
return builder.Length == 0 ? null : builder.ToString();
|
return builder.Length == 0 ? null : builder.ToString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Builds the curl command-line arguments for piping an HTTP stream to FFmpeg.
|
||||||
|
/// Curl does not have FFmpeg's 4096-byte response header limit, so it avoids
|
||||||
|
/// the "overlong headers" error that occurs when CDNs return oversized headers.
|
||||||
|
/// </summary>
|
||||||
|
internal static IReadOnlyList<string> BuildCurlArgumentList(
|
||||||
|
string streamUrl,
|
||||||
|
StreamInputHeaders? inputHeaders)
|
||||||
|
{
|
||||||
|
var args = new List<string> { "-sL" };
|
||||||
|
|
||||||
|
if (!string.IsNullOrWhiteSpace(inputHeaders?.UserAgent))
|
||||||
|
{
|
||||||
|
args.AddRange(["-A", inputHeaders.UserAgent]);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!string.IsNullOrWhiteSpace(inputHeaders?.Referer))
|
||||||
|
{
|
||||||
|
args.AddRange(["-e", inputHeaders.Referer]);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!string.IsNullOrWhiteSpace(inputHeaders?.Cookie))
|
||||||
|
{
|
||||||
|
args.AddRange(["-b", inputHeaders.Cookie.Trim()]);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (inputHeaders?.AdditionalHeaders is not null)
|
||||||
|
{
|
||||||
|
foreach (var pair in inputHeaders.AdditionalHeaders.Where(static p => !string.IsNullOrWhiteSpace(p.Key)))
|
||||||
|
{
|
||||||
|
args.AddRange(["-H", $"{pair.Key.Trim()}: {pair.Value?.Trim() ?? ""}"]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
args.Add(streamUrl);
|
||||||
|
return args;
|
||||||
|
}
|
||||||
|
|
||||||
private static bool TryParseSegmentOpenPath(string line, out string openedPath)
|
private static bool TryParseSegmentOpenPath(string line, out string openedPath)
|
||||||
{
|
{
|
||||||
var match = SegmentOpeningRegex.Match(line);
|
var match = SegmentOpeningRegex.Match(line);
|
||||||
|
|||||||
@@ -195,6 +195,69 @@ public sealed partial class FfmpegService : IFfmpegService
|
|||||||
throw new InvalidOperationException("A running ffmpeg process already exists for the recording session.");
|
throw new InvalidOperationException("A running ffmpeg process already exists for the recording session.");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// When the stream URL is HTTP, launch curl to handle the HTTP connection and
|
||||||
|
// pipe its stdout into FFmpeg's stdin. This bypasses FFmpeg's built-in HTTP
|
||||||
|
// handler which has a hard-coded 4096-byte response header buffer that triggers
|
||||||
|
// "overlong headers" errors with CDNs that return oversized headers.
|
||||||
|
if (IsHttpInput(streamUrlResult.SelectedUrl))
|
||||||
|
{
|
||||||
|
var curlProcess = new Process
|
||||||
|
{
|
||||||
|
StartInfo = new ProcessStartInfo
|
||||||
|
{
|
||||||
|
FileName = "curl",
|
||||||
|
UseShellExecute = false,
|
||||||
|
RedirectStandardOutput = true,
|
||||||
|
RedirectStandardError = true,
|
||||||
|
CreateNoWindow = true
|
||||||
|
},
|
||||||
|
EnableRaisingEvents = true
|
||||||
|
};
|
||||||
|
|
||||||
|
foreach (var arg in BuildCurlArgumentList(streamUrlResult.SelectedUrl, streamUrlResult.InputHeaders))
|
||||||
|
{
|
||||||
|
curlProcess.StartInfo.ArgumentList.Add(arg);
|
||||||
|
}
|
||||||
|
|
||||||
|
curlProcess.ErrorDataReceived += (_, args) =>
|
||||||
|
{
|
||||||
|
if (!string.IsNullOrWhiteSpace(args.Data))
|
||||||
|
{
|
||||||
|
_logger.LogDebug("curl[{SessionId}] {Line}", recordSession.Id, args.Data);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!curlProcess.Start())
|
||||||
|
{
|
||||||
|
process.Kill(true);
|
||||||
|
process.Dispose();
|
||||||
|
throw new InvalidOperationException("curl failed to start.");
|
||||||
|
}
|
||||||
|
|
||||||
|
curlProcess.BeginErrorReadLine();
|
||||||
|
runtime.AttachCurlProcess(curlProcess);
|
||||||
|
|
||||||
|
// Pipe curl stdout → FFmpeg stdin in the background.
|
||||||
|
// When curl exits or is killed, the pipe closes and FFmpeg sees EOF on stdin,
|
||||||
|
// which causes it to exit gracefully after finalizing the output.
|
||||||
|
_ = Task.Run(async () =>
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await curlProcess.StandardOutput.BaseStream.CopyToAsync(
|
||||||
|
process.StandardInput.BaseStream);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogDebug(ex, "curl→ffmpeg pipe ended for session {RecordSessionId}", recordSession.Id);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
try { process.StandardInput.Close(); } catch { /* stdin may already be closed */ }
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
await PersistProcessBindingAsync(recordSession.Id, initialTask.Id, process.Id, cancellationToken);
|
await PersistProcessBindingAsync(recordSession.Id, initialTask.Id, process.Id, cancellationToken);
|
||||||
await PersistStartupProfileAsync(runtime, cancellationToken);
|
await PersistStartupProfileAsync(runtime, cancellationToken);
|
||||||
await StartDanmakuAsync(runtime, initialTask, cancellationToken);
|
await StartDanmakuAsync(runtime, initialTask, cancellationToken);
|
||||||
@@ -714,6 +777,23 @@ public sealed partial class FfmpegService : IFfmpegService
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// When curl is piping the stream, kill curl first so the pipe closes
|
||||||
|
// and FFmpeg sees EOF on stdin, causing a graceful exit.
|
||||||
|
var curlProcess = runtime.CurlProcess;
|
||||||
|
if (curlProcess is not null && !curlProcess.HasExited)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
curlProcess.Kill(true);
|
||||||
|
}
|
||||||
|
catch (Exception curlEx)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(curlEx, "Kill curl process failed for session {RecordSessionId}", recordSessionId);
|
||||||
|
}
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
await process.StandardInput.WriteLineAsync("q");
|
await process.StandardInput.WriteLineAsync("q");
|
||||||
await process.StandardInput.FlushAsync();
|
await process.StandardInput.FlushAsync();
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user