fix: use curl pipe to bypass FFmpeg HTTP header buffer limit

The previous -max_alloc approach does not fix the 'overlong headers' error
because FFmpeg 5.1 (Debian bookworm) uses a compile-time stack-allocated
buffer (MAX_URL_SIZE=4096) for HTTP response headers, which -max_alloc
cannot change.

Instead, when the stream URL is HTTP/HTTPS, launch curl to handle the
HTTP connection and pipe its stdout to FFmpeg via stdin (pipe:0). Curl
does not have the 4096-byte header limit, so it handles oversized CDN
response headers from Douyin without error.

Additional changes:
- RequestStopAsync kills curl first (instead of sending 'q' to FFmpeg),
  causing the pipe to close and FFmpeg to exit gracefully on EOF.
- SessionProcessRuntime tracks the curl process for cleanup.
This commit is contained in:
2026-07-09 12:19:34 +08:00
parent 8a079b4698
commit ad86c080e3
3 changed files with 132 additions and 21 deletions
@@ -1905,6 +1905,7 @@ public sealed partial class FfmpegService
public bool HasRetriedWithRefreshedStream { get; }
public bool HasRetriedWithAlternateProtocol { get; set; }
public Process? Process { get; private set; }
public Process? CurlProcess { get; private set; }
public int ProcessId => Process?.Id ?? 0;
public bool CompletionRequested { get; private set; }
public bool StopRequested { get; private set; }
@@ -1977,6 +1978,7 @@ public sealed partial class FfmpegService
private bool RuntimeSourceFailureVerificationInProgress { get; set; }
public void AttachProcess(Process process) => Process = process;
public void AttachCurlProcess(Process curlProcess) => CurlProcess = curlProcess;
public void MarkStopRequested(bool markAsCompletedOnExit)
{
@@ -2094,6 +2096,11 @@ public sealed partial class FfmpegService
public void Dispose()
{
if (CurlProcess is not null)
{
try { if (!CurlProcess.HasExited) CurlProcess.Kill(true); } catch { }
CurlProcess.Dispose();
}
DanmakuCancellation.Dispose();
Gate.Dispose();
}
@@ -659,28 +659,14 @@ public sealed partial class FfmpegService
var arguments = new List<string> { "-hide_banner", "-y", "-progress", "pipe:1" };
var useIntermediateTransportStream = ShouldUseIntermediateTransportStream(outputFilePath, outputFormat, saveMode);
var effectiveStreamUrl = streamUrl;
if (IsHttpInput(streamUrl))
{
if (!string.IsNullOrWhiteSpace(inputHeaders?.UserAgent))
{
arguments.AddRange(["-user_agent", inputHeaders.UserAgent]);
}
if (!string.IsNullOrWhiteSpace(inputHeaders?.Referer))
{
arguments.AddRange(["-referer", inputHeaders.Referer]);
}
var customHeaders = BuildCustomHeaderArgument(inputHeaders);
if (!string.IsNullOrWhiteSpace(customHeaders))
{
arguments.AddRange(["-headers", customHeaders]);
}
// Increase the max allocation size to avoid "overlong headers" errors
// when CDN (e.g. Douyin) returns HTTP response headers exceeding FFmpeg's
// default internal buffer (4096 bytes).
arguments.AddRange(["-max_alloc", "100000000"]);
// When the input is HTTP, use pipe:0 so that curl handles the HTTP connection.
// This avoids FFmpeg's built-in HTTP handler which has a hard-coded 4096-byte
// response header buffer (MAX_URL_SIZE) that triggers "overlong headers" errors
// when CDNs like Douyin return oversized headers.
effectiveStreamUrl = "pipe:0";
}
if (enableReconnect &&
@@ -696,7 +682,7 @@ public sealed partial class FfmpegService
]);
}
arguments.AddRange(["-rw_timeout", readWriteTimeoutMilliseconds.ToString(), "-fflags", "+discardcorrupt+genpts", "-i", streamUrl]);
arguments.AddRange(["-rw_timeout", readWriteTimeoutMilliseconds.ToString(), "-fflags", "+discardcorrupt+genpts", "-i", effectiveStreamUrl]);
arguments.AddRange(BuildCodecArguments(recordingTemplate));
var writesTransportStream = useIntermediateTransportStream || outputFormat == RecordOutputFormat.Ts;
@@ -988,6 +974,44 @@ public sealed partial class FfmpegService
return builder.Length == 0 ? null : builder.ToString();
}
/// <summary>
/// Builds the curl command-line arguments for piping an HTTP stream to FFmpeg.
/// Curl does not have FFmpeg's 4096-byte response header limit, so it avoids
/// the "overlong headers" error that occurs when CDNs return oversized headers.
/// </summary>
internal static IReadOnlyList<string> BuildCurlArgumentList(
string streamUrl,
StreamInputHeaders? inputHeaders)
{
var args = new List<string> { "-sL" };
if (!string.IsNullOrWhiteSpace(inputHeaders?.UserAgent))
{
args.AddRange(["-A", inputHeaders.UserAgent]);
}
if (!string.IsNullOrWhiteSpace(inputHeaders?.Referer))
{
args.AddRange(["-e", inputHeaders.Referer]);
}
if (!string.IsNullOrWhiteSpace(inputHeaders?.Cookie))
{
args.AddRange(["-b", inputHeaders.Cookie.Trim()]);
}
if (inputHeaders?.AdditionalHeaders is not null)
{
foreach (var pair in inputHeaders.AdditionalHeaders.Where(static p => !string.IsNullOrWhiteSpace(p.Key)))
{
args.AddRange(["-H", $"{pair.Key.Trim()}: {pair.Value?.Trim() ?? ""}"]);
}
}
args.Add(streamUrl);
return args;
}
private static bool TryParseSegmentOpenPath(string line, out string openedPath)
{
var match = SegmentOpeningRegex.Match(line);
@@ -195,6 +195,69 @@ public sealed partial class FfmpegService : IFfmpegService
throw new InvalidOperationException("A running ffmpeg process already exists for the recording session.");
}
// When the stream URL is HTTP, launch curl to handle the HTTP connection and
// pipe its stdout into FFmpeg's stdin. This bypasses FFmpeg's built-in HTTP
// handler which has a hard-coded 4096-byte response header buffer that triggers
// "overlong headers" errors with CDNs that return oversized headers.
if (IsHttpInput(streamUrlResult.SelectedUrl))
{
var curlProcess = new Process
{
StartInfo = new ProcessStartInfo
{
FileName = "curl",
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true
},
EnableRaisingEvents = true
};
foreach (var arg in BuildCurlArgumentList(streamUrlResult.SelectedUrl, streamUrlResult.InputHeaders))
{
curlProcess.StartInfo.ArgumentList.Add(arg);
}
curlProcess.ErrorDataReceived += (_, args) =>
{
if (!string.IsNullOrWhiteSpace(args.Data))
{
_logger.LogDebug("curl[{SessionId}] {Line}", recordSession.Id, args.Data);
}
};
if (!curlProcess.Start())
{
process.Kill(true);
process.Dispose();
throw new InvalidOperationException("curl failed to start.");
}
curlProcess.BeginErrorReadLine();
runtime.AttachCurlProcess(curlProcess);
// Pipe curl stdout → FFmpeg stdin in the background.
// When curl exits or is killed, the pipe closes and FFmpeg sees EOF on stdin,
// which causes it to exit gracefully after finalizing the output.
_ = Task.Run(async () =>
{
try
{
await curlProcess.StandardOutput.BaseStream.CopyToAsync(
process.StandardInput.BaseStream);
}
catch (Exception ex)
{
_logger.LogDebug(ex, "curl→ffmpeg pipe ended for session {RecordSessionId}", recordSession.Id);
}
finally
{
try { process.StandardInput.Close(); } catch { /* stdin may already be closed */ }
}
});
}
await PersistProcessBindingAsync(recordSession.Id, initialTask.Id, process.Id, cancellationToken);
await PersistStartupProfileAsync(runtime, cancellationToken);
await StartDanmakuAsync(runtime, initialTask, cancellationToken);
@@ -714,6 +777,23 @@ public sealed partial class FfmpegService : IFfmpegService
return;
}
// When curl is piping the stream, kill curl first so the pipe closes
// and FFmpeg sees EOF on stdin, causing a graceful exit.
var curlProcess = runtime.CurlProcess;
if (curlProcess is not null && !curlProcess.HasExited)
{
try
{
curlProcess.Kill(true);
}
catch (Exception curlEx)
{
_logger.LogWarning(curlEx, "Kill curl process failed for session {RecordSessionId}", recordSessionId);
}
return;
}
await process.StandardInput.WriteLineAsync("q");
await process.StandardInput.FlushAsync();
}