fix: use curl pipe to bypass FFmpeg HTTP header buffer limit
The previous -max_alloc approach does not fix the 'overlong headers' error because FFmpeg 5.1 (Debian bookworm) uses a compile-time stack-allocated buffer (MAX_URL_SIZE=4096) for HTTP response headers, which -max_alloc cannot change. Instead, when the stream URL is HTTP/HTTPS, launch curl to handle the HTTP connection and pipe its stdout to FFmpeg via stdin (pipe:0). Curl does not have the 4096-byte header limit, so it handles oversized CDN response headers from Douyin without error. Additional changes: - RequestStopAsync kills curl first (instead of sending 'q' to FFmpeg), causing the pipe to close and FFmpeg to exit gracefully on EOF. - SessionProcessRuntime tracks the curl process for cleanup.
This commit is contained in:
@@ -1905,6 +1905,7 @@ public sealed partial class FfmpegService
|
||||
public bool HasRetriedWithRefreshedStream { get; }
|
||||
public bool HasRetriedWithAlternateProtocol { get; set; }
|
||||
public Process? Process { get; private set; }
|
||||
public Process? CurlProcess { get; private set; }
|
||||
public int ProcessId => Process?.Id ?? 0;
|
||||
public bool CompletionRequested { get; private set; }
|
||||
public bool StopRequested { get; private set; }
|
||||
@@ -1977,6 +1978,7 @@ public sealed partial class FfmpegService
|
||||
private bool RuntimeSourceFailureVerificationInProgress { get; set; }
|
||||
|
||||
public void AttachProcess(Process process) => Process = process;
|
||||
public void AttachCurlProcess(Process curlProcess) => CurlProcess = curlProcess;
|
||||
|
||||
public void MarkStopRequested(bool markAsCompletedOnExit)
|
||||
{
|
||||
@@ -2094,6 +2096,11 @@ public sealed partial class FfmpegService
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (CurlProcess is not null)
|
||||
{
|
||||
try { if (!CurlProcess.HasExited) CurlProcess.Kill(true); } catch { }
|
||||
CurlProcess.Dispose();
|
||||
}
|
||||
DanmakuCancellation.Dispose();
|
||||
Gate.Dispose();
|
||||
}
|
||||
|
||||
@@ -659,28 +659,14 @@ public sealed partial class FfmpegService
|
||||
var arguments = new List<string> { "-hide_banner", "-y", "-progress", "pipe:1" };
|
||||
var useIntermediateTransportStream = ShouldUseIntermediateTransportStream(outputFilePath, outputFormat, saveMode);
|
||||
|
||||
var effectiveStreamUrl = streamUrl;
|
||||
if (IsHttpInput(streamUrl))
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(inputHeaders?.UserAgent))
|
||||
{
|
||||
arguments.AddRange(["-user_agent", inputHeaders.UserAgent]);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(inputHeaders?.Referer))
|
||||
{
|
||||
arguments.AddRange(["-referer", inputHeaders.Referer]);
|
||||
}
|
||||
|
||||
var customHeaders = BuildCustomHeaderArgument(inputHeaders);
|
||||
if (!string.IsNullOrWhiteSpace(customHeaders))
|
||||
{
|
||||
arguments.AddRange(["-headers", customHeaders]);
|
||||
}
|
||||
|
||||
// Increase the max allocation size to avoid "overlong headers" errors
|
||||
// when CDN (e.g. Douyin) returns HTTP response headers exceeding FFmpeg's
|
||||
// default internal buffer (4096 bytes).
|
||||
arguments.AddRange(["-max_alloc", "100000000"]);
|
||||
// When the input is HTTP, use pipe:0 so that curl handles the HTTP connection.
|
||||
// This avoids FFmpeg's built-in HTTP handler which has a hard-coded 4096-byte
|
||||
// response header buffer (MAX_URL_SIZE) that triggers "overlong headers" errors
|
||||
// when CDNs like Douyin return oversized headers.
|
||||
effectiveStreamUrl = "pipe:0";
|
||||
}
|
||||
|
||||
if (enableReconnect &&
|
||||
@@ -696,7 +682,7 @@ public sealed partial class FfmpegService
|
||||
]);
|
||||
}
|
||||
|
||||
arguments.AddRange(["-rw_timeout", readWriteTimeoutMilliseconds.ToString(), "-fflags", "+discardcorrupt+genpts", "-i", streamUrl]);
|
||||
arguments.AddRange(["-rw_timeout", readWriteTimeoutMilliseconds.ToString(), "-fflags", "+discardcorrupt+genpts", "-i", effectiveStreamUrl]);
|
||||
arguments.AddRange(BuildCodecArguments(recordingTemplate));
|
||||
|
||||
var writesTransportStream = useIntermediateTransportStream || outputFormat == RecordOutputFormat.Ts;
|
||||
@@ -988,6 +974,44 @@ public sealed partial class FfmpegService
|
||||
return builder.Length == 0 ? null : builder.ToString();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds the curl command-line arguments for piping an HTTP stream to FFmpeg.
|
||||
/// Curl does not have FFmpeg's 4096-byte response header limit, so it avoids
|
||||
/// the "overlong headers" error that occurs when CDNs return oversized headers.
|
||||
/// </summary>
|
||||
internal static IReadOnlyList<string> BuildCurlArgumentList(
|
||||
string streamUrl,
|
||||
StreamInputHeaders? inputHeaders)
|
||||
{
|
||||
var args = new List<string> { "-sL" };
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(inputHeaders?.UserAgent))
|
||||
{
|
||||
args.AddRange(["-A", inputHeaders.UserAgent]);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(inputHeaders?.Referer))
|
||||
{
|
||||
args.AddRange(["-e", inputHeaders.Referer]);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(inputHeaders?.Cookie))
|
||||
{
|
||||
args.AddRange(["-b", inputHeaders.Cookie.Trim()]);
|
||||
}
|
||||
|
||||
if (inputHeaders?.AdditionalHeaders is not null)
|
||||
{
|
||||
foreach (var pair in inputHeaders.AdditionalHeaders.Where(static p => !string.IsNullOrWhiteSpace(p.Key)))
|
||||
{
|
||||
args.AddRange(["-H", $"{pair.Key.Trim()}: {pair.Value?.Trim() ?? ""}"]);
|
||||
}
|
||||
}
|
||||
|
||||
args.Add(streamUrl);
|
||||
return args;
|
||||
}
|
||||
|
||||
private static bool TryParseSegmentOpenPath(string line, out string openedPath)
|
||||
{
|
||||
var match = SegmentOpeningRegex.Match(line);
|
||||
|
||||
@@ -195,6 +195,69 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
throw new InvalidOperationException("A running ffmpeg process already exists for the recording session.");
|
||||
}
|
||||
|
||||
// When the stream URL is HTTP, launch curl to handle the HTTP connection and
|
||||
// pipe its stdout into FFmpeg's stdin. This bypasses FFmpeg's built-in HTTP
|
||||
// handler which has a hard-coded 4096-byte response header buffer that triggers
|
||||
// "overlong headers" errors with CDNs that return oversized headers.
|
||||
if (IsHttpInput(streamUrlResult.SelectedUrl))
|
||||
{
|
||||
var curlProcess = new Process
|
||||
{
|
||||
StartInfo = new ProcessStartInfo
|
||||
{
|
||||
FileName = "curl",
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
CreateNoWindow = true
|
||||
},
|
||||
EnableRaisingEvents = true
|
||||
};
|
||||
|
||||
foreach (var arg in BuildCurlArgumentList(streamUrlResult.SelectedUrl, streamUrlResult.InputHeaders))
|
||||
{
|
||||
curlProcess.StartInfo.ArgumentList.Add(arg);
|
||||
}
|
||||
|
||||
curlProcess.ErrorDataReceived += (_, args) =>
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(args.Data))
|
||||
{
|
||||
_logger.LogDebug("curl[{SessionId}] {Line}", recordSession.Id, args.Data);
|
||||
}
|
||||
};
|
||||
|
||||
if (!curlProcess.Start())
|
||||
{
|
||||
process.Kill(true);
|
||||
process.Dispose();
|
||||
throw new InvalidOperationException("curl failed to start.");
|
||||
}
|
||||
|
||||
curlProcess.BeginErrorReadLine();
|
||||
runtime.AttachCurlProcess(curlProcess);
|
||||
|
||||
// Pipe curl stdout → FFmpeg stdin in the background.
|
||||
// When curl exits or is killed, the pipe closes and FFmpeg sees EOF on stdin,
|
||||
// which causes it to exit gracefully after finalizing the output.
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
try
|
||||
{
|
||||
await curlProcess.StandardOutput.BaseStream.CopyToAsync(
|
||||
process.StandardInput.BaseStream);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "curl→ffmpeg pipe ended for session {RecordSessionId}", recordSession.Id);
|
||||
}
|
||||
finally
|
||||
{
|
||||
try { process.StandardInput.Close(); } catch { /* stdin may already be closed */ }
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
await PersistProcessBindingAsync(recordSession.Id, initialTask.Id, process.Id, cancellationToken);
|
||||
await PersistStartupProfileAsync(runtime, cancellationToken);
|
||||
await StartDanmakuAsync(runtime, initialTask, cancellationToken);
|
||||
@@ -714,6 +777,23 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
return;
|
||||
}
|
||||
|
||||
// When curl is piping the stream, kill curl first so the pipe closes
|
||||
// and FFmpeg sees EOF on stdin, causing a graceful exit.
|
||||
var curlProcess = runtime.CurlProcess;
|
||||
if (curlProcess is not null && !curlProcess.HasExited)
|
||||
{
|
||||
try
|
||||
{
|
||||
curlProcess.Kill(true);
|
||||
}
|
||||
catch (Exception curlEx)
|
||||
{
|
||||
_logger.LogWarning(curlEx, "Kill curl process failed for session {RecordSessionId}", recordSessionId);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
await process.StandardInput.WriteLineAsync("q");
|
||||
await process.StandardInput.FlushAsync();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user