From ad86c080e3fc889277ef27a97afa283757d61de9 Mon Sep 17 00:00:00 2001 From: nanxun Date: Thu, 9 Jul 2026 12:19:34 +0800 Subject: [PATCH] fix: use curl pipe to bypass FFmpeg HTTP header buffer limit The previous -max_alloc approach does not fix the 'overlong headers' error because FFmpeg 5.1 (Debian bookworm) uses a compile-time stack-allocated buffer (MAX_URL_SIZE=4096) for HTTP response headers, which -max_alloc cannot change. Instead, when the stream URL is HTTP/HTTPS, launch curl to handle the HTTP connection and pipe its stdout to FFmpeg via stdin (pipe:0). Curl does not have the 4096-byte header limit, so it handles oversized CDN response headers from Douyin without error. Additional changes: - RequestStopAsync kills curl first (instead of sending 'q' to FFmpeg), causing the pipe to close and FFmpeg to exit gracefully on EOF. - SessionProcessRuntime tracks the curl process for cleanup. --- .../Services/FfmpegService.Runtime.cs | 7 ++ .../Services/FfmpegService.Utils.cs | 66 ++++++++++----- .../Services/FfmpegService.cs | 80 +++++++++++++++++++ 3 files changed, 132 insertions(+), 21 deletions(-) diff --git a/src/LiveRecorder.Infrastructure/Services/FfmpegService.Runtime.cs b/src/LiveRecorder.Infrastructure/Services/FfmpegService.Runtime.cs index cce1fd1..5fb41eb 100644 --- a/src/LiveRecorder.Infrastructure/Services/FfmpegService.Runtime.cs +++ b/src/LiveRecorder.Infrastructure/Services/FfmpegService.Runtime.cs @@ -1905,6 +1905,7 @@ public sealed partial class FfmpegService public bool HasRetriedWithRefreshedStream { get; } public bool HasRetriedWithAlternateProtocol { get; set; } public Process? Process { get; private set; } + public Process? CurlProcess { get; private set; } public int ProcessId => Process?.Id ?? 0; public bool CompletionRequested { get; private set; } public bool StopRequested { get; private set; } @@ -1977,6 +1978,7 @@ public sealed partial class FfmpegService private bool RuntimeSourceFailureVerificationInProgress { get; set; } public void AttachProcess(Process process) => Process = process; + public void AttachCurlProcess(Process curlProcess) => CurlProcess = curlProcess; public void MarkStopRequested(bool markAsCompletedOnExit) { @@ -2094,6 +2096,11 @@ public sealed partial class FfmpegService public void Dispose() { + if (CurlProcess is not null) + { + try { if (!CurlProcess.HasExited) CurlProcess.Kill(true); } catch { } + CurlProcess.Dispose(); + } DanmakuCancellation.Dispose(); Gate.Dispose(); } diff --git a/src/LiveRecorder.Infrastructure/Services/FfmpegService.Utils.cs b/src/LiveRecorder.Infrastructure/Services/FfmpegService.Utils.cs index 506b0b8..8c874f8 100644 --- a/src/LiveRecorder.Infrastructure/Services/FfmpegService.Utils.cs +++ b/src/LiveRecorder.Infrastructure/Services/FfmpegService.Utils.cs @@ -659,28 +659,14 @@ public sealed partial class FfmpegService var arguments = new List { "-hide_banner", "-y", "-progress", "pipe:1" }; var useIntermediateTransportStream = ShouldUseIntermediateTransportStream(outputFilePath, outputFormat, saveMode); + var effectiveStreamUrl = streamUrl; if (IsHttpInput(streamUrl)) { - if (!string.IsNullOrWhiteSpace(inputHeaders?.UserAgent)) - { - arguments.AddRange(["-user_agent", inputHeaders.UserAgent]); - } - - if (!string.IsNullOrWhiteSpace(inputHeaders?.Referer)) - { - arguments.AddRange(["-referer", inputHeaders.Referer]); - } - - var customHeaders = BuildCustomHeaderArgument(inputHeaders); - if (!string.IsNullOrWhiteSpace(customHeaders)) - { - arguments.AddRange(["-headers", customHeaders]); - } - - // Increase the max allocation size to avoid "overlong headers" errors - // when CDN (e.g. Douyin) returns HTTP response headers exceeding FFmpeg's - // default internal buffer (4096 bytes). - arguments.AddRange(["-max_alloc", "100000000"]); + // When the input is HTTP, use pipe:0 so that curl handles the HTTP connection. + // This avoids FFmpeg's built-in HTTP handler which has a hard-coded 4096-byte + // response header buffer (MAX_URL_SIZE) that triggers "overlong headers" errors + // when CDNs like Douyin return oversized headers. + effectiveStreamUrl = "pipe:0"; } if (enableReconnect && @@ -696,7 +682,7 @@ public sealed partial class FfmpegService ]); } - arguments.AddRange(["-rw_timeout", readWriteTimeoutMilliseconds.ToString(), "-fflags", "+discardcorrupt+genpts", "-i", streamUrl]); + arguments.AddRange(["-rw_timeout", readWriteTimeoutMilliseconds.ToString(), "-fflags", "+discardcorrupt+genpts", "-i", effectiveStreamUrl]); arguments.AddRange(BuildCodecArguments(recordingTemplate)); var writesTransportStream = useIntermediateTransportStream || outputFormat == RecordOutputFormat.Ts; @@ -988,6 +974,44 @@ public sealed partial class FfmpegService return builder.Length == 0 ? null : builder.ToString(); } + /// + /// Builds the curl command-line arguments for piping an HTTP stream to FFmpeg. + /// Curl does not have FFmpeg's 4096-byte response header limit, so it avoids + /// the "overlong headers" error that occurs when CDNs return oversized headers. + /// + internal static IReadOnlyList BuildCurlArgumentList( + string streamUrl, + StreamInputHeaders? inputHeaders) + { + var args = new List { "-sL" }; + + if (!string.IsNullOrWhiteSpace(inputHeaders?.UserAgent)) + { + args.AddRange(["-A", inputHeaders.UserAgent]); + } + + if (!string.IsNullOrWhiteSpace(inputHeaders?.Referer)) + { + args.AddRange(["-e", inputHeaders.Referer]); + } + + if (!string.IsNullOrWhiteSpace(inputHeaders?.Cookie)) + { + args.AddRange(["-b", inputHeaders.Cookie.Trim()]); + } + + if (inputHeaders?.AdditionalHeaders is not null) + { + foreach (var pair in inputHeaders.AdditionalHeaders.Where(static p => !string.IsNullOrWhiteSpace(p.Key))) + { + args.AddRange(["-H", $"{pair.Key.Trim()}: {pair.Value?.Trim() ?? ""}"]); + } + } + + args.Add(streamUrl); + return args; + } + private static bool TryParseSegmentOpenPath(string line, out string openedPath) { var match = SegmentOpeningRegex.Match(line); diff --git a/src/LiveRecorder.Infrastructure/Services/FfmpegService.cs b/src/LiveRecorder.Infrastructure/Services/FfmpegService.cs index c3e9b09..237a070 100644 --- a/src/LiveRecorder.Infrastructure/Services/FfmpegService.cs +++ b/src/LiveRecorder.Infrastructure/Services/FfmpegService.cs @@ -195,6 +195,69 @@ public sealed partial class FfmpegService : IFfmpegService throw new InvalidOperationException("A running ffmpeg process already exists for the recording session."); } + // When the stream URL is HTTP, launch curl to handle the HTTP connection and + // pipe its stdout into FFmpeg's stdin. This bypasses FFmpeg's built-in HTTP + // handler which has a hard-coded 4096-byte response header buffer that triggers + // "overlong headers" errors with CDNs that return oversized headers. + if (IsHttpInput(streamUrlResult.SelectedUrl)) + { + var curlProcess = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = "curl", + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true + }, + EnableRaisingEvents = true + }; + + foreach (var arg in BuildCurlArgumentList(streamUrlResult.SelectedUrl, streamUrlResult.InputHeaders)) + { + curlProcess.StartInfo.ArgumentList.Add(arg); + } + + curlProcess.ErrorDataReceived += (_, args) => + { + if (!string.IsNullOrWhiteSpace(args.Data)) + { + _logger.LogDebug("curl[{SessionId}] {Line}", recordSession.Id, args.Data); + } + }; + + if (!curlProcess.Start()) + { + process.Kill(true); + process.Dispose(); + throw new InvalidOperationException("curl failed to start."); + } + + curlProcess.BeginErrorReadLine(); + runtime.AttachCurlProcess(curlProcess); + + // Pipe curl stdout → FFmpeg stdin in the background. + // When curl exits or is killed, the pipe closes and FFmpeg sees EOF on stdin, + // which causes it to exit gracefully after finalizing the output. + _ = Task.Run(async () => + { + try + { + await curlProcess.StandardOutput.BaseStream.CopyToAsync( + process.StandardInput.BaseStream); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "curl→ffmpeg pipe ended for session {RecordSessionId}", recordSession.Id); + } + finally + { + try { process.StandardInput.Close(); } catch { /* stdin may already be closed */ } + } + }); + } + await PersistProcessBindingAsync(recordSession.Id, initialTask.Id, process.Id, cancellationToken); await PersistStartupProfileAsync(runtime, cancellationToken); await StartDanmakuAsync(runtime, initialTask, cancellationToken); @@ -714,6 +777,23 @@ public sealed partial class FfmpegService : IFfmpegService return; } + // When curl is piping the stream, kill curl first so the pipe closes + // and FFmpeg sees EOF on stdin, causing a graceful exit. + var curlProcess = runtime.CurlProcess; + if (curlProcess is not null && !curlProcess.HasExited) + { + try + { + curlProcess.Kill(true); + } + catch (Exception curlEx) + { + _logger.LogWarning(curlEx, "Kill curl process failed for session {RecordSessionId}", recordSessionId); + } + + return; + } + await process.StandardInput.WriteLineAsync("q"); await process.StandardInput.FlushAsync(); }