diff --git a/src/LiveRecorder.Infrastructure/Services/FfmpegService.Recovery.cs b/src/LiveRecorder.Infrastructure/Services/FfmpegService.Recovery.cs index cd36b1e..4388981 100644 --- a/src/LiveRecorder.Infrastructure/Services/FfmpegService.Recovery.cs +++ b/src/LiveRecorder.Infrastructure/Services/FfmpegService.Recovery.cs @@ -13,6 +13,8 @@ public sealed partial class FfmpegService HasRetriedWithCompatibilityProfile: false, HasRetriedWithRefreshedStream: false, HasRetriedWithAlternateProtocol: false, + UseCurlFallback: false, + HasTriedCurlFallback: false, ForceSoftwareEncoder: false, HasRetriedWithSoftwareEncoder: false); @@ -25,6 +27,7 @@ public sealed partial class FfmpegService current with { InputOptionProfile = inputOptionProfile, + UseCurlFallback = false, AttemptCount = current.AttemptCount + 1, HasRetriedWithRefreshedStream = current.HasRetriedWithRefreshedStream || refreshedStream, HasRetriedWithAlternateProtocol = current.HasRetriedWithAlternateProtocol || @@ -37,6 +40,14 @@ public sealed partial class FfmpegService hasHlsOverlongHeadersFailure && string.Equals(selectedProtocol, "hls", StringComparison.OrdinalIgnoreCase); + internal static bool ShouldFallbackToCurl( + string selectedProtocol, + bool hasOverlongHeadersFailure, + bool hasTriedCurlFallback) => + hasOverlongHeadersFailure && + !hasTriedCurlFallback && + string.Equals(selectedProtocol, "flv", StringComparison.OrdinalIgnoreCase); + internal static bool ShouldApplyRuntimeFailureBackoff( RecordSessionStatus status, TimeSpan processRuntime) => @@ -278,6 +289,8 @@ internal sealed record FfmpegRecoveryContext( bool HasRetriedWithCompatibilityProfile, bool HasRetriedWithRefreshedStream, bool HasRetriedWithAlternateProtocol, + bool UseCurlFallback, + bool HasTriedCurlFallback, bool ForceSoftwareEncoder, bool HasRetriedWithSoftwareEncoder); diff --git a/src/LiveRecorder.Infrastructure/Services/FfmpegService.Runtime.cs b/src/LiveRecorder.Infrastructure/Services/FfmpegService.Runtime.cs index f9a23d5..22ecac7 100644 --- a/src/LiveRecorder.Infrastructure/Services/FfmpegService.Runtime.cs +++ b/src/LiveRecorder.Infrastructure/Services/FfmpegService.Runtime.cs @@ -911,6 +911,52 @@ public sealed partial class FfmpegService return false; } + if (ShouldFallbackToCurl( + runtime.SelectedProtocol, + runtime.HasHlsOverlongHeadersFailure, + runtime.RecoveryContext.HasTriedCurlFallback)) + { + await logService.WriteAsync( + SystemLogLevel.Warning, + "FFmpeg", + "Native FLV input exceeded the FFmpeg response header limit; switching once to the curl compatibility fallback.", + runtime.LastStartupFailureLine, + session.LiveRoomId, + session.Id, + currentTask.Id); + + var retryStream = new StreamUrlResult( + runtime.SelectedQuality, + runtime.SelectedProtocol, + runtime.StreamUrl, + runtime.InputHeaders, + Array.Empty(), + runtime.SelectedVideoCodec); + var curlFallbackContext = runtime.RecoveryContext with + { + AttemptCount = runtime.RetryAttemptCount + 1, + UseCurlFallback = true, + HasTriedCurlFallback = true + }; + + session.MarkStarting(retryStream.SelectedUrl, session.OutputPathPattern ?? runtime.OutputPathPattern, observedAt); + session.ActivateSegment(Math.Max(1, runtime.CurrentSegmentIndex), observedAt); + currentTask.MarkStarting(retryStream.SelectedUrl, currentTask.OutputFilePath ?? runtime.CurrentOutputFilePath, observedAt); + await dbContext.SaveChangesAsync(); + + await StartInternalAsync( + session, + currentTask, + retryStream, + runtime.RecordingSettings, + curlFallbackContext); + + var restartedAt = DateTimeOffset.UtcNow; + session.MarkRunning(restartedAt); + currentTask.MarkRunning(restartedAt); + await dbContext.SaveChangesAsync(); + return true; + } if (ShouldImmediatelyFallbackFromHls(runtime.SelectedProtocol, runtime.HasHlsOverlongHeadersFailure)) { return await TryRetryWithAlternateProtocolAsync( @@ -1105,6 +1151,25 @@ public sealed partial class FfmpegService retryInputOptionProfile, runtime.SelectedProtocol, retryStream.SelectedProtocol); + if (ShouldFallbackToCurl( + runtime.SelectedProtocol, + runtime.HasHlsOverlongHeadersFailure, + recoveryBase.HasTriedCurlFallback)) + { + recoveryContext = recoveryContext with + { + UseCurlFallback = true, + HasTriedCurlFallback = true + }; + await logService.WriteAsync( + SystemLogLevel.Warning, + "FFmpeg", + "Native FLV input exceeded the FFmpeg response header limit; retrying with the curl compatibility fallback.", + runtime.GetRecentOutputSummary(), + session.LiveRoomId, + session.Id, + retryTask.Id); + } var retryAttempt = recoveryContext.AttemptCount; try { @@ -1207,8 +1272,8 @@ public sealed partial class FfmpegService await logService.WriteAsync( SystemLogLevel.Warning, "FFmpeg", - $"ffmpeg exited unexpectedly. Retrying within the current session (attempt {retryAttempt}).", - $"transition={runtime.InputOptionProfile}/{runtime.SelectedProtocol} -> {retryInputOptionProfile}/{retryStream.SelectedProtocol}; output={runtime.GetRecentOutputSummary()}", + $"ffmpeg exited unexpectedly. Refreshed the stream URL and retrying within the current session (attempt {retryAttempt}).", + $"transition={runtime.InputOptionProfile}/{runtime.SelectedProtocol}/{(runtime.RecoveryContext.UseCurlFallback ? "curl" : "native-http")} -> {retryInputOptionProfile}/{retryStream.SelectedProtocol}/{(recoveryContext.UseCurlFallback ? "curl" : "native-http")}; output={runtime.GetRecentOutputSummary()}", session.LiveRoomId, session.Id, retryTask.Id); diff --git a/src/LiveRecorder.Infrastructure/Services/FfmpegService.Utils.cs b/src/LiveRecorder.Infrastructure/Services/FfmpegService.Utils.cs index 4260ccd..6e88ce3 100644 --- a/src/LiveRecorder.Infrastructure/Services/FfmpegService.Utils.cs +++ b/src/LiveRecorder.Infrastructure/Services/FfmpegService.Utils.cs @@ -848,14 +848,15 @@ public sealed partial class FfmpegService string? selectedProtocol, string? selectedVideoCodec, FfmpegInputOptionProfile inputOptionProfile, - RecoveryVideoEncoderSelection? recoveryVideoEncoder = null) + RecoveryVideoEncoderSelection? recoveryVideoEncoder = null, + bool useCurlFallback = false) { var arguments = new List { "-hide_banner", "-n", "-progress", "pipe:1" }; recoveryVideoEncoder ??= RecoveryVideoEncoderSelection.Software; var useIntermediateTransportStream = ShouldUseIntermediateTransportStream(outputFilePath, outputFormat, saveMode); var effectiveStreamUrl = streamUrl; - var isPipeInput = ShouldUseCurlPipe(streamUrl, selectedProtocol); + var isPipeInput = ShouldUseCurlPipe(streamUrl, selectedProtocol, useCurlFallback); if (isPipeInput) { // When the input is HTTP, use pipe:0 so that curl handles the HTTP connection. @@ -1333,8 +1334,11 @@ public sealed partial class FfmpegService streamUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || streamUrl.StartsWith("https://", StringComparison.OrdinalIgnoreCase); - internal static bool ShouldUseCurlPipe(string streamUrl, string? selectedProtocol) => - IsHttpInput(streamUrl) && !IsHlsInput(streamUrl, selectedProtocol); + internal static bool ShouldUseCurlPipe( + string streamUrl, + string? selectedProtocol, + bool useCurlFallback = false) => + useCurlFallback && IsHttpInput(streamUrl) && !IsHlsInput(streamUrl, selectedProtocol); private static bool IsHlsInput(string streamUrl, string? selectedProtocol) => string.Equals(selectedProtocol, "hls", StringComparison.OrdinalIgnoreCase) || @@ -1424,7 +1428,17 @@ public sealed partial class FfmpegService string streamUrl, StreamInputHeaders? inputHeaders) { - var args = new List { "-sL" }; + var args = new List + { + "--silent", + "--show-error", + "--location", + "--fail", + "--retry", "3", + "--retry-all-errors", + "--retry-delay", "2", + "--retry-max-time", "20" + }; if (!string.IsNullOrWhiteSpace(inputHeaders?.UserAgent)) { diff --git a/src/LiveRecorder.Infrastructure/Services/FfmpegService.cs b/src/LiveRecorder.Infrastructure/Services/FfmpegService.cs index 64e4a82..086927c 100644 --- a/src/LiveRecorder.Infrastructure/Services/FfmpegService.cs +++ b/src/LiveRecorder.Infrastructure/Services/FfmpegService.cs @@ -304,7 +304,8 @@ public sealed partial class FfmpegService : IFfmpegService streamUrlResult.SelectedProtocol, streamUrlResult.SelectedVideoCodec, recoveryContext.InputOptionProfile, - recoveryVideoEncoder)) + recoveryVideoEncoder, + recoveryContext.UseCurlFallback)) { process.StartInfo.ArgumentList.Add(argument); } @@ -326,11 +327,13 @@ public sealed partial class FfmpegService : IFfmpegService throw new InvalidOperationException("A running ffmpeg process already exists for the recording session."); } - // When the stream URL is HTTP, launch curl to handle the HTTP connection and - // pipe its stdout into FFmpeg's stdin. This bypasses FFmpeg's built-in HTTP - // handler which has a hard-coded 4096-byte response header buffer that triggers - // "overlong headers" errors with CDNs that return oversized headers. - if (ShouldUseCurlPipe(streamUrlResult.SelectedUrl, streamUrlResult.SelectedProtocol)) + // curl is a one-shot compatibility fallback for native FLV inputs whose response + // headers exceed FFmpeg's built-in HTTP header limit. Normal FLV and all HLS + // inputs stay on FFmpeg's native HTTP stack so native reconnect remains active. + if (ShouldUseCurlPipe( + streamUrlResult.SelectedUrl, + streamUrlResult.SelectedProtocol, + recoveryContext.UseCurlFallback)) { var curlProcess = new Process { @@ -350,6 +353,14 @@ public sealed partial class FfmpegService : IFfmpegService curlProcess.StartInfo.ArgumentList.Add(arg); } + curlProcess.Exited += (_, _) => + { + _logger.LogInformation( + "curl fallback connection ended for session {RecordSessionId} with exit code {ExitCode}.", + recordSession.Id, + runtime.CurlExitCode?.ToString() ?? "unknown"); + }; + curlProcess.ErrorDataReceived += (_, args) => { if (!string.IsNullOrWhiteSpace(args.Data)) @@ -1451,7 +1462,7 @@ public sealed partial class FfmpegService : IFfmpegService SystemLogLevel.Info, "FFmpeg", $"ffmpeg input profile={runtime.InputOptionProfile}.", - detail: $"stream={runtime.SelectedProtocol}:{runtime.SelectedQuality}; videoCodec={runtime.SelectedVideoCodec ?? "unknown"}; recoveryEncoder={runtime.RecoveryVideoEncoder.Kind}; recoveryDevice={runtime.RecoveryVideoEncoder.DevicePath ?? "default"}; attempt={runtime.RetryAttemptCount}; compatibilityRetry={runtime.HasRetriedWithCompatibilityProfile}; refreshRetry={runtime.HasRetriedWithRefreshedStream}; alternateProtocolRetry={runtime.HasRetriedWithAlternateProtocol}; softwareEncoderRetry={runtime.RecoveryContext.HasRetriedWithSoftwareEncoder}", + detail: $"stream={runtime.SelectedProtocol}:{runtime.SelectedQuality}; inputTransport={(runtime.RecoveryContext.UseCurlFallback ? "curl-fallback" : "native-http")}; nativeReconnect={(runtime.RecordingSettings.EnableAutoReconnect && !runtime.RecoveryContext.UseCurlFallback ? "enabled" : "disabled")}; videoCodec={runtime.SelectedVideoCodec ?? "unknown"}; recoveryEncoder={runtime.RecoveryVideoEncoder.Kind}; recoveryDevice={runtime.RecoveryVideoEncoder.DevicePath ?? "default"}; attempt={runtime.RetryAttemptCount}; compatibilityRetry={runtime.HasRetriedWithCompatibilityProfile}; refreshRetry={runtime.HasRetriedWithRefreshedStream}; alternateProtocolRetry={runtime.HasRetriedWithAlternateProtocol}; curlFallbackTried={runtime.RecoveryContext.HasTriedCurlFallback}; softwareEncoderRetry={runtime.RecoveryContext.HasRetriedWithSoftwareEncoder}", liveRoomId: runtime.LiveRoomId, recordSessionId: runtime.RecordSessionId, recordTaskId: runtime.CurrentTaskId, diff --git a/tests/LiveRecorder.Tests/FfmpegFailureClassificationTests.cs b/tests/LiveRecorder.Tests/FfmpegFailureClassificationTests.cs index e94bfec..ca1c311 100644 --- a/tests/LiveRecorder.Tests/FfmpegFailureClassificationTests.cs +++ b/tests/LiveRecorder.Tests/FfmpegFailureClassificationTests.cs @@ -114,12 +114,100 @@ public sealed class FfmpegFailureClassificationTests } [Theory] - [InlineData("https://example.test/live.flv", "flv", true)] - [InlineData("https://example.test/live.m3u8", "hls", false)] - [InlineData("https://example.test/playlist", "hls", false)] - public void HttpInput_UsesCurlExceptForHls(string url, string protocol, bool expected) + [InlineData("https://example.test/live.flv", "flv", false, false)] + [InlineData("https://example.test/live.flv", "flv", true, true)] + [InlineData("https://example.test/live.m3u8", "hls", true, false)] + [InlineData("https://example.test/playlist", "hls", true, false)] + public void CurlPipe_IsOnlyUsedForExplicitFlvFallback( + string url, + string protocol, + bool useCurlFallback, + bool expected) { - Assert.Equal(expected, FfmpegService.ShouldUseCurlPipe(url, protocol)); + Assert.Equal(expected, FfmpegService.ShouldUseCurlPipe(url, protocol, useCurlFallback)); + } + + [Fact] + public void NativeFlvInput_UsesFfmpegReconnectByDefault() + { + var arguments = FfmpegService.BuildArgumentList( + "https://cdn.example.test/live.flv", + "/tmp/output.ts", + RecordOutputFormat.Ts, + RecordSaveMode.SingleFile, + 1, + RecordingTemplateType.StreamCopy, + true, + 10, + 30_000_000, + 30, + null, + "flv", + "h264", + FfmpegService.FfmpegInputOptionProfile.Baseline); + + Assert.Contains("https://cdn.example.test/live.flv", arguments); + Assert.DoesNotContain("pipe:0", arguments); + Assert.Contains("-reconnect", arguments); + Assert.Contains("-reconnect_streamed", arguments); + Assert.Contains("-reconnect_at_eof", arguments); + } + + [Theory] + [InlineData("flv", true, false, true)] + [InlineData("flv", true, true, false)] + [InlineData("flv", false, false, false)] + [InlineData("hls", true, false, false)] + public void OverlongHeaders_UsesCurlFallbackAtMostOnce( + string protocol, + bool hasOverlongHeadersFailure, + bool hasTriedCurlFallback, + bool expected) + { + Assert.Equal( + expected, + FfmpegService.ShouldFallbackToCurl(protocol, hasOverlongHeadersFailure, hasTriedCurlFallback)); + } + + [Fact] + public void CurlFallback_UsesPipeWithFiniteRetriesAndPreservesHeaders() + { + var headers = new StreamInputHeaders( + "RecorderTest/1.0", + "https://live.example.test/room", + "session=abc", + new Dictionary { ["X-Test"] = "yes" }); + var ffmpegArguments = FfmpegService.BuildArgumentList( + "https://cdn.example.test/live.flv", + "/tmp/output.ts", + RecordOutputFormat.Ts, + RecordSaveMode.SingleFile, + 1, + RecordingTemplateType.StreamCopy, + true, + 10, + 30_000_000, + 30, + headers, + "flv", + "h264", + FfmpegService.FfmpegInputOptionProfile.Baseline, + useCurlFallback: true); + var curlArguments = FfmpegService.BuildCurlArgumentList( + "https://cdn.example.test/live.flv", + headers); + + Assert.Contains("pipe:0", ffmpegArguments); + Assert.DoesNotContain("-reconnect", ffmpegArguments); + Assert.Contains("--show-error", curlArguments); + Assert.Contains("--fail", curlArguments); + Assert.Contains("--retry", curlArguments); + Assert.Contains("--retry-all-errors", curlArguments); + Assert.Contains("--retry-max-time", curlArguments); + Assert.Contains("RecorderTest/1.0", curlArguments); + Assert.Contains("https://live.example.test/room", curlArguments); + Assert.Contains("session=abc", curlArguments); + Assert.Contains("X-Test: yes", curlArguments); } [Fact] @@ -145,7 +233,8 @@ public sealed class FfmpegFailureClassificationTests headers, "hls", "h264", - FfmpegService.FfmpegInputOptionProfile.Baseline); + FfmpegService.FfmpegInputOptionProfile.Baseline, + useCurlFallback: true); Assert.Contains("https://cdn.example.test/live/index.m3u8", arguments); Assert.DoesNotContain("pipe:0", arguments); @@ -182,6 +271,27 @@ public sealed class FfmpegFailureClassificationTests Assert.Contains("-reset_timestamps", arguments); } + [Fact] + public void RecoveryContext_LeavesCurlAfterExitWithoutResettingOneShotBudget() + { + var curlContext = FfmpegService.InitialRecoveryContext with + { + UseCurlFallback = true, + HasTriedCurlFallback = true + }; + + var nextContext = FfmpegService.AdvanceRecoveryContext( + curlContext, + curlContext.InputOptionProfile, + "flv", + "flv", + refreshedStream: true); + + Assert.False(nextContext.UseCurlFallback); + Assert.True(nextContext.HasTriedCurlFallback); + Assert.True(nextContext.HasRetriedWithRefreshedStream); + Assert.False(FfmpegService.ShouldFallbackToCurl("flv", true, nextContext.HasTriedCurlFallback)); + } [Fact] public void RecoveryContext_PreservesBudgetAcrossHlsFallbackAndReachesTimestampTranscode() {