fix: restore native FLV reconnect with curl fallback

This commit is contained in:
2026-08-12 12:14:43 +08:00
parent e6068d93e0
commit e0bd001541
5 changed files with 233 additions and 20 deletions
@@ -13,6 +13,8 @@ public sealed partial class FfmpegService
HasRetriedWithCompatibilityProfile: false, HasRetriedWithCompatibilityProfile: false,
HasRetriedWithRefreshedStream: false, HasRetriedWithRefreshedStream: false,
HasRetriedWithAlternateProtocol: false, HasRetriedWithAlternateProtocol: false,
UseCurlFallback: false,
HasTriedCurlFallback: false,
ForceSoftwareEncoder: false, ForceSoftwareEncoder: false,
HasRetriedWithSoftwareEncoder: false); HasRetriedWithSoftwareEncoder: false);
@@ -25,6 +27,7 @@ public sealed partial class FfmpegService
current with current with
{ {
InputOptionProfile = inputOptionProfile, InputOptionProfile = inputOptionProfile,
UseCurlFallback = false,
AttemptCount = current.AttemptCount + 1, AttemptCount = current.AttemptCount + 1,
HasRetriedWithRefreshedStream = current.HasRetriedWithRefreshedStream || refreshedStream, HasRetriedWithRefreshedStream = current.HasRetriedWithRefreshedStream || refreshedStream,
HasRetriedWithAlternateProtocol = current.HasRetriedWithAlternateProtocol || HasRetriedWithAlternateProtocol = current.HasRetriedWithAlternateProtocol ||
@@ -37,6 +40,14 @@ public sealed partial class FfmpegService
hasHlsOverlongHeadersFailure && hasHlsOverlongHeadersFailure &&
string.Equals(selectedProtocol, "hls", StringComparison.OrdinalIgnoreCase); string.Equals(selectedProtocol, "hls", StringComparison.OrdinalIgnoreCase);
internal static bool ShouldFallbackToCurl(
string selectedProtocol,
bool hasOverlongHeadersFailure,
bool hasTriedCurlFallback) =>
hasOverlongHeadersFailure &&
!hasTriedCurlFallback &&
string.Equals(selectedProtocol, "flv", StringComparison.OrdinalIgnoreCase);
internal static bool ShouldApplyRuntimeFailureBackoff( internal static bool ShouldApplyRuntimeFailureBackoff(
RecordSessionStatus status, RecordSessionStatus status,
TimeSpan processRuntime) => TimeSpan processRuntime) =>
@@ -278,6 +289,8 @@ internal sealed record FfmpegRecoveryContext(
bool HasRetriedWithCompatibilityProfile, bool HasRetriedWithCompatibilityProfile,
bool HasRetriedWithRefreshedStream, bool HasRetriedWithRefreshedStream,
bool HasRetriedWithAlternateProtocol, bool HasRetriedWithAlternateProtocol,
bool UseCurlFallback,
bool HasTriedCurlFallback,
bool ForceSoftwareEncoder, bool ForceSoftwareEncoder,
bool HasRetriedWithSoftwareEncoder); bool HasRetriedWithSoftwareEncoder);
@@ -911,6 +911,52 @@ public sealed partial class FfmpegService
return false; return false;
} }
if (ShouldFallbackToCurl(
runtime.SelectedProtocol,
runtime.HasHlsOverlongHeadersFailure,
runtime.RecoveryContext.HasTriedCurlFallback))
{
await logService.WriteAsync(
SystemLogLevel.Warning,
"FFmpeg",
"Native FLV input exceeded the FFmpeg response header limit; switching once to the curl compatibility fallback.",
runtime.LastStartupFailureLine,
session.LiveRoomId,
session.Id,
currentTask.Id);
var retryStream = new StreamUrlResult(
runtime.SelectedQuality,
runtime.SelectedProtocol,
runtime.StreamUrl,
runtime.InputHeaders,
Array.Empty<StreamQualityOption>(),
runtime.SelectedVideoCodec);
var curlFallbackContext = runtime.RecoveryContext with
{
AttemptCount = runtime.RetryAttemptCount + 1,
UseCurlFallback = true,
HasTriedCurlFallback = true
};
session.MarkStarting(retryStream.SelectedUrl, session.OutputPathPattern ?? runtime.OutputPathPattern, observedAt);
session.ActivateSegment(Math.Max(1, runtime.CurrentSegmentIndex), observedAt);
currentTask.MarkStarting(retryStream.SelectedUrl, currentTask.OutputFilePath ?? runtime.CurrentOutputFilePath, observedAt);
await dbContext.SaveChangesAsync();
await StartInternalAsync(
session,
currentTask,
retryStream,
runtime.RecordingSettings,
curlFallbackContext);
var restartedAt = DateTimeOffset.UtcNow;
session.MarkRunning(restartedAt);
currentTask.MarkRunning(restartedAt);
await dbContext.SaveChangesAsync();
return true;
}
if (ShouldImmediatelyFallbackFromHls(runtime.SelectedProtocol, runtime.HasHlsOverlongHeadersFailure)) if (ShouldImmediatelyFallbackFromHls(runtime.SelectedProtocol, runtime.HasHlsOverlongHeadersFailure))
{ {
return await TryRetryWithAlternateProtocolAsync( return await TryRetryWithAlternateProtocolAsync(
@@ -1105,6 +1151,25 @@ public sealed partial class FfmpegService
retryInputOptionProfile, retryInputOptionProfile,
runtime.SelectedProtocol, runtime.SelectedProtocol,
retryStream.SelectedProtocol); retryStream.SelectedProtocol);
if (ShouldFallbackToCurl(
runtime.SelectedProtocol,
runtime.HasHlsOverlongHeadersFailure,
recoveryBase.HasTriedCurlFallback))
{
recoveryContext = recoveryContext with
{
UseCurlFallback = true,
HasTriedCurlFallback = true
};
await logService.WriteAsync(
SystemLogLevel.Warning,
"FFmpeg",
"Native FLV input exceeded the FFmpeg response header limit; retrying with the curl compatibility fallback.",
runtime.GetRecentOutputSummary(),
session.LiveRoomId,
session.Id,
retryTask.Id);
}
var retryAttempt = recoveryContext.AttemptCount; var retryAttempt = recoveryContext.AttemptCount;
try try
{ {
@@ -1207,8 +1272,8 @@ public sealed partial class FfmpegService
await logService.WriteAsync( await logService.WriteAsync(
SystemLogLevel.Warning, SystemLogLevel.Warning,
"FFmpeg", "FFmpeg",
$"ffmpeg exited unexpectedly. Retrying within the current session (attempt {retryAttempt}).", $"ffmpeg exited unexpectedly. Refreshed the stream URL and retrying within the current session (attempt {retryAttempt}).",
$"transition={runtime.InputOptionProfile}/{runtime.SelectedProtocol} -> {retryInputOptionProfile}/{retryStream.SelectedProtocol}; output={runtime.GetRecentOutputSummary()}", $"transition={runtime.InputOptionProfile}/{runtime.SelectedProtocol}/{(runtime.RecoveryContext.UseCurlFallback ? "curl" : "native-http")} -> {retryInputOptionProfile}/{retryStream.SelectedProtocol}/{(recoveryContext.UseCurlFallback ? "curl" : "native-http")}; output={runtime.GetRecentOutputSummary()}",
session.LiveRoomId, session.LiveRoomId,
session.Id, session.Id,
retryTask.Id); retryTask.Id);
@@ -848,14 +848,15 @@ public sealed partial class FfmpegService
string? selectedProtocol, string? selectedProtocol,
string? selectedVideoCodec, string? selectedVideoCodec,
FfmpegInputOptionProfile inputOptionProfile, FfmpegInputOptionProfile inputOptionProfile,
RecoveryVideoEncoderSelection? recoveryVideoEncoder = null) RecoveryVideoEncoderSelection? recoveryVideoEncoder = null,
bool useCurlFallback = false)
{ {
var arguments = new List<string> { "-hide_banner", "-n", "-progress", "pipe:1" }; var arguments = new List<string> { "-hide_banner", "-n", "-progress", "pipe:1" };
recoveryVideoEncoder ??= RecoveryVideoEncoderSelection.Software; recoveryVideoEncoder ??= RecoveryVideoEncoderSelection.Software;
var useIntermediateTransportStream = ShouldUseIntermediateTransportStream(outputFilePath, outputFormat, saveMode); var useIntermediateTransportStream = ShouldUseIntermediateTransportStream(outputFilePath, outputFormat, saveMode);
var effectiveStreamUrl = streamUrl; var effectiveStreamUrl = streamUrl;
var isPipeInput = ShouldUseCurlPipe(streamUrl, selectedProtocol); var isPipeInput = ShouldUseCurlPipe(streamUrl, selectedProtocol, useCurlFallback);
if (isPipeInput) if (isPipeInput)
{ {
// When the input is HTTP, use pipe:0 so that curl handles the HTTP connection. // When the input is HTTP, use pipe:0 so that curl handles the HTTP connection.
@@ -1333,8 +1334,11 @@ public sealed partial class FfmpegService
streamUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || streamUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ||
streamUrl.StartsWith("https://", StringComparison.OrdinalIgnoreCase); streamUrl.StartsWith("https://", StringComparison.OrdinalIgnoreCase);
internal static bool ShouldUseCurlPipe(string streamUrl, string? selectedProtocol) => internal static bool ShouldUseCurlPipe(
IsHttpInput(streamUrl) && !IsHlsInput(streamUrl, selectedProtocol); string streamUrl,
string? selectedProtocol,
bool useCurlFallback = false) =>
useCurlFallback && IsHttpInput(streamUrl) && !IsHlsInput(streamUrl, selectedProtocol);
private static bool IsHlsInput(string streamUrl, string? selectedProtocol) => private static bool IsHlsInput(string streamUrl, string? selectedProtocol) =>
string.Equals(selectedProtocol, "hls", StringComparison.OrdinalIgnoreCase) || string.Equals(selectedProtocol, "hls", StringComparison.OrdinalIgnoreCase) ||
@@ -1424,7 +1428,17 @@ public sealed partial class FfmpegService
string streamUrl, string streamUrl,
StreamInputHeaders? inputHeaders) StreamInputHeaders? inputHeaders)
{ {
var args = new List<string> { "-sL" }; var args = new List<string>
{
"--silent",
"--show-error",
"--location",
"--fail",
"--retry", "3",
"--retry-all-errors",
"--retry-delay", "2",
"--retry-max-time", "20"
};
if (!string.IsNullOrWhiteSpace(inputHeaders?.UserAgent)) if (!string.IsNullOrWhiteSpace(inputHeaders?.UserAgent))
{ {
@@ -304,7 +304,8 @@ public sealed partial class FfmpegService : IFfmpegService
streamUrlResult.SelectedProtocol, streamUrlResult.SelectedProtocol,
streamUrlResult.SelectedVideoCodec, streamUrlResult.SelectedVideoCodec,
recoveryContext.InputOptionProfile, recoveryContext.InputOptionProfile,
recoveryVideoEncoder)) recoveryVideoEncoder,
recoveryContext.UseCurlFallback))
{ {
process.StartInfo.ArgumentList.Add(argument); process.StartInfo.ArgumentList.Add(argument);
} }
@@ -326,11 +327,13 @@ public sealed partial class FfmpegService : IFfmpegService
throw new InvalidOperationException("A running ffmpeg process already exists for the recording session."); throw new InvalidOperationException("A running ffmpeg process already exists for the recording session.");
} }
// When the stream URL is HTTP, launch curl to handle the HTTP connection and // curl is a one-shot compatibility fallback for native FLV inputs whose response
// pipe its stdout into FFmpeg's stdin. This bypasses FFmpeg's built-in HTTP // headers exceed FFmpeg's built-in HTTP header limit. Normal FLV and all HLS
// handler which has a hard-coded 4096-byte response header buffer that triggers // inputs stay on FFmpeg's native HTTP stack so native reconnect remains active.
// "overlong headers" errors with CDNs that return oversized headers. if (ShouldUseCurlPipe(
if (ShouldUseCurlPipe(streamUrlResult.SelectedUrl, streamUrlResult.SelectedProtocol)) streamUrlResult.SelectedUrl,
streamUrlResult.SelectedProtocol,
recoveryContext.UseCurlFallback))
{ {
var curlProcess = new Process var curlProcess = new Process
{ {
@@ -350,6 +353,14 @@ public sealed partial class FfmpegService : IFfmpegService
curlProcess.StartInfo.ArgumentList.Add(arg); curlProcess.StartInfo.ArgumentList.Add(arg);
} }
curlProcess.Exited += (_, _) =>
{
_logger.LogInformation(
"curl fallback connection ended for session {RecordSessionId} with exit code {ExitCode}.",
recordSession.Id,
runtime.CurlExitCode?.ToString() ?? "unknown");
};
curlProcess.ErrorDataReceived += (_, args) => curlProcess.ErrorDataReceived += (_, args) =>
{ {
if (!string.IsNullOrWhiteSpace(args.Data)) if (!string.IsNullOrWhiteSpace(args.Data))
@@ -1451,7 +1462,7 @@ public sealed partial class FfmpegService : IFfmpegService
SystemLogLevel.Info, SystemLogLevel.Info,
"FFmpeg", "FFmpeg",
$"ffmpeg input profile={runtime.InputOptionProfile}.", $"ffmpeg input profile={runtime.InputOptionProfile}.",
detail: $"stream={runtime.SelectedProtocol}:{runtime.SelectedQuality}; videoCodec={runtime.SelectedVideoCodec ?? "unknown"}; recoveryEncoder={runtime.RecoveryVideoEncoder.Kind}; recoveryDevice={runtime.RecoveryVideoEncoder.DevicePath ?? "default"}; attempt={runtime.RetryAttemptCount}; compatibilityRetry={runtime.HasRetriedWithCompatibilityProfile}; refreshRetry={runtime.HasRetriedWithRefreshedStream}; alternateProtocolRetry={runtime.HasRetriedWithAlternateProtocol}; softwareEncoderRetry={runtime.RecoveryContext.HasRetriedWithSoftwareEncoder}", detail: $"stream={runtime.SelectedProtocol}:{runtime.SelectedQuality}; inputTransport={(runtime.RecoveryContext.UseCurlFallback ? "curl-fallback" : "native-http")}; nativeReconnect={(runtime.RecordingSettings.EnableAutoReconnect && !runtime.RecoveryContext.UseCurlFallback ? "enabled" : "disabled")}; videoCodec={runtime.SelectedVideoCodec ?? "unknown"}; recoveryEncoder={runtime.RecoveryVideoEncoder.Kind}; recoveryDevice={runtime.RecoveryVideoEncoder.DevicePath ?? "default"}; attempt={runtime.RetryAttemptCount}; compatibilityRetry={runtime.HasRetriedWithCompatibilityProfile}; refreshRetry={runtime.HasRetriedWithRefreshedStream}; alternateProtocolRetry={runtime.HasRetriedWithAlternateProtocol}; curlFallbackTried={runtime.RecoveryContext.HasTriedCurlFallback}; softwareEncoderRetry={runtime.RecoveryContext.HasRetriedWithSoftwareEncoder}",
liveRoomId: runtime.LiveRoomId, liveRoomId: runtime.LiveRoomId,
recordSessionId: runtime.RecordSessionId, recordSessionId: runtime.RecordSessionId,
recordTaskId: runtime.CurrentTaskId, recordTaskId: runtime.CurrentTaskId,
@@ -114,12 +114,100 @@ public sealed class FfmpegFailureClassificationTests
} }
[Theory] [Theory]
[InlineData("https://example.test/live.flv", "flv", true)] [InlineData("https://example.test/live.flv", "flv", false, false)]
[InlineData("https://example.test/live.m3u8", "hls", false)] [InlineData("https://example.test/live.flv", "flv", true, true)]
[InlineData("https://example.test/playlist", "hls", false)] [InlineData("https://example.test/live.m3u8", "hls", true, false)]
public void HttpInput_UsesCurlExceptForHls(string url, string protocol, bool expected) [InlineData("https://example.test/playlist", "hls", true, false)]
public void CurlPipe_IsOnlyUsedForExplicitFlvFallback(
string url,
string protocol,
bool useCurlFallback,
bool expected)
{ {
Assert.Equal(expected, FfmpegService.ShouldUseCurlPipe(url, protocol)); Assert.Equal(expected, FfmpegService.ShouldUseCurlPipe(url, protocol, useCurlFallback));
}
[Fact]
public void NativeFlvInput_UsesFfmpegReconnectByDefault()
{
var arguments = FfmpegService.BuildArgumentList(
"https://cdn.example.test/live.flv",
"/tmp/output.ts",
RecordOutputFormat.Ts,
RecordSaveMode.SingleFile,
1,
RecordingTemplateType.StreamCopy,
true,
10,
30_000_000,
30,
null,
"flv",
"h264",
FfmpegService.FfmpegInputOptionProfile.Baseline);
Assert.Contains("https://cdn.example.test/live.flv", arguments);
Assert.DoesNotContain("pipe:0", arguments);
Assert.Contains("-reconnect", arguments);
Assert.Contains("-reconnect_streamed", arguments);
Assert.Contains("-reconnect_at_eof", arguments);
}
[Theory]
[InlineData("flv", true, false, true)]
[InlineData("flv", true, true, false)]
[InlineData("flv", false, false, false)]
[InlineData("hls", true, false, false)]
public void OverlongHeaders_UsesCurlFallbackAtMostOnce(
string protocol,
bool hasOverlongHeadersFailure,
bool hasTriedCurlFallback,
bool expected)
{
Assert.Equal(
expected,
FfmpegService.ShouldFallbackToCurl(protocol, hasOverlongHeadersFailure, hasTriedCurlFallback));
}
[Fact]
public void CurlFallback_UsesPipeWithFiniteRetriesAndPreservesHeaders()
{
var headers = new StreamInputHeaders(
"RecorderTest/1.0",
"https://live.example.test/room",
"session=abc",
new Dictionary<string, string> { ["X-Test"] = "yes" });
var ffmpegArguments = FfmpegService.BuildArgumentList(
"https://cdn.example.test/live.flv",
"/tmp/output.ts",
RecordOutputFormat.Ts,
RecordSaveMode.SingleFile,
1,
RecordingTemplateType.StreamCopy,
true,
10,
30_000_000,
30,
headers,
"flv",
"h264",
FfmpegService.FfmpegInputOptionProfile.Baseline,
useCurlFallback: true);
var curlArguments = FfmpegService.BuildCurlArgumentList(
"https://cdn.example.test/live.flv",
headers);
Assert.Contains("pipe:0", ffmpegArguments);
Assert.DoesNotContain("-reconnect", ffmpegArguments);
Assert.Contains("--show-error", curlArguments);
Assert.Contains("--fail", curlArguments);
Assert.Contains("--retry", curlArguments);
Assert.Contains("--retry-all-errors", curlArguments);
Assert.Contains("--retry-max-time", curlArguments);
Assert.Contains("RecorderTest/1.0", curlArguments);
Assert.Contains("https://live.example.test/room", curlArguments);
Assert.Contains("session=abc", curlArguments);
Assert.Contains("X-Test: yes", curlArguments);
} }
[Fact] [Fact]
@@ -145,7 +233,8 @@ public sealed class FfmpegFailureClassificationTests
headers, headers,
"hls", "hls",
"h264", "h264",
FfmpegService.FfmpegInputOptionProfile.Baseline); FfmpegService.FfmpegInputOptionProfile.Baseline,
useCurlFallback: true);
Assert.Contains("https://cdn.example.test/live/index.m3u8", arguments); Assert.Contains("https://cdn.example.test/live/index.m3u8", arguments);
Assert.DoesNotContain("pipe:0", arguments); Assert.DoesNotContain("pipe:0", arguments);
@@ -182,6 +271,27 @@ public sealed class FfmpegFailureClassificationTests
Assert.Contains("-reset_timestamps", arguments); Assert.Contains("-reset_timestamps", arguments);
} }
[Fact]
public void RecoveryContext_LeavesCurlAfterExitWithoutResettingOneShotBudget()
{
var curlContext = FfmpegService.InitialRecoveryContext with
{
UseCurlFallback = true,
HasTriedCurlFallback = true
};
var nextContext = FfmpegService.AdvanceRecoveryContext(
curlContext,
curlContext.InputOptionProfile,
"flv",
"flv",
refreshedStream: true);
Assert.False(nextContext.UseCurlFallback);
Assert.True(nextContext.HasTriedCurlFallback);
Assert.True(nextContext.HasRetriedWithRefreshedStream);
Assert.False(FfmpegService.ShouldFallbackToCurl("flv", true, nextContext.HasTriedCurlFallback));
}
[Fact] [Fact]
public void RecoveryContext_PreservesBudgetAcrossHlsFallbackAndReachesTimestampTranscode() public void RecoveryContext_PreservesBudgetAcrossHlsFallbackAndReachesTimestampTranscode()
{ {