fix: add max_alloc to prevent overlong headers error and throttle startup failure notifications
- Add -max_alloc 100000000 to FFmpeg arguments for HTTP inputs to avoid 'overlong headers' error when CDN (e.g. Douyin) returns oversized HTTP response headers exceeding FFmpeg's default 4096-byte buffer. - Add exponential backoff for repeated startup failures (30s → 15min cap) to break the tight fail→retry→re-poll loop that floods notifications. - Throttle startup failure notifications to at most one per 30 minutes per room to prevent email/webhook storms during persistent failures. - Reset backoff counter when a session successfully opens its first segment.
This commit is contained in:
@@ -324,6 +324,7 @@ public sealed partial class FfmpegService
|
||||
if (runtime.SaveMode == RecordSaveMode.SingleFile)
|
||||
{
|
||||
runtime.HasOpenedFirstSegment = true;
|
||||
ResetStartupFailureBackoff(runtime.LiveRoomId);
|
||||
currentTask.AttachProcess(runtime.ProcessId, now);
|
||||
currentTask.MarkRunning(now);
|
||||
session.AttachProcess(runtime.ProcessId, now);
|
||||
@@ -361,6 +362,7 @@ public sealed partial class FfmpegService
|
||||
if (segmentIndex == runtime.CurrentSegmentIndex)
|
||||
{
|
||||
runtime.HasOpenedFirstSegment = true;
|
||||
ResetStartupFailureBackoff(runtime.LiveRoomId);
|
||||
var finalOutputPath = NormalizeAbsolutePath(
|
||||
currentTask.OutputFilePath ??
|
||||
ResolveSegmentOutputPath(session.OutputPathPattern ?? runtime.OutputPathPattern, session.SaveMode, segmentIndex));
|
||||
@@ -435,6 +437,7 @@ public sealed partial class FfmpegService
|
||||
runtime.CurrentOutputFilePath = normalizedOpenedPath;
|
||||
runtime.ResetCurrentRecorderSegmentPaths(normalizedOpenedPath);
|
||||
runtime.HasOpenedFirstSegment = true;
|
||||
ResetStartupFailureBackoff(runtime.LiveRoomId);
|
||||
|
||||
if (runtime.DanmakuRecorder is not null)
|
||||
{
|
||||
@@ -1213,10 +1216,23 @@ public sealed partial class FfmpegService
|
||||
|
||||
if (!runtime.StopRequested && session.LiveRoomId != Guid.Empty)
|
||||
{
|
||||
_liveRoomPollingSignal.RequestImmediatePoll(session.LiveRoomId, TimeSpan.FromSeconds(2));
|
||||
var isStartupFailure = !runtime.HasOpenedFirstSegment &&
|
||||
runtime.StartupFailureKind != StartupFailureKind.None;
|
||||
var pollDelay = isStartupFailure
|
||||
? RecordStartupFailure(session.LiveRoomId)
|
||||
: TimeSpan.FromSeconds(2);
|
||||
_liveRoomPollingSignal.RequestImmediatePoll(session.LiveRoomId, pollDelay);
|
||||
}
|
||||
|
||||
if (session.Status == RecordSessionStatus.Failed && session.LiveRoom is not null)
|
||||
{
|
||||
var isStartupFailure = !runtime.HasOpenedFirstSegment &&
|
||||
runtime.StartupFailureKind != StartupFailureKind.None;
|
||||
if (isStartupFailure && ShouldThrottleStartupFailureNotification(session.LiveRoomId))
|
||||
{
|
||||
// Notification suppressed — the failure has already been logged above.
|
||||
}
|
||||
else
|
||||
{
|
||||
var failureDetail = BuildExitLogDetail(runtime, process.ExitCode, effectiveOutputPath, toleratedNonZeroExit: false, finalizationError);
|
||||
await emailNotificationService.SendExceptionAsync(
|
||||
@@ -1233,6 +1249,7 @@ public sealed partial class FfmpegService
|
||||
currentTask);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static string BuildExitLogDetail(
|
||||
SessionProcessRuntime runtime,
|
||||
|
||||
@@ -676,6 +676,11 @@ public sealed partial class FfmpegService
|
||||
{
|
||||
arguments.AddRange(["-headers", customHeaders]);
|
||||
}
|
||||
|
||||
// Increase the max allocation size to avoid "overlong headers" errors
|
||||
// when CDN (e.g. Douyin) returns HTTP response headers exceeding FFmpeg's
|
||||
// default internal buffer (4096 bytes).
|
||||
arguments.AddRange(["-max_alloc", "100000000"]);
|
||||
}
|
||||
|
||||
if (enableReconnect &&
|
||||
|
||||
@@ -24,8 +24,13 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
"""Opening '([^']+)' for writing""",
|
||||
RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
|
||||
private static readonly TimeSpan StartupFailureNotificationCooldown = TimeSpan.FromMinutes(30);
|
||||
private static readonly TimeSpan StartupFailureMaxBackoff = TimeSpan.FromMinutes(15);
|
||||
private static readonly TimeSpan StartupFailureBaseBackoff = TimeSpan.FromSeconds(30);
|
||||
|
||||
private readonly ConcurrentDictionary<Guid, SessionProcessRuntime> _processes = new();
|
||||
private readonly ConcurrentDictionary<Guid, PostProcessRuntimeEntry> _postProcessStates = new();
|
||||
private readonly ConcurrentDictionary<Guid, RoomStartupFailureState> _roomStartupFailureStates = new();
|
||||
private readonly object _transcodeConcurrencyLock = new();
|
||||
private readonly Queue<TaskCompletionSource<IDisposable>> _transcodeWaiters = new();
|
||||
private int _activeTranscodeTasks;
|
||||
@@ -795,6 +800,84 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records a startup failure for the given live room and returns the backoff delay
|
||||
/// that should be applied before the next re-poll attempt. Uses exponential backoff
|
||||
/// to prevent tight fail→retry→fail→re-poll loops from flooding notifications.
|
||||
/// </summary>
|
||||
private TimeSpan RecordStartupFailure(Guid liveRoomId)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var state = _roomStartupFailureStates.AddOrUpdate(
|
||||
liveRoomId,
|
||||
_ => new RoomStartupFailureState { ConsecutiveFailures = 1, FirstFailureAt = now, LastFailureAt = now },
|
||||
(_, existing) =>
|
||||
{
|
||||
existing.ConsecutiveFailures++;
|
||||
existing.LastFailureAt = now;
|
||||
return existing;
|
||||
});
|
||||
|
||||
var backoffSeconds = StartupFailureBaseBackoff.TotalSeconds * Math.Pow(2, Math.Min(state.ConsecutiveFailures - 1, 5));
|
||||
var backoff = TimeSpan.FromSeconds(Math.Min(backoffSeconds, StartupFailureMaxBackoff.TotalSeconds));
|
||||
_logger.LogWarning(
|
||||
"Startup failure backoff for room {LiveRoomId}: {ConsecutiveFailures} consecutive failures, next poll delayed by {BackoffSeconds:F0}s",
|
||||
liveRoomId, state.ConsecutiveFailures, backoff.TotalSeconds);
|
||||
return backoff;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resets the startup failure backoff counter when a session successfully opens its first segment.
|
||||
/// </summary>
|
||||
private void ResetStartupFailureBackoff(Guid liveRoomId)
|
||||
{
|
||||
if (_roomStartupFailureStates.TryRemove(liveRoomId, out var state) && state.ConsecutiveFailures > 1)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Startup failure backoff reset for room {LiveRoomId} after {ConsecutiveFailures} failures",
|
||||
liveRoomId, state.ConsecutiveFailures);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns true if the failure notification for this room should be throttled
|
||||
/// (i.e., at most one notification per <see cref="StartupFailureNotificationCooldown"/>).
|
||||
/// </summary>
|
||||
private bool ShouldThrottleStartupFailureNotification(Guid liveRoomId)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var state = _roomStartupFailureStates.GetOrAdd(
|
||||
liveRoomId,
|
||||
_ => new RoomStartupFailureState { ConsecutiveFailures = 0, FirstFailureAt = now, LastFailureAt = now });
|
||||
|
||||
// Always allow the first failure notification through
|
||||
if (state.LastNotificationAt is null)
|
||||
{
|
||||
state.LastNotificationAt = now;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (now - state.LastNotificationAt.Value < StartupFailureNotificationCooldown)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Suppressed duplicate startup failure notification for room {LiveRoomId}. " +
|
||||
"Last notification was at {LastNotificationAt} (cooldown={CooldownMinutes}min)",
|
||||
liveRoomId, state.LastNotificationAt, StartupFailureNotificationCooldown.TotalMinutes);
|
||||
return true;
|
||||
}
|
||||
|
||||
state.LastNotificationAt = now;
|
||||
return false;
|
||||
}
|
||||
|
||||
private sealed class RoomStartupFailureState
|
||||
{
|
||||
public int ConsecutiveFailures;
|
||||
public DateTimeOffset FirstFailureAt;
|
||||
public DateTimeOffset LastFailureAt;
|
||||
public DateTimeOffset? LastNotificationAt;
|
||||
}
|
||||
|
||||
private sealed record PostProcessRuntimeEntry(Guid RecordSessionId, RecordTaskRuntimeState State);
|
||||
|
||||
private sealed class TranscodeSlotLease : IDisposable
|
||||
|
||||
Reference in New Issue
Block a user