fix: add max_alloc to prevent overlong headers error and throttle startup failure notifications

- Add -max_alloc 100000000 to FFmpeg arguments for HTTP inputs to avoid
  'overlong headers' error when CDN (e.g. Douyin) returns oversized HTTP
  response headers exceeding FFmpeg's default 4096-byte buffer.

- Add exponential backoff for repeated startup failures (30s → 15min cap)
  to break the tight fail→retry→re-poll loop that floods notifications.

- Throttle startup failure notifications to at most one per 30 minutes per
  room to prevent email/webhook storms during persistent failures.

- Reset backoff counter when a session successfully opens its first segment.
This commit is contained in:
2026-07-09 11:51:52 +08:00
parent 48da49ac72
commit 8a079b4698
3 changed files with 119 additions and 14 deletions
@@ -324,6 +324,7 @@ public sealed partial class FfmpegService
if (runtime.SaveMode == RecordSaveMode.SingleFile)
{
runtime.HasOpenedFirstSegment = true;
ResetStartupFailureBackoff(runtime.LiveRoomId);
currentTask.AttachProcess(runtime.ProcessId, now);
currentTask.MarkRunning(now);
session.AttachProcess(runtime.ProcessId, now);
@@ -361,6 +362,7 @@ public sealed partial class FfmpegService
if (segmentIndex == runtime.CurrentSegmentIndex)
{
runtime.HasOpenedFirstSegment = true;
ResetStartupFailureBackoff(runtime.LiveRoomId);
var finalOutputPath = NormalizeAbsolutePath(
currentTask.OutputFilePath ??
ResolveSegmentOutputPath(session.OutputPathPattern ?? runtime.OutputPathPattern, session.SaveMode, segmentIndex));
@@ -435,6 +437,7 @@ public sealed partial class FfmpegService
runtime.CurrentOutputFilePath = normalizedOpenedPath;
runtime.ResetCurrentRecorderSegmentPaths(normalizedOpenedPath);
runtime.HasOpenedFirstSegment = true;
ResetStartupFailureBackoff(runtime.LiveRoomId);
if (runtime.DanmakuRecorder is not null)
{
@@ -1213,10 +1216,23 @@ public sealed partial class FfmpegService
if (!runtime.StopRequested && session.LiveRoomId != Guid.Empty)
{
_liveRoomPollingSignal.RequestImmediatePoll(session.LiveRoomId, TimeSpan.FromSeconds(2));
var isStartupFailure = !runtime.HasOpenedFirstSegment &&
runtime.StartupFailureKind != StartupFailureKind.None;
var pollDelay = isStartupFailure
? RecordStartupFailure(session.LiveRoomId)
: TimeSpan.FromSeconds(2);
_liveRoomPollingSignal.RequestImmediatePoll(session.LiveRoomId, pollDelay);
}
if (session.Status == RecordSessionStatus.Failed && session.LiveRoom is not null)
{
var isStartupFailure = !runtime.HasOpenedFirstSegment &&
runtime.StartupFailureKind != StartupFailureKind.None;
if (isStartupFailure && ShouldThrottleStartupFailureNotification(session.LiveRoomId))
{
// Notification suppressed — the failure has already been logged above.
}
else
{
var failureDetail = BuildExitLogDetail(runtime, process.ExitCode, effectiveOutputPath, toleratedNonZeroExit: false, finalizationError);
await emailNotificationService.SendExceptionAsync(
@@ -1233,6 +1249,7 @@ public sealed partial class FfmpegService
currentTask);
}
}
}
private static string BuildExitLogDetail(
SessionProcessRuntime runtime,
@@ -676,6 +676,11 @@ public sealed partial class FfmpegService
{
arguments.AddRange(["-headers", customHeaders]);
}
// Increase the max allocation size to avoid "overlong headers" errors
// when CDN (e.g. Douyin) returns HTTP response headers exceeding FFmpeg's
// default internal buffer (4096 bytes).
arguments.AddRange(["-max_alloc", "100000000"]);
}
if (enableReconnect &&
@@ -24,8 +24,13 @@ public sealed partial class FfmpegService : IFfmpegService
"""Opening '([^']+)' for writing""",
RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private static readonly TimeSpan StartupFailureNotificationCooldown = TimeSpan.FromMinutes(30);
private static readonly TimeSpan StartupFailureMaxBackoff = TimeSpan.FromMinutes(15);
private static readonly TimeSpan StartupFailureBaseBackoff = TimeSpan.FromSeconds(30);
private readonly ConcurrentDictionary<Guid, SessionProcessRuntime> _processes = new();
private readonly ConcurrentDictionary<Guid, PostProcessRuntimeEntry> _postProcessStates = new();
private readonly ConcurrentDictionary<Guid, RoomStartupFailureState> _roomStartupFailureStates = new();
private readonly object _transcodeConcurrencyLock = new();
private readonly Queue<TaskCompletionSource<IDisposable>> _transcodeWaiters = new();
private int _activeTranscodeTasks;
@@ -795,6 +800,84 @@ public sealed partial class FfmpegService : IFfmpegService
}
}
/// <summary>
/// Records a startup failure for the given live room and returns the backoff delay
/// that should be applied before the next re-poll attempt. Uses exponential backoff
/// to prevent tight fail→retry→fail→re-poll loops from flooding notifications.
/// </summary>
private TimeSpan RecordStartupFailure(Guid liveRoomId)
{
var now = DateTimeOffset.UtcNow;
var state = _roomStartupFailureStates.AddOrUpdate(
liveRoomId,
_ => new RoomStartupFailureState { ConsecutiveFailures = 1, FirstFailureAt = now, LastFailureAt = now },
(_, existing) =>
{
existing.ConsecutiveFailures++;
existing.LastFailureAt = now;
return existing;
});
var backoffSeconds = StartupFailureBaseBackoff.TotalSeconds * Math.Pow(2, Math.Min(state.ConsecutiveFailures - 1, 5));
var backoff = TimeSpan.FromSeconds(Math.Min(backoffSeconds, StartupFailureMaxBackoff.TotalSeconds));
_logger.LogWarning(
"Startup failure backoff for room {LiveRoomId}: {ConsecutiveFailures} consecutive failures, next poll delayed by {BackoffSeconds:F0}s",
liveRoomId, state.ConsecutiveFailures, backoff.TotalSeconds);
return backoff;
}
/// <summary>
/// Resets the startup failure backoff counter when a session successfully opens its first segment.
/// </summary>
private void ResetStartupFailureBackoff(Guid liveRoomId)
{
if (_roomStartupFailureStates.TryRemove(liveRoomId, out var state) && state.ConsecutiveFailures > 1)
{
_logger.LogInformation(
"Startup failure backoff reset for room {LiveRoomId} after {ConsecutiveFailures} failures",
liveRoomId, state.ConsecutiveFailures);
}
}
/// <summary>
/// Returns true if the failure notification for this room should be throttled
/// (i.e., at most one notification per <see cref="StartupFailureNotificationCooldown"/>).
/// </summary>
private bool ShouldThrottleStartupFailureNotification(Guid liveRoomId)
{
var now = DateTimeOffset.UtcNow;
var state = _roomStartupFailureStates.GetOrAdd(
liveRoomId,
_ => new RoomStartupFailureState { ConsecutiveFailures = 0, FirstFailureAt = now, LastFailureAt = now });
// Always allow the first failure notification through
if (state.LastNotificationAt is null)
{
state.LastNotificationAt = now;
return false;
}
if (now - state.LastNotificationAt.Value < StartupFailureNotificationCooldown)
{
_logger.LogWarning(
"Suppressed duplicate startup failure notification for room {LiveRoomId}. " +
"Last notification was at {LastNotificationAt} (cooldown={CooldownMinutes}min)",
liveRoomId, state.LastNotificationAt, StartupFailureNotificationCooldown.TotalMinutes);
return true;
}
state.LastNotificationAt = now;
return false;
}
private sealed class RoomStartupFailureState
{
public int ConsecutiveFailures;
public DateTimeOffset FirstFailureAt;
public DateTimeOffset LastFailureAt;
public DateTimeOffset? LastNotificationAt;
}
private sealed record PostProcessRuntimeEntry(Guid RecordSessionId, RecordTaskRuntimeState State);
private sealed class TranscodeSlotLease : IDisposable