fix: add max_alloc to prevent overlong headers error and throttle startup failure notifications

- Add -max_alloc 100000000 to FFmpeg arguments for HTTP inputs to avoid
  'overlong headers' error when CDN (e.g. Douyin) returns oversized HTTP
  response headers exceeding FFmpeg's default 4096-byte buffer.

- Add exponential backoff for repeated startup failures (30s → 15min cap)
  to break the tight fail→retry→re-poll loop that floods notifications.

- Throttle startup failure notifications to at most one per 30 minutes per
  room to prevent email/webhook storms during persistent failures.

- Reset backoff counter when a session successfully opens its first segment.
This commit is contained in:
2026-07-09 11:51:52 +08:00
parent 48da49ac72
commit 8a079b4698
3 changed files with 119 additions and 14 deletions
@@ -24,8 +24,13 @@ public sealed partial class FfmpegService : IFfmpegService
"""Opening '([^']+)' for writing""",
RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private static readonly TimeSpan StartupFailureNotificationCooldown = TimeSpan.FromMinutes(30);
private static readonly TimeSpan StartupFailureMaxBackoff = TimeSpan.FromMinutes(15);
private static readonly TimeSpan StartupFailureBaseBackoff = TimeSpan.FromSeconds(30);
private readonly ConcurrentDictionary<Guid, SessionProcessRuntime> _processes = new();
private readonly ConcurrentDictionary<Guid, PostProcessRuntimeEntry> _postProcessStates = new();
private readonly ConcurrentDictionary<Guid, RoomStartupFailureState> _roomStartupFailureStates = new();
private readonly object _transcodeConcurrencyLock = new();
private readonly Queue<TaskCompletionSource<IDisposable>> _transcodeWaiters = new();
private int _activeTranscodeTasks;
@@ -795,6 +800,84 @@ public sealed partial class FfmpegService : IFfmpegService
}
}
/// <summary>
/// Records a startup failure for the given live room and returns the backoff delay
/// that should be applied before the next re-poll attempt. Uses exponential backoff
/// to prevent tight fail→retry→fail→re-poll loops from flooding notifications.
/// </summary>
private TimeSpan RecordStartupFailure(Guid liveRoomId)
{
var now = DateTimeOffset.UtcNow;
var state = _roomStartupFailureStates.AddOrUpdate(
liveRoomId,
_ => new RoomStartupFailureState { ConsecutiveFailures = 1, FirstFailureAt = now, LastFailureAt = now },
(_, existing) =>
{
existing.ConsecutiveFailures++;
existing.LastFailureAt = now;
return existing;
});
var backoffSeconds = StartupFailureBaseBackoff.TotalSeconds * Math.Pow(2, Math.Min(state.ConsecutiveFailures - 1, 5));
var backoff = TimeSpan.FromSeconds(Math.Min(backoffSeconds, StartupFailureMaxBackoff.TotalSeconds));
_logger.LogWarning(
"Startup failure backoff for room {LiveRoomId}: {ConsecutiveFailures} consecutive failures, next poll delayed by {BackoffSeconds:F0}s",
liveRoomId, state.ConsecutiveFailures, backoff.TotalSeconds);
return backoff;
}
/// <summary>
/// Resets the startup failure backoff counter when a session successfully opens its first segment.
/// </summary>
private void ResetStartupFailureBackoff(Guid liveRoomId)
{
if (_roomStartupFailureStates.TryRemove(liveRoomId, out var state) && state.ConsecutiveFailures > 1)
{
_logger.LogInformation(
"Startup failure backoff reset for room {LiveRoomId} after {ConsecutiveFailures} failures",
liveRoomId, state.ConsecutiveFailures);
}
}
/// <summary>
/// Returns true if the failure notification for this room should be throttled
/// (i.e., at most one notification per <see cref="StartupFailureNotificationCooldown"/>).
/// </summary>
private bool ShouldThrottleStartupFailureNotification(Guid liveRoomId)
{
var now = DateTimeOffset.UtcNow;
var state = _roomStartupFailureStates.GetOrAdd(
liveRoomId,
_ => new RoomStartupFailureState { ConsecutiveFailures = 0, FirstFailureAt = now, LastFailureAt = now });
// Always allow the first failure notification through
if (state.LastNotificationAt is null)
{
state.LastNotificationAt = now;
return false;
}
if (now - state.LastNotificationAt.Value < StartupFailureNotificationCooldown)
{
_logger.LogWarning(
"Suppressed duplicate startup failure notification for room {LiveRoomId}. " +
"Last notification was at {LastNotificationAt} (cooldown={CooldownMinutes}min)",
liveRoomId, state.LastNotificationAt, StartupFailureNotificationCooldown.TotalMinutes);
return true;
}
state.LastNotificationAt = now;
return false;
}
private sealed class RoomStartupFailureState
{
public int ConsecutiveFailures;
public DateTimeOffset FirstFailureAt;
public DateTimeOffset LastFailureAt;
public DateTimeOffset? LastNotificationAt;
}
private sealed record PostProcessRuntimeEntry(Guid RecordSessionId, RecordTaskRuntimeState State);
private sealed class TranscodeSlotLease : IDisposable