fix: add max_alloc to prevent overlong headers error and throttle startup failure notifications
- Add -max_alloc 100000000 to FFmpeg arguments for HTTP inputs to avoid 'overlong headers' error when CDN (e.g. Douyin) returns oversized HTTP response headers exceeding FFmpeg's default 4096-byte buffer. - Add exponential backoff for repeated startup failures (30s → 15min cap) to break the tight fail→retry→re-poll loop that floods notifications. - Throttle startup failure notifications to at most one per 30 minutes per room to prevent email/webhook storms during persistent failures. - Reset backoff counter when a session successfully opens its first segment.
This commit is contained in:
@@ -324,6 +324,7 @@ public sealed partial class FfmpegService
|
|||||||
if (runtime.SaveMode == RecordSaveMode.SingleFile)
|
if (runtime.SaveMode == RecordSaveMode.SingleFile)
|
||||||
{
|
{
|
||||||
runtime.HasOpenedFirstSegment = true;
|
runtime.HasOpenedFirstSegment = true;
|
||||||
|
ResetStartupFailureBackoff(runtime.LiveRoomId);
|
||||||
currentTask.AttachProcess(runtime.ProcessId, now);
|
currentTask.AttachProcess(runtime.ProcessId, now);
|
||||||
currentTask.MarkRunning(now);
|
currentTask.MarkRunning(now);
|
||||||
session.AttachProcess(runtime.ProcessId, now);
|
session.AttachProcess(runtime.ProcessId, now);
|
||||||
@@ -361,6 +362,7 @@ public sealed partial class FfmpegService
|
|||||||
if (segmentIndex == runtime.CurrentSegmentIndex)
|
if (segmentIndex == runtime.CurrentSegmentIndex)
|
||||||
{
|
{
|
||||||
runtime.HasOpenedFirstSegment = true;
|
runtime.HasOpenedFirstSegment = true;
|
||||||
|
ResetStartupFailureBackoff(runtime.LiveRoomId);
|
||||||
var finalOutputPath = NormalizeAbsolutePath(
|
var finalOutputPath = NormalizeAbsolutePath(
|
||||||
currentTask.OutputFilePath ??
|
currentTask.OutputFilePath ??
|
||||||
ResolveSegmentOutputPath(session.OutputPathPattern ?? runtime.OutputPathPattern, session.SaveMode, segmentIndex));
|
ResolveSegmentOutputPath(session.OutputPathPattern ?? runtime.OutputPathPattern, session.SaveMode, segmentIndex));
|
||||||
@@ -435,6 +437,7 @@ public sealed partial class FfmpegService
|
|||||||
runtime.CurrentOutputFilePath = normalizedOpenedPath;
|
runtime.CurrentOutputFilePath = normalizedOpenedPath;
|
||||||
runtime.ResetCurrentRecorderSegmentPaths(normalizedOpenedPath);
|
runtime.ResetCurrentRecorderSegmentPaths(normalizedOpenedPath);
|
||||||
runtime.HasOpenedFirstSegment = true;
|
runtime.HasOpenedFirstSegment = true;
|
||||||
|
ResetStartupFailureBackoff(runtime.LiveRoomId);
|
||||||
|
|
||||||
if (runtime.DanmakuRecorder is not null)
|
if (runtime.DanmakuRecorder is not null)
|
||||||
{
|
{
|
||||||
@@ -1213,24 +1216,38 @@ public sealed partial class FfmpegService
|
|||||||
|
|
||||||
if (!runtime.StopRequested && session.LiveRoomId != Guid.Empty)
|
if (!runtime.StopRequested && session.LiveRoomId != Guid.Empty)
|
||||||
{
|
{
|
||||||
_liveRoomPollingSignal.RequestImmediatePoll(session.LiveRoomId, TimeSpan.FromSeconds(2));
|
var isStartupFailure = !runtime.HasOpenedFirstSegment &&
|
||||||
|
runtime.StartupFailureKind != StartupFailureKind.None;
|
||||||
|
var pollDelay = isStartupFailure
|
||||||
|
? RecordStartupFailure(session.LiveRoomId)
|
||||||
|
: TimeSpan.FromSeconds(2);
|
||||||
|
_liveRoomPollingSignal.RequestImmediatePoll(session.LiveRoomId, pollDelay);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (session.Status == RecordSessionStatus.Failed && session.LiveRoom is not null)
|
if (session.Status == RecordSessionStatus.Failed && session.LiveRoom is not null)
|
||||||
{
|
{
|
||||||
var failureDetail = BuildExitLogDetail(runtime, process.ExitCode, effectiveOutputPath, toleratedNonZeroExit: false, finalizationError);
|
var isStartupFailure = !runtime.HasOpenedFirstSegment &&
|
||||||
await emailNotificationService.SendExceptionAsync(
|
runtime.StartupFailureKind != StartupFailureKind.None;
|
||||||
"FFmpeg",
|
if (isStartupFailure && ShouldThrottleStartupFailureNotification(session.LiveRoomId))
|
||||||
"Recording session exited abnormally.",
|
{
|
||||||
failureDetail,
|
// Notification suppressed — the failure has already been logged above.
|
||||||
session.LiveRoom,
|
}
|
||||||
currentTask);
|
else
|
||||||
await webhookNotificationService.SendExceptionAsync(
|
{
|
||||||
"FFmpeg",
|
var failureDetail = BuildExitLogDetail(runtime, process.ExitCode, effectiveOutputPath, toleratedNonZeroExit: false, finalizationError);
|
||||||
"Recording session exited abnormally.",
|
await emailNotificationService.SendExceptionAsync(
|
||||||
failureDetail,
|
"FFmpeg",
|
||||||
session.LiveRoom,
|
"Recording session exited abnormally.",
|
||||||
currentTask);
|
failureDetail,
|
||||||
|
session.LiveRoom,
|
||||||
|
currentTask);
|
||||||
|
await webhookNotificationService.SendExceptionAsync(
|
||||||
|
"FFmpeg",
|
||||||
|
"Recording session exited abnormally.",
|
||||||
|
failureDetail,
|
||||||
|
session.LiveRoom,
|
||||||
|
currentTask);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -676,6 +676,11 @@ public sealed partial class FfmpegService
|
|||||||
{
|
{
|
||||||
arguments.AddRange(["-headers", customHeaders]);
|
arguments.AddRange(["-headers", customHeaders]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Increase the max allocation size to avoid "overlong headers" errors
|
||||||
|
// when CDN (e.g. Douyin) returns HTTP response headers exceeding FFmpeg's
|
||||||
|
// default internal buffer (4096 bytes).
|
||||||
|
arguments.AddRange(["-max_alloc", "100000000"]);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (enableReconnect &&
|
if (enableReconnect &&
|
||||||
|
|||||||
@@ -24,8 +24,13 @@ public sealed partial class FfmpegService : IFfmpegService
|
|||||||
"""Opening '([^']+)' for writing""",
|
"""Opening '([^']+)' for writing""",
|
||||||
RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||||
|
|
||||||
|
private static readonly TimeSpan StartupFailureNotificationCooldown = TimeSpan.FromMinutes(30);
|
||||||
|
private static readonly TimeSpan StartupFailureMaxBackoff = TimeSpan.FromMinutes(15);
|
||||||
|
private static readonly TimeSpan StartupFailureBaseBackoff = TimeSpan.FromSeconds(30);
|
||||||
|
|
||||||
private readonly ConcurrentDictionary<Guid, SessionProcessRuntime> _processes = new();
|
private readonly ConcurrentDictionary<Guid, SessionProcessRuntime> _processes = new();
|
||||||
private readonly ConcurrentDictionary<Guid, PostProcessRuntimeEntry> _postProcessStates = new();
|
private readonly ConcurrentDictionary<Guid, PostProcessRuntimeEntry> _postProcessStates = new();
|
||||||
|
private readonly ConcurrentDictionary<Guid, RoomStartupFailureState> _roomStartupFailureStates = new();
|
||||||
private readonly object _transcodeConcurrencyLock = new();
|
private readonly object _transcodeConcurrencyLock = new();
|
||||||
private readonly Queue<TaskCompletionSource<IDisposable>> _transcodeWaiters = new();
|
private readonly Queue<TaskCompletionSource<IDisposable>> _transcodeWaiters = new();
|
||||||
private int _activeTranscodeTasks;
|
private int _activeTranscodeTasks;
|
||||||
@@ -795,6 +800,84 @@ public sealed partial class FfmpegService : IFfmpegService
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Records a startup failure for the given live room and returns the backoff delay
|
||||||
|
/// that should be applied before the next re-poll attempt. Uses exponential backoff
|
||||||
|
/// to prevent tight fail→retry→fail→re-poll loops from flooding notifications.
|
||||||
|
/// </summary>
|
||||||
|
private TimeSpan RecordStartupFailure(Guid liveRoomId)
|
||||||
|
{
|
||||||
|
var now = DateTimeOffset.UtcNow;
|
||||||
|
var state = _roomStartupFailureStates.AddOrUpdate(
|
||||||
|
liveRoomId,
|
||||||
|
_ => new RoomStartupFailureState { ConsecutiveFailures = 1, FirstFailureAt = now, LastFailureAt = now },
|
||||||
|
(_, existing) =>
|
||||||
|
{
|
||||||
|
existing.ConsecutiveFailures++;
|
||||||
|
existing.LastFailureAt = now;
|
||||||
|
return existing;
|
||||||
|
});
|
||||||
|
|
||||||
|
var backoffSeconds = StartupFailureBaseBackoff.TotalSeconds * Math.Pow(2, Math.Min(state.ConsecutiveFailures - 1, 5));
|
||||||
|
var backoff = TimeSpan.FromSeconds(Math.Min(backoffSeconds, StartupFailureMaxBackoff.TotalSeconds));
|
||||||
|
_logger.LogWarning(
|
||||||
|
"Startup failure backoff for room {LiveRoomId}: {ConsecutiveFailures} consecutive failures, next poll delayed by {BackoffSeconds:F0}s",
|
||||||
|
liveRoomId, state.ConsecutiveFailures, backoff.TotalSeconds);
|
||||||
|
return backoff;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Resets the startup failure backoff counter when a session successfully opens its first segment.
|
||||||
|
/// </summary>
|
||||||
|
private void ResetStartupFailureBackoff(Guid liveRoomId)
|
||||||
|
{
|
||||||
|
if (_roomStartupFailureStates.TryRemove(liveRoomId, out var state) && state.ConsecutiveFailures > 1)
|
||||||
|
{
|
||||||
|
_logger.LogInformation(
|
||||||
|
"Startup failure backoff reset for room {LiveRoomId} after {ConsecutiveFailures} failures",
|
||||||
|
liveRoomId, state.ConsecutiveFailures);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Returns true if the failure notification for this room should be throttled
|
||||||
|
/// (i.e., at most one notification per <see cref="StartupFailureNotificationCooldown"/>).
|
||||||
|
/// </summary>
|
||||||
|
private bool ShouldThrottleStartupFailureNotification(Guid liveRoomId)
|
||||||
|
{
|
||||||
|
var now = DateTimeOffset.UtcNow;
|
||||||
|
var state = _roomStartupFailureStates.GetOrAdd(
|
||||||
|
liveRoomId,
|
||||||
|
_ => new RoomStartupFailureState { ConsecutiveFailures = 0, FirstFailureAt = now, LastFailureAt = now });
|
||||||
|
|
||||||
|
// Always allow the first failure notification through
|
||||||
|
if (state.LastNotificationAt is null)
|
||||||
|
{
|
||||||
|
state.LastNotificationAt = now;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (now - state.LastNotificationAt.Value < StartupFailureNotificationCooldown)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(
|
||||||
|
"Suppressed duplicate startup failure notification for room {LiveRoomId}. " +
|
||||||
|
"Last notification was at {LastNotificationAt} (cooldown={CooldownMinutes}min)",
|
||||||
|
liveRoomId, state.LastNotificationAt, StartupFailureNotificationCooldown.TotalMinutes);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
state.LastNotificationAt = now;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class RoomStartupFailureState
|
||||||
|
{
|
||||||
|
public int ConsecutiveFailures;
|
||||||
|
public DateTimeOffset FirstFailureAt;
|
||||||
|
public DateTimeOffset LastFailureAt;
|
||||||
|
public DateTimeOffset? LastNotificationAt;
|
||||||
|
}
|
||||||
|
|
||||||
private sealed record PostProcessRuntimeEntry(Guid RecordSessionId, RecordTaskRuntimeState State);
|
private sealed record PostProcessRuntimeEntry(Guid RecordSessionId, RecordTaskRuntimeState State);
|
||||||
|
|
||||||
private sealed class TranscodeSlotLease : IDisposable
|
private sealed class TranscodeSlotLease : IDisposable
|
||||||
|
|||||||
Reference in New Issue
Block a user