diff --git a/src/LiveRecorder.Infrastructure/Services/FfmpegService.Runtime.cs b/src/LiveRecorder.Infrastructure/Services/FfmpegService.Runtime.cs index 151b12f..cce1fd1 100644 --- a/src/LiveRecorder.Infrastructure/Services/FfmpegService.Runtime.cs +++ b/src/LiveRecorder.Infrastructure/Services/FfmpegService.Runtime.cs @@ -324,6 +324,7 @@ public sealed partial class FfmpegService if (runtime.SaveMode == RecordSaveMode.SingleFile) { runtime.HasOpenedFirstSegment = true; + ResetStartupFailureBackoff(runtime.LiveRoomId); currentTask.AttachProcess(runtime.ProcessId, now); currentTask.MarkRunning(now); session.AttachProcess(runtime.ProcessId, now); @@ -361,6 +362,7 @@ public sealed partial class FfmpegService if (segmentIndex == runtime.CurrentSegmentIndex) { runtime.HasOpenedFirstSegment = true; + ResetStartupFailureBackoff(runtime.LiveRoomId); var finalOutputPath = NormalizeAbsolutePath( currentTask.OutputFilePath ?? ResolveSegmentOutputPath(session.OutputPathPattern ?? runtime.OutputPathPattern, session.SaveMode, segmentIndex)); @@ -435,6 +437,7 @@ public sealed partial class FfmpegService runtime.CurrentOutputFilePath = normalizedOpenedPath; runtime.ResetCurrentRecorderSegmentPaths(normalizedOpenedPath); runtime.HasOpenedFirstSegment = true; + ResetStartupFailureBackoff(runtime.LiveRoomId); if (runtime.DanmakuRecorder is not null) { @@ -1213,24 +1216,38 @@ public sealed partial class FfmpegService if (!runtime.StopRequested && session.LiveRoomId != Guid.Empty) { - _liveRoomPollingSignal.RequestImmediatePoll(session.LiveRoomId, TimeSpan.FromSeconds(2)); + var isStartupFailure = !runtime.HasOpenedFirstSegment && + runtime.StartupFailureKind != StartupFailureKind.None; + var pollDelay = isStartupFailure + ? RecordStartupFailure(session.LiveRoomId) + : TimeSpan.FromSeconds(2); + _liveRoomPollingSignal.RequestImmediatePoll(session.LiveRoomId, pollDelay); } if (session.Status == RecordSessionStatus.Failed && session.LiveRoom is not null) { - var failureDetail = BuildExitLogDetail(runtime, process.ExitCode, effectiveOutputPath, toleratedNonZeroExit: false, finalizationError); - await emailNotificationService.SendExceptionAsync( - "FFmpeg", - "Recording session exited abnormally.", - failureDetail, - session.LiveRoom, - currentTask); - await webhookNotificationService.SendExceptionAsync( - "FFmpeg", - "Recording session exited abnormally.", - failureDetail, - session.LiveRoom, - currentTask); + var isStartupFailure = !runtime.HasOpenedFirstSegment && + runtime.StartupFailureKind != StartupFailureKind.None; + if (isStartupFailure && ShouldThrottleStartupFailureNotification(session.LiveRoomId)) + { + // Notification suppressed — the failure has already been logged above. + } + else + { + var failureDetail = BuildExitLogDetail(runtime, process.ExitCode, effectiveOutputPath, toleratedNonZeroExit: false, finalizationError); + await emailNotificationService.SendExceptionAsync( + "FFmpeg", + "Recording session exited abnormally.", + failureDetail, + session.LiveRoom, + currentTask); + await webhookNotificationService.SendExceptionAsync( + "FFmpeg", + "Recording session exited abnormally.", + failureDetail, + session.LiveRoom, + currentTask); + } } } diff --git a/src/LiveRecorder.Infrastructure/Services/FfmpegService.Utils.cs b/src/LiveRecorder.Infrastructure/Services/FfmpegService.Utils.cs index 93cf7d2..506b0b8 100644 --- a/src/LiveRecorder.Infrastructure/Services/FfmpegService.Utils.cs +++ b/src/LiveRecorder.Infrastructure/Services/FfmpegService.Utils.cs @@ -676,6 +676,11 @@ public sealed partial class FfmpegService { arguments.AddRange(["-headers", customHeaders]); } + + // Increase the max allocation size to avoid "overlong headers" errors + // when CDN (e.g. Douyin) returns HTTP response headers exceeding FFmpeg's + // default internal buffer (4096 bytes). + arguments.AddRange(["-max_alloc", "100000000"]); } if (enableReconnect && diff --git a/src/LiveRecorder.Infrastructure/Services/FfmpegService.cs b/src/LiveRecorder.Infrastructure/Services/FfmpegService.cs index e5fde13..c3e9b09 100644 --- a/src/LiveRecorder.Infrastructure/Services/FfmpegService.cs +++ b/src/LiveRecorder.Infrastructure/Services/FfmpegService.cs @@ -24,8 +24,13 @@ public sealed partial class FfmpegService : IFfmpegService """Opening '([^']+)' for writing""", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly TimeSpan StartupFailureNotificationCooldown = TimeSpan.FromMinutes(30); + private static readonly TimeSpan StartupFailureMaxBackoff = TimeSpan.FromMinutes(15); + private static readonly TimeSpan StartupFailureBaseBackoff = TimeSpan.FromSeconds(30); + private readonly ConcurrentDictionary _processes = new(); private readonly ConcurrentDictionary _postProcessStates = new(); + private readonly ConcurrentDictionary _roomStartupFailureStates = new(); private readonly object _transcodeConcurrencyLock = new(); private readonly Queue> _transcodeWaiters = new(); private int _activeTranscodeTasks; @@ -795,6 +800,84 @@ public sealed partial class FfmpegService : IFfmpegService } } + /// + /// Records a startup failure for the given live room and returns the backoff delay + /// that should be applied before the next re-poll attempt. Uses exponential backoff + /// to prevent tight fail→retry→fail→re-poll loops from flooding notifications. + /// + private TimeSpan RecordStartupFailure(Guid liveRoomId) + { + var now = DateTimeOffset.UtcNow; + var state = _roomStartupFailureStates.AddOrUpdate( + liveRoomId, + _ => new RoomStartupFailureState { ConsecutiveFailures = 1, FirstFailureAt = now, LastFailureAt = now }, + (_, existing) => + { + existing.ConsecutiveFailures++; + existing.LastFailureAt = now; + return existing; + }); + + var backoffSeconds = StartupFailureBaseBackoff.TotalSeconds * Math.Pow(2, Math.Min(state.ConsecutiveFailures - 1, 5)); + var backoff = TimeSpan.FromSeconds(Math.Min(backoffSeconds, StartupFailureMaxBackoff.TotalSeconds)); + _logger.LogWarning( + "Startup failure backoff for room {LiveRoomId}: {ConsecutiveFailures} consecutive failures, next poll delayed by {BackoffSeconds:F0}s", + liveRoomId, state.ConsecutiveFailures, backoff.TotalSeconds); + return backoff; + } + + /// + /// Resets the startup failure backoff counter when a session successfully opens its first segment. + /// + private void ResetStartupFailureBackoff(Guid liveRoomId) + { + if (_roomStartupFailureStates.TryRemove(liveRoomId, out var state) && state.ConsecutiveFailures > 1) + { + _logger.LogInformation( + "Startup failure backoff reset for room {LiveRoomId} after {ConsecutiveFailures} failures", + liveRoomId, state.ConsecutiveFailures); + } + } + + /// + /// Returns true if the failure notification for this room should be throttled + /// (i.e., at most one notification per ). + /// + private bool ShouldThrottleStartupFailureNotification(Guid liveRoomId) + { + var now = DateTimeOffset.UtcNow; + var state = _roomStartupFailureStates.GetOrAdd( + liveRoomId, + _ => new RoomStartupFailureState { ConsecutiveFailures = 0, FirstFailureAt = now, LastFailureAt = now }); + + // Always allow the first failure notification through + if (state.LastNotificationAt is null) + { + state.LastNotificationAt = now; + return false; + } + + if (now - state.LastNotificationAt.Value < StartupFailureNotificationCooldown) + { + _logger.LogWarning( + "Suppressed duplicate startup failure notification for room {LiveRoomId}. " + + "Last notification was at {LastNotificationAt} (cooldown={CooldownMinutes}min)", + liveRoomId, state.LastNotificationAt, StartupFailureNotificationCooldown.TotalMinutes); + return true; + } + + state.LastNotificationAt = now; + return false; + } + + private sealed class RoomStartupFailureState + { + public int ConsecutiveFailures; + public DateTimeOffset FirstFailureAt; + public DateTimeOffset LastFailureAt; + public DateTimeOffset? LastNotificationAt; + } + private sealed record PostProcessRuntimeEntry(Guid RecordSessionId, RecordTaskRuntimeState State); private sealed class TranscodeSlotLease : IDisposable