feat: improve recording recovery and upload workflow

2026-08-10 11:14:34 +08:00
parent e0d3969e46
commit 19d358b312
58 changed files with 5979 additions and 586 deletions
@@ -19,6 +19,7 @@ namespace LiveRecorder.Infrastructure.Services;
 public sealed partial class FfmpegService
 {
    private const int MaxInSessionRetryAttempts = 3;
+    private const string RuntimeRecoveryMarker = "[runtime-recovery]";
    private const string ShortUnexpectedExitArtifactError =
        "Unexpected recorder exit produced only a short fragment. The file was kept locally and excluded from automatic upload.";
    internal const string FfprobeUnreadableArtifactError = "ffprobe could not read the recorded media file.";
@@ -29,6 +30,15 @@ public sealed partial class FfmpegService
    private static readonly TimeSpan RuntimeSourceFailureWindow = TimeSpan.FromSeconds(20);
    private static readonly TimeSpan RuntimeOfflineVerificationStopTimeout = TimeSpan.FromSeconds(20);
    private static readonly TimeSpan RuntimeOfflineVerificationKillTimeout = TimeSpan.FromSeconds(8);
+    private static readonly TimeSpan OfflineConfirmationDelay = TimeSpan.FromSeconds(10);
+    private static readonly TimeSpan[] RuntimeRecoveryBackoff =
+    [
+        TimeSpan.FromSeconds(5),
+        TimeSpan.FromSeconds(15),
+        TimeSpan.FromSeconds(30),
+        TimeSpan.FromSeconds(60),
+        TimeSpan.FromSeconds(120)
+    ];

    private async Task HandleProcessOutputAsync(SessionProcessRuntime runtime, string? line, bool isError)
    {
@@ -221,6 +231,9 @@ public sealed partial class FfmpegService
    internal static bool IsMeaningfulUnexpectedExitArtifact(double? durationSeconds, long? fileSizeBytes) =>
        durationSeconds >= MinimumUnexpectedExitArtifactDuration.TotalSeconds;

+    internal static bool CanResetInSessionRetryBudget(DateTimeOffset? processStartedAt, DateTimeOffset observedAt) =>
+        processStartedAt.HasValue && observedAt - processStartedAt.Value >= StableRuntimeResetThreshold;
+
    private static bool IsRuntimeSourceFailureLine(string line)
    {
        if (line.Contains("Will reconnect at", StringComparison.OrdinalIgnoreCase) ||
@@ -567,6 +580,19 @@ public sealed partial class FfmpegService

    private async Task HandleProcessExitedAsync(SessionProcessRuntime runtime, Process process)
    {
+        var transition = new SessionTransitionRuntime(runtime.RecordSessionId);
+        if (!_sessionTransitions.TryAdd(runtime.RecordSessionId, transition))
+        {
+            transition.Dispose();
+            if (_sessionTransitions.TryGetValue(runtime.RecordSessionId, out var activeTransition))
+            {
+                await activeTransition.Completion.Task;
+            }
+
+            await HandleProcessExitedAsync(runtime, process);
+            return;
+        }
+
        _processes.TryRemove(runtime.RecordSessionId, out _);

        try
@@ -604,7 +630,16 @@ public sealed partial class FfmpegService
                return;
            }

-            if (await TryRecoverUnexpectedExitAsync(runtime, activeDanmakuSummary))
+            if (!runtime.HasOpenedFirstSegment &&
+                !runtime.StopRequested &&
+                !runtime.CompletionRequested &&
+                !runtime.ShutdownRequested &&
+                await TryRecoverStartupUntilAvailableAsync(runtime, transition))
+            {
+                return;
+            }
+
+            if (await TryRecoverUnexpectedExitAsync(runtime, activeDanmakuSummary, transition))
            {
                return;
            }
@@ -617,12 +652,110 @@ public sealed partial class FfmpegService
        }
        finally
        {
+            _sessionTransitions.TryRemove(runtime.RecordSessionId, out _);
+            transition.Completion.TrySetResult(true);
+            transition.Dispose();
            runtime.ExitCompletion.TrySetResult(true);
            runtime.Dispose();
            process.Dispose();
        }
    }

+    private async Task<bool> TryRecoverStartupUntilAvailableAsync(
+        SessionProcessRuntime runtime,
+        SessionTransitionRuntime transition)
+    {
+        var attempt = transition.NextAttempt(Math.Max(1, runtime.RetryAttemptCount + 1));
+        while (!transition.Token.IsCancellationRequested)
+        {
+            try
+            {
+                using var scope = _serviceScopeFactory.CreateScope();
+                var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
+                var session = await dbContext.RecordSessions
+                    .Include(item => item.LiveRoom)
+                    .Include(item => item.RecordTasks)
+                    .ThenInclude(item => item.Result)
+                    .FirstOrDefaultAsync(item => item.Id == runtime.RecordSessionId, transition.Token);
+                if (session?.LiveRoom is null || !IsActiveSessionStatus(session.Status))
+                {
+                    return false;
+                }
+
+                var currentTask = session.RecordTasks.FirstOrDefault(item => item.Id == runtime.CurrentTaskId)
+                    ?? session.RecordTasks.OrderByDescending(item => item.SegmentIndex).FirstOrDefault();
+                if (currentTask is null)
+                {
+                    return false;
+                }
+
+                var delay = GetRuntimeRecoveryDelay(attempt);
+                session.MarkRecovering(
+                    $"{RuntimeRecoveryMarker} attempt={attempt}; startup did not open a media segment",
+                    DateTimeOffset.UtcNow);
+                currentTask.MarkStarting(runtime.StreamUrl, currentTask.OutputFilePath ?? runtime.CurrentOutputFilePath, DateTimeOffset.UtcNow);
+                await dbContext.SaveChangesAsync(transition.Token);
+
+                var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
+                await logService.WriteAsync(
+                    SystemLogLevel.Warning,
+                    "FFmpeg",
+                    $"Recorder startup is recovering in the same session (attempt {attempt}).",
+                    $"delaySeconds={delay.TotalSeconds:0}; ffmpegExit={runtime.Process?.ExitCode}; curlExit={runtime.CurlExitCode?.ToString() ?? "unknown"}; output={runtime.GetRecentOutputSummary()}; curl={runtime.GetRecentCurlErrorSummary()}",
+                    session.LiveRoomId,
+                    session.Id,
+                    currentTask.Id,
+                    transition.Token);
+
+                await Task.Delay(delay, transition.Token);
+                var adapterFactory = scope.ServiceProvider.GetRequiredService<ILivePlatformAdapterFactory>();
+                var adapter = adapterFactory.GetByPlatform(session.LiveRoom.Platform);
+                var liveStatus = await adapter.GetLiveStatusAsync(session.LiveRoom.RoomId, transition.Token);
+                if (!liveStatus.IsLive)
+                {
+                    await Task.Delay(OfflineConfirmationDelay, transition.Token);
+                    liveStatus = await adapter.GetLiveStatusAsync(session.LiveRoom.RoomId, transition.Token);
+                }
+
+                if (!liveStatus.IsLive)
+                {
+                    runtime.MarkStopRequested(markAsCompletedOnExit: true);
+                    return false;
+                }
+
+                var stream = await adapter.GetStreamUrlAsync(session.LiveRoom.RoomId, session.PreferredQuality, transition.Token);
+                var selected = SelectRetryStreamForCurrentSession(stream, runtime);
+                var context = runtime.RecoveryContext with
+                {
+                    AttemptCount = attempt,
+                    HasRetriedWithRefreshedStream = true
+                };
+                session.MarkStarting(selected.SelectedUrl, session.OutputPathPattern ?? runtime.OutputPathPattern, DateTimeOffset.UtcNow);
+                currentTask.MarkStarting(selected.SelectedUrl, currentTask.OutputFilePath ?? runtime.CurrentOutputFilePath, DateTimeOffset.UtcNow);
+                await dbContext.SaveChangesAsync(transition.Token);
+
+                await StartInternalAsync(session, currentTask, selected, runtime.RecordingSettings, context, transition.Token);
+                var restartedAt = DateTimeOffset.UtcNow;
+                session.MarkRunning(restartedAt);
+                currentTask.MarkRunning(restartedAt);
+                await dbContext.SaveChangesAsync(transition.Token);
+                return true;
+            }
+            catch (OperationCanceledException) when (transition.Token.IsCancellationRequested)
+            {
+                runtime.MarkStopRequested(transition.MarkAsCompletedOnExit);
+                return false;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning(ex, "Recorder startup recovery attempt {Attempt} failed for session {RecordSessionId}", attempt, runtime.RecordSessionId);
+                attempt = transition.NextAttempt(attempt + 1);
+            }
+        }
+
+        return false;
+    }
+
    private async Task<bool> TryRecoverStartupFailureAsync(SessionProcessRuntime runtime)
    {
        if (runtime.ShutdownRequested ||
@@ -885,14 +1018,14 @@ public sealed partial class FfmpegService

    private async Task<bool> TryRecoverUnexpectedExitAsync(
        SessionProcessRuntime runtime,
-        SessionDanmakuXmlRecorder.DanmakuSegmentSummary? activeDanmakuSummary)
+        SessionDanmakuXmlRecorder.DanmakuSegmentSummary? activeDanmakuSummary,
+        SessionTransitionRuntime transition)
    {
        if (!runtime.HasOpenedFirstSegment ||
            runtime.StopRequested ||
            runtime.CompletionRequested ||
            runtime.ShutdownRequested ||
-            runtime.SaveMode != RecordSaveMode.Segmented ||
-            runtime.RetryAttemptCount >= MaxInSessionRetryAttempts)
+            runtime.SaveMode != RecordSaveMode.Segmented)
        {
            return false;
        }
@@ -929,15 +1062,20 @@ public sealed partial class FfmpegService
            var observedAt = DateTimeOffset.UtcNow;
            var adapter = adapterFactory.GetByPlatform(session.LiveRoom.Platform);
            var liveStatus = await adapter.GetLiveStatusAsync(session.LiveRoom.RoomId);
+            if (!liveStatus.IsLive)
+            {
+                await Task.Delay(OfflineConfirmationDelay, transition.Token);
+                liveStatus = await adapter.GetLiveStatusAsync(session.LiveRoom.RoomId, transition.Token);
+            }
            await liveRoomStatusService.ApplySnapshotAsync(session.LiveRoom, liveStatus, observedAt);
            await dbContext.SaveChangesAsync();

            if (!liveStatus.IsLive)
            {
+                runtime.MarkStopRequested(markAsCompletedOnExit: true);
                return false;
            }

-            var retryAttempt = runtime.RetryAttemptCount + 1;
            var refreshedStream = await adapter.GetStreamUrlAsync(session.LiveRoom.RoomId, session.PreferredQuality);
            var retryStream = SelectRetryStreamForCurrentSession(refreshedStream, runtime);
            var nextSegmentIndex = Math.Max(currentTask.SegmentIndex + 1, session.ActiveSegmentIndex + 1);
@@ -957,11 +1095,17 @@ public sealed partial class FfmpegService
            await dbContext.SaveChangesAsync();

            var retryInputOptionProfile = ResolveRetryInputOptionProfile(runtime);
+            var stableBeforeExit = runtime.ProcessStartedAt.HasValue &&
+                                   observedAt - runtime.ProcessStartedAt.Value >= StableRuntimeResetThreshold;
+            var recoveryBase = stableBeforeExit
+                ? InitialRecoveryContext
+                : runtime.RecoveryContext;
            var recoveryContext = AdvanceRecoveryContext(
-                runtime.RecoveryContext,
+                recoveryBase,
                retryInputOptionProfile,
                runtime.SelectedProtocol,
                retryStream.SelectedProtocol);
+            var retryAttempt = recoveryContext.AttemptCount;
            try
            {
                await StartInternalAsync(
@@ -1063,7 +1207,7 @@ public sealed partial class FfmpegService
            await logService.WriteAsync(
                SystemLogLevel.Warning,
                "FFmpeg",
-                $"ffmpeg exited unexpectedly. Retrying within the current session ({retryAttempt}/{MaxInSessionRetryAttempts}).",
+                $"ffmpeg exited unexpectedly. Retrying within the current session (attempt {retryAttempt}).",
                $"transition={runtime.InputOptionProfile}/{runtime.SelectedProtocol} -> {retryInputOptionProfile}/{retryStream.SelectedProtocol}; output={runtime.GetRecentOutputSummary()}",
                session.LiveRoomId,
                session.Id,
@@ -1074,10 +1218,82 @@ public sealed partial class FfmpegService
        catch (Exception ex)
        {
            _logger.LogWarning(ex, "In-session ffmpeg retry failed for session {RecordSessionId}", runtime.RecordSessionId);
-            return false;
+            return await WaitAndRetryRecoveringSessionAsync(runtime, activeDanmakuSummary, transition, ex);
        }
    }

+    private async Task<bool> WaitAndRetryRecoveringSessionAsync(
+        SessionProcessRuntime runtime,
+        SessionDanmakuXmlRecorder.DanmakuSegmentSummary? activeDanmakuSummary,
+        SessionTransitionRuntime transition,
+        Exception failure)
+    {
+        var attempt = transition.NextAttempt(Math.Max(1, runtime.RetryAttemptCount + 1));
+        while (!transition.Token.IsCancellationRequested && !runtime.ShutdownRequested)
+        {
+            try
+            {
+                using (var scope = _serviceScopeFactory.CreateScope())
+                {
+                    var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
+                    var session = await dbContext.RecordSessions.FirstOrDefaultAsync(
+                        item => item.Id == runtime.RecordSessionId,
+                        CancellationToken.None);
+                    if (session is null || !IsActiveSessionStatus(session.Status))
+                    {
+                        return false;
+                    }
+
+                    var reason = $"{RuntimeRecoveryMarker} attempt={attempt}; {failure.GetBaseException().Message}";
+                    session.MarkRecovering(reason, DateTimeOffset.UtcNow);
+                    await dbContext.SaveChangesAsync(CancellationToken.None);
+
+                    var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
+                    await logService.WriteAsync(
+                        SystemLogLevel.Warning,
+                        "FFmpeg",
+                        $"Recorder recovery is waiting before attempt {attempt}.",
+                        $"delaySeconds={GetRuntimeRecoveryDelay(attempt).TotalSeconds:0}; failure={failure}",
+                        runtime.LiveRoomId,
+                        runtime.RecordSessionId,
+                        runtime.CurrentTaskId,
+                        CancellationToken.None);
+                }
+
+                await Task.Delay(GetRuntimeRecoveryDelay(attempt), transition.Token);
+                if (transition.MarkAsCompletedOnExit)
+                {
+                    runtime.MarkStopRequested(markAsCompletedOnExit: true);
+                    return false;
+                }
+
+                if (transition.StopRequested)
+                {
+                    runtime.MarkStopRequested(markAsCompletedOnExit: false);
+                    return false;
+                }
+
+                // Re-enter the normal recovery path with a fresh platform status and stream URL.
+                return await TryRecoverUnexpectedExitAsync(runtime, activeDanmakuSummary, transition);
+            }
+            catch (OperationCanceledException) when (transition.Token.IsCancellationRequested)
+            {
+                runtime.MarkStopRequested(transition.MarkAsCompletedOnExit);
+                return false;
+            }
+            catch (Exception ex)
+            {
+                failure = ex;
+                attempt = transition.NextAttempt(attempt + 1);
+            }
+        }
+
+        return false;
+    }
+
+    internal static TimeSpan GetRuntimeRecoveryDelay(int attempt) =>
+        RuntimeRecoveryBackoff[Math.Clamp(attempt - 1, 0, RuntimeRecoveryBackoff.Length - 1)];
+
    private async Task<bool> TryRetryWithAlternateProtocolAsync(
        SessionProcessRuntime runtime,
        RecordSession session,
@@ -1543,7 +1759,9 @@ public sealed partial class FfmpegService
            $"exitCode={exitCode}",
            $"output={effectiveOutputPath}",
            $"recorderOutput={runtime.RecorderOutputPath}",
-            $"shutdownRequested={runtime.ShutdownRequested}"
+            $"shutdownRequested={runtime.ShutdownRequested}",
+            $"lifetimeSeconds={(runtime.ProcessStartedAt.HasValue ? Math.Max(0, (DateTimeOffset.UtcNow - runtime.ProcessStartedAt.Value).TotalSeconds).ToString("F1", CultureInfo.InvariantCulture) : "unknown")}",
+            $"curlExitCode={runtime.CurlExitCode?.ToString(CultureInfo.InvariantCulture) ?? "unknown"}"
        };

        if (toleratedNonZeroExit)
@@ -1567,6 +1785,12 @@ public sealed partial class FfmpegService
            parts.Add($"recentOutput={recentOutput}");
        }

+        var recentCurlError = runtime.GetRecentCurlErrorSummary();
+        if (!string.IsNullOrWhiteSpace(recentCurlError))
+        {
+            parts.Add($"curlStderr={recentCurlError}");
+        }
+
        return string.Join("; ", parts);
    }

@@ -2280,6 +2504,7 @@ public sealed partial class FfmpegService
        private object RuntimeSourceFailureSync { get; } = new();
        private object RecentOutputSync { get; } = new();
        private Queue<string> RecentOutputLines { get; } = new();
+        private Queue<string> RecentCurlErrorLines { get; } = new();
        private DateTimeOffset RuntimeSourceFailureWindowStartedAt { get; set; }
        private int RuntimeSourceFailureCount { get; set; }
        private bool RuntimeSourceFailureVerificationInProgress { get; set; }
@@ -2290,6 +2515,14 @@ public sealed partial class FfmpegService
            ProcessStartedAt = DateTimeOffset.UtcNow;
        }
        public void AttachCurlProcess(Process curlProcess) => CurlProcess = curlProcess;
+        public int? CurlExitCode
+        {
+            get
+            {
+                try { return CurlProcess?.HasExited == true ? CurlProcess.ExitCode : null; }
+                catch { return null; }
+            }
+        }
        public void MarkTimestampDiscontinuityFailure() => HasTimestampDiscontinuityFailure = true;
        public void MarkTimestampMuxerFailure() => HasTimestampMuxerFailure = true;
        public void MarkHlsOverlongHeadersFailure() => HasHlsOverlongHeadersFailure = true;
@@ -2353,6 +2586,28 @@ public sealed partial class FfmpegService
            }
        }

+        public void RememberCurlErrorLine(string line)
+        {
+            lock (RecentOutputSync)
+            {
+                RecentCurlErrorLines.Enqueue(line.Trim());
+                while (RecentCurlErrorLines.Count > 8)
+                {
+                    RecentCurlErrorLines.Dequeue();
+                }
+            }
+        }
+
+        public string? GetRecentCurlErrorSummary()
+        {
+            lock (RecentOutputSync)
+            {
+                return RecentCurlErrorLines.Count == 0
+                    ? null
+                    : string.Join(" | ", RecentCurlErrorLines);
+            }
+        }
+
        public void ResetCurrentRecorderSegmentPaths(string openedPath)
        {
            CurrentRecorderSegmentPaths.Clear();
@@ -2428,6 +2683,41 @@ public sealed partial class FfmpegService
            Gate.Dispose();
        }
    }
+
+    private sealed class SessionTransitionRuntime : IDisposable
+    {
+        private readonly CancellationTokenSource _cancellation = new();
+
+        public SessionTransitionRuntime(Guid recordSessionId) => RecordSessionId = recordSessionId;
+
+        public Guid RecordSessionId { get; }
+        public bool StopRequested { get; private set; }
+        public bool MarkAsCompletedOnExit { get; private set; }
+        public CancellationToken Token => _cancellation.Token;
+        public TaskCompletionSource<bool> Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+        private int _attemptCount;
+
+        public int NextAttempt(int minimum)
+        {
+            var next = Interlocked.Increment(ref _attemptCount);
+            if (next >= minimum)
+            {
+                return next;
+            }
+
+            Interlocked.Exchange(ref _attemptCount, minimum);
+            return minimum;
+        }
+
+        public void RequestStop(bool markAsCompletedOnExit)
+        {
+            StopRequested = true;
+            MarkAsCompletedOnExit = markAsCompletedOnExit;
+            _cancellation.Cancel();
+        }
+
+        public void Dispose() => _cancellation.Dispose();
+    }
 }

 internal enum ExitedRecordingDisposition