feat: harden recording lifecycle and refresh fnOS UI

2026-08-03 23:45:26 +08:00
parent e5b50ea85c
commit ecc737f0bd
90 changed files with 6995 additions and 1826 deletions
@@ -5,6 +5,7 @@ using LiveRecorder.Application.Abstractions.Logging;
 using LiveRecorder.Application.Abstractions.Notifications;
 using LiveRecorder.Application.Abstractions.Platforms;
 using LiveRecorder.Application.Abstractions.Recording;
+using LiveRecorder.Application.Abstractions.Scripting;
 using LiveRecorder.Application.Abstractions.Settings;
 using LiveRecorder.Application.Abstractions.Storage;
 using LiveRecorder.Application.Models.Media;
@@ -27,14 +28,23 @@ public sealed partial class FfmpegService : IFfmpegService
    private static readonly TimeSpan StartupFailureNotificationCooldown = TimeSpan.FromMinutes(30);
    private static readonly TimeSpan StartupFailureMaxBackoff = TimeSpan.FromMinutes(15);
    private static readonly TimeSpan StartupFailureBaseBackoff = TimeSpan.FromSeconds(30);
+    private static readonly TimeSpan RuntimeFailureMaxBackoff = TimeSpan.FromMinutes(5);
+    private static readonly TimeSpan RuntimeFailureBaseBackoff = TimeSpan.FromSeconds(15);

    private readonly ConcurrentDictionary<Guid, SessionProcessRuntime> _processes = new();
    private readonly ConcurrentDictionary<Guid, PostProcessRuntimeEntry> _postProcessStates = new();
+    private readonly ConcurrentDictionary<Guid, Process> _postProcessProcesses = new();
    private readonly ConcurrentDictionary<Guid, RoomStartupFailureState> _roomStartupFailureStates = new();
+    private readonly ConcurrentDictionary<Guid, RoomStartupFailureState> _roomRuntimeFailureStates = new();
+    private readonly SemaphoreSlim _recoveryEncoderProbeGate = new(1, 1);
    private readonly object _transcodeConcurrencyLock = new();
+    private readonly SemaphoreSlim _orphanRecoveryGate = new(1, 1);
    private readonly Queue<TaskCompletionSource<IDisposable>> _transcodeWaiters = new();
+    private readonly CancellationTokenSource _shutdownCts = new();
    private int _activeTranscodeTasks;
    private int _maxConcurrentTranscodeTasks = 1;
+    private bool _hasProbedRecoveryVideoEncoder;
+    private RecoveryVideoEncoderSelection _cachedRecoveryVideoEncoder = RecoveryVideoEncoderSelection.Software;
    private readonly IServiceScopeFactory _serviceScopeFactory;
    private readonly IStorageGuardService _storageGuardService;
    private readonly ILiveRoomPollingSignal _liveRoomPollingSignal;
@@ -87,10 +97,7 @@ public sealed partial class FfmpegService : IFfmpegService
            initialTask,
            streamUrlResult,
            recordingSettings,
-            inputOptionProfile: FfmpegInputOptionProfile.Baseline,
-            hasRetriedWithCompatibilityProfile: false,
-            hasRetriedWithRefreshedStream: false,
-            retryAttemptCount: 0,
+            InitialRecoveryContext,
            cancellationToken);

    private async Task StartInternalAsync(
@@ -98,10 +105,7 @@ public sealed partial class FfmpegService : IFfmpegService
        RecordTask initialTask,
        StreamUrlResult streamUrlResult,
        RecordingExecutionSettings recordingSettings,
-        FfmpegInputOptionProfile inputOptionProfile,
-        bool hasRetriedWithCompatibilityProfile,
-        bool hasRetriedWithRefreshedStream,
-        int retryAttemptCount,
+        FfmpegRecoveryContext recoveryContext,
        CancellationToken cancellationToken = default)
    {
        ArgumentNullException.ThrowIfNull(recordSession);
@@ -117,6 +121,9 @@ public sealed partial class FfmpegService : IFfmpegService
        using var settingsScope = _serviceScopeFactory.CreateScope();
        var settingsService = settingsScope.ServiceProvider.GetRequiredService<ISystemSettingsService>();
        var settings = await settingsService.GetAsync(cancellationToken);
+        var recoveryVideoEncoder = recoveryContext.InputOptionProfile == FfmpegInputOptionProfile.TimestampTranscode
+            ? await ResolveRecoveryVideoEncoderAsync(settings.FfmpegPath, recoveryContext.ForceSoftwareEncoder, cancellationToken)
+            : RecoveryVideoEncoderSelection.Software;

        var outputPathPattern = Path.IsPathRooted(recordSession.OutputPathPattern)
            ? recordSession.OutputPathPattern
@@ -154,10 +161,8 @@ public sealed partial class FfmpegService : IFfmpegService
            streamUrlResult.SelectedVideoCodec,
            streamUrlResult.InputHeaders,
            recordingSettings,
-            inputOptionProfile,
-            hasRetriedWithCompatibilityProfile,
-            hasRetriedWithRefreshedStream,
-            retryAttemptCount);
+            recoveryContext,
+            recoveryVideoEncoder);

        foreach (var argument in BuildArgumentList(
                     streamUrlResult.SelectedUrl,
@@ -173,7 +178,8 @@ public sealed partial class FfmpegService : IFfmpegService
                     streamUrlResult.InputHeaders,
                     streamUrlResult.SelectedProtocol,
                     streamUrlResult.SelectedVideoCodec,
-                     inputOptionProfile))
+                     recoveryContext.InputOptionProfile,
+                     recoveryVideoEncoder))
        {
            process.StartInfo.ArgumentList.Add(argument);
        }
@@ -199,7 +205,7 @@ public sealed partial class FfmpegService : IFfmpegService
        // pipe its stdout into FFmpeg's stdin. This bypasses FFmpeg's built-in HTTP
        // handler which has a hard-coded 4096-byte response header buffer that triggers
        // "overlong headers" errors with CDNs that return oversized headers.
-        if (IsHttpInput(streamUrlResult.SelectedUrl))
+        if (ShouldUseCurlPipe(streamUrlResult.SelectedUrl, streamUrlResult.SelectedProtocol))
        {
            var curlProcess = new Process
            {
@@ -295,6 +301,21 @@ public sealed partial class FfmpegService : IFfmpegService
        }

        var process = runtime.Process;
+        var curlProcess = runtime.CurlProcess;
+        if (curlProcess is not null)
+        {
+            try
+            {
+                if (!curlProcess.HasExited)
+                {
+                    curlProcess.Kill(true);
+                }
+            }
+            catch (InvalidOperationException)
+            {
+            }
+        }
+
        if (process is not null)
        {
            try
@@ -313,7 +334,178 @@ public sealed partial class FfmpegService : IFfmpegService
        return await WaitForExitAsync(runtime, timeout, cancellationToken);
    }

-    public async Task<bool> TryReconcileInactiveSessionAsync(Guid recordSessionId, CancellationToken cancellationToken = default)
+    public async Task<int> StopAllAndWaitAsync(
+        TimeSpan gracefulTimeout,
+        TimeSpan forceKillTimeout,
+        CancellationToken cancellationToken = default)
+    {
+        var runtimes = _processes.Values.ToArray();
+        foreach (var runtime in runtimes)
+        {
+            // Mark the captured runtime before looking it up again. The process may exit
+            // between the snapshot and the stop signal, but its exit handler must still
+            // observe that this was an application shutdown rather than a stream failure.
+            runtime.MarkShutdownRequested();
+        }
+
+        await Task.WhenAll(runtimes.Select(runtime =>
+            RequestStopAsync(
+                runtime.RecordSessionId,
+                markAsCompletedOnExit: true,
+                cancellationToken,
+                shutdownRequested: true)));
+
+        await WaitForRuntimeCompletionsAsync(runtimes, gracefulTimeout, cancellationToken);
+
+        foreach (var runtime in runtimes.Where(static runtime => !runtime.ExitCompletion.Task.IsCompleted))
+        {
+            var process = runtime.Process;
+            try
+            {
+                if (process is not null && !process.HasExited)
+                {
+                    runtime.MarkForceKilled();
+                    process.Kill(true);
+                }
+            }
+            catch (InvalidOperationException)
+            {
+            }
+        }
+
+        await WaitForRuntimeCompletionsAsync(runtimes, forceKillTimeout, CancellationToken.None);
+
+        _shutdownCts.Cancel();
+        foreach (var process in _postProcessProcesses.Values.ToArray())
+        {
+            try
+            {
+                if (!process.HasExited)
+                {
+                    process.Kill(true);
+                }
+            }
+            catch (InvalidOperationException)
+            {
+            }
+        }
+
+        await WaitForAllProcessesAsync(forceKillTimeout, CancellationToken.None);
+        return runtimes.Length;
+    }
+
+    private static async Task WaitForRuntimeCompletionsAsync(
+        IReadOnlyCollection<SessionProcessRuntime> runtimes,
+        TimeSpan timeout,
+        CancellationToken cancellationToken)
+    {
+        if (runtimes.Count == 0 || runtimes.All(static runtime => runtime.ExitCompletion.Task.IsCompleted))
+        {
+            return;
+        }
+
+        var completionTask = Task.WhenAll(runtimes.Select(static runtime => runtime.ExitCompletion.Task));
+        using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+        var delayTask = Task.Delay(timeout, timeoutCts.Token);
+        if (await Task.WhenAny(completionTask, delayTask) == completionTask)
+        {
+            timeoutCts.Cancel();
+            await completionTask;
+        }
+    }
+
+    private async Task WaitForAllProcessesAsync(TimeSpan timeout, CancellationToken cancellationToken)
+    {
+        var deadline = DateTimeOffset.UtcNow + timeout;
+        while ((!_processes.IsEmpty || !_postProcessProcesses.IsEmpty || !_postProcessStates.IsEmpty) &&
+               DateTimeOffset.UtcNow < deadline)
+        {
+            await Task.Delay(TimeSpan.FromMilliseconds(200), cancellationToken);
+        }
+    }
+
+    public Task<bool> TryReconcileInactiveSessionAsync(Guid recordSessionId, CancellationToken cancellationToken = default) =>
+        ReconcileInactiveSessionAsync(recordSessionId, allowTerminalSession: false, cancellationToken);
+
+    public async Task<int> RecoverOrphanedTerminalSessionTasksAsync(CancellationToken cancellationToken = default)
+    {
+        if (!await _orphanRecoveryGate.WaitAsync(0, cancellationToken))
+        {
+            return 0;
+        }
+
+        try
+        {
+            using var scope = _serviceScopeFactory.CreateScope();
+            var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
+            var candidateIds = await dbContext.RecordSessions
+                .AsNoTracking()
+                .Where(session =>
+                    (session.Status == RecordSessionStatus.Completed ||
+                     session.Status == RecordSessionStatus.Failed ||
+                     session.Status == RecordSessionStatus.Stopped) &&
+                    session.RecordTasks.Any(task =>
+                        task.Status == RecordTaskStatus.Starting ||
+                        task.Status == RecordTaskStatus.Running ||
+                        task.Status == RecordTaskStatus.Stopping))
+                .OrderBy(session => session.UpdatedAt)
+                .Select(session => session.Id)
+                .Take(20)
+                .ToListAsync(cancellationToken);
+
+            var recovered = 0;
+            foreach (var candidateId in candidateIds)
+            {
+                if (await ReconcileInactiveSessionAsync(candidateId, allowTerminalSession: true, cancellationToken))
+                {
+                    recovered++;
+                }
+            }
+
+            return recovered;
+        }
+        finally
+        {
+            _orphanRecoveryGate.Release();
+        }
+    }
+
+    public async Task<bool> TryRecoverOrphanedTerminalTaskAsync(
+        Guid recordTaskId,
+        CancellationToken cancellationToken = default)
+    {
+        await _orphanRecoveryGate.WaitAsync(cancellationToken);
+        try
+        {
+            using var lookupScope = _serviceScopeFactory.CreateScope();
+            var lookupDbContext = lookupScope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
+            var sessionId = await lookupDbContext.RecordTasks
+                .AsNoTracking()
+                .Where(task =>
+                    task.Id == recordTaskId &&
+                    (task.Status == RecordTaskStatus.Starting ||
+                     task.Status == RecordTaskStatus.Running ||
+                     task.Status == RecordTaskStatus.Stopping) &&
+                    task.RecordSession != null &&
+                    (task.RecordSession.Status == RecordSessionStatus.Completed ||
+                     task.RecordSession.Status == RecordSessionStatus.Failed ||
+                     task.RecordSession.Status == RecordSessionStatus.Stopped))
+                .Select(task => (Guid?)task.RecordSessionId)
+                .FirstOrDefaultAsync(cancellationToken);
+
+            return sessionId.HasValue &&
+                   await ReconcileInactiveSessionAsync(sessionId.Value, allowTerminalSession: true, cancellationToken);
+        }
+        finally
+        {
+            _orphanRecoveryGate.Release();
+        }
+    }
+
+    private async Task<bool> ReconcileInactiveSessionAsync(
+        Guid recordSessionId,
+        bool allowTerminalSession,
+        CancellationToken cancellationToken)
    {
        if (IsRunning(recordSessionId))
        {
@@ -330,11 +522,53 @@ public sealed partial class FfmpegService : IFfmpegService
            .ThenInclude(item => item.Result)
            .FirstOrDefaultAsync(item => item.Id == recordSessionId, cancellationToken);

-        if (recordSession is null || !IsActiveSessionStatus(recordSession.Status))
+        if (recordSession is null ||
+            (!IsActiveSessionStatus(recordSession.Status) &&
+             !(allowTerminalSession && IsTerminalSessionStatus(recordSession.Status))))
        {
            return false;
        }

+        var hasActiveTasks = recordSession.RecordTasks.Any(task => IsActiveTaskStatus(task.Status));
+        if (allowTerminalSession && !hasActiveTasks)
+        {
+            return false;
+        }
+
+        var discoveredSegments = DiscoverRecoverableSegments(recordSession.OutputPathPattern, recordSession.OutputFormat, recordSession.SaveMode);
+        var existingSegmentIndexes = recordSession.RecordTasks
+            .Select(static task => task.SegmentIndex)
+            .ToHashSet();
+        var addedMissingTasks = false;
+        foreach (var segment in discoveredSegments.Where(segment => !existingSegmentIndexes.Contains(segment.SegmentIndex)))
+        {
+            var discoveredAt = File.GetLastWriteTimeUtc(segment.RecorderPath);
+            var createdAt = discoveredAt == DateTime.MinValue
+                ? DateTimeOffset.UtcNow
+                : new DateTimeOffset(DateTime.SpecifyKind(discoveredAt, DateTimeKind.Utc));
+            var missingTask = new RecordTask(
+                recordSession.LiveRoomId,
+                recordSession.Id,
+                segment.SegmentIndex,
+                recordSession.PreferredQuality,
+                recordSession.OutputFormat,
+                createdAt);
+            var recoveryStartedAt = DateTimeOffset.UtcNow;
+            missingTask.MarkStarting(recordSession.StreamUrl ?? string.Empty, segment.OutputPath, recoveryStartedAt);
+            missingTask.MarkRunning(recoveryStartedAt);
+            await dbContext.RecordTasks.AddAsync(missingTask, cancellationToken);
+            recordSession.RecordTasks.Add(missingTask);
+            existingSegmentIndexes.Add(segment.SegmentIndex);
+            addedMissingTasks = true;
+        }
+
+        // RecordResult is upserted with raw SQL below, so newly discovered tasks must
+        // exist first to satisfy the RecordTaskId foreign key.
+        if (addedMissingTasks)
+        {
+            await dbContext.SaveChangesAsync(cancellationToken);
+        }
+
        var settings = await settingsService.GetAsync(cancellationToken);
        var endedAt = DateTimeOffset.UtcNow;
        var tasks = recordSession.RecordTasks
@@ -345,6 +579,7 @@ public sealed partial class FfmpegService : IFfmpegService
        var anyUsableOutput = tasks.Any(static item => item.Status == RecordTaskStatus.Completed);
        var hasBackgroundPostProcessing = false;
        string? sessionFinalizationError = null;
+        var newlyCompletedTasks = new List<(RecordTask Task, string OutputPath)>();

        foreach (var task in tasks)
        {
@@ -372,11 +607,21 @@ public sealed partial class FfmpegService : IFfmpegService
                cancellationToken);
            var effectiveOutputPath = finalizationResult.OutputPath;
            var taskFinalizationError = finalizationResult.ErrorMessage;
+            var mediaValidation = await ValidateMediaArtifactAsync(
+                effectiveOutputPath,
+                recordSession.OutputFormat,
+                cancellationToken);
+            durationSeconds = mediaValidation.DurationSeconds;
            if (!IsLowStoragePauseError(taskFinalizationError))
            {
                sessionFinalizationError ??= taskFinalizationError;
            }

+            if (string.IsNullOrWhiteSpace(taskFinalizationError) && !mediaValidation.IsValid)
+            {
+                sessionFinalizationError ??= mediaValidation.ErrorMessage;
+            }
+
            var fileSize = CalculateFileSize(effectiveOutputPath);
            var danmakuPath = task.Result?.DanmakuFilePath ?? GuessDanmakuPath(task.OutputFilePath);
            var danmakuCount = CountDanmakuMessages(danmakuPath);
@@ -388,23 +633,34 @@ public sealed partial class FfmpegService : IFfmpegService
            }
            else if (!string.IsNullOrWhiteSpace(taskFinalizationError))
            {
-                task.MarkFailed(taskFinalizationError, endedAt);
+                task.MarkFailed(taskFinalizationError, endedAt, durationSeconds);
            }
-            else if (HasUsableOutput(effectiveOutputPath, fileSize))
+            else if (mediaValidation.IsValid)
            {
                task.MarkCompleted(endedAt, durationSeconds);
                anyUsableOutput = true;
+                newlyCompletedTasks.Add((task, effectiveOutputPath));
            }
            else
            {
-                task.MarkStopped(endedAt, durationSeconds, "Recording process was no longer running when the session was reconciled.");
+                task.MarkFailed(mediaValidation.ErrorMessage!, endedAt, durationSeconds);
            }

-            await UpsertRecordResultAsync(task, dbContext, effectiveOutputPath, fileSize, durationSeconds, danmakuPath, danmakuCount, endedAt, cancellationToken);
+            await UpsertRecordResultAsync(
+                task,
+                dbContext,
+                effectiveOutputPath,
+                fileSize,
+                durationSeconds,
+                danmakuPath,
+                danmakuCount,
+                endedAt,
+                mediaValidatedForDispatch: mediaValidation.IsValid,
+                cancellationToken: cancellationToken);
            ClearPostProcessState(task.Id);
        }

-        recordSession.SyncSegmentCount(tasks.Count, endedAt);
+        recordSession.SyncSegmentCount(tasks.Count == 0 ? 0 : tasks.Max(static task => task.SegmentIndex), endedAt);
        if (!string.IsNullOrWhiteSpace(sessionFinalizationError))
        {
            recordSession.MarkFailed(sessionFinalizationError, endedAt);
@@ -425,6 +681,26 @@ public sealed partial class FfmpegService : IFfmpegService
        }

        await dbContext.SaveChangesAsync(cancellationToken);
+
+        if (newlyCompletedTasks.Count > 0 && recordSession.LiveRoom is not null)
+        {
+            var completionDispatchService = scope.ServiceProvider.GetRequiredService<CompletionDispatchService>();
+            foreach (var completed in newlyCompletedTasks)
+            {
+                try
+                {
+                    await completionDispatchService.TryDispatchTaskAsync(completed.Task.Id, cancellationToken);
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogWarning(
+                        ex,
+                        "Recovered segment completion hooks failed for task {RecordTaskId}",
+                        completed.Task.Id);
+                }
+            }
+        }
+
        return true;
    }

@@ -459,8 +735,8 @@ public sealed partial class FfmpegService : IFfmpegService
            return false;
        }

-        var recorderOutputPath = ResolveManualFinalizeSourcePath(recordTask, recordTask.RecordSession);
-        if (!File.Exists(recorderOutputPath))
+        var recorderOutputPaths = ResolveManualFinalizeSourcePaths(recordTask, recordTask.RecordSession);
+        if (recorderOutputPaths.Count == 0)
        {
            return false;
        }
@@ -548,9 +824,10 @@ public sealed partial class FfmpegService : IFfmpegService
                    settings.Mp4FinalizeTimeoutMinutes,
                    syntheticSessionId,
                    syntheticTaskId,
-                    absoluteSourcePath,
+                    [absoluteSourcePath],
                    absoluteTargetPath,
                    expectedDurationSeconds: null,
+                    segmentsManifestPath: null,
                    CancellationToken.None);

                using var scope = _serviceScopeFactory.CreateScope();
@@ -610,32 +887,30 @@ public sealed partial class FfmpegService : IFfmpegService
        var settings = await settingsService.GetAsync(cancellationToken);

        var storageCheck = _storageGuardService.CheckCanStartOrResume(settings);
-        if (storageCheck.ShouldPauseActive)
-        {
-            await logService.WriteAsync(
-                SystemLogLevel.Warning,
-                "Storage",
-                "MP4 finalization remains paused because storage tier is Red.",
-                storageCheck.Message,
-                cancellationToken: cancellationToken);
-            return 0;
-        }
-
        var candidates = await dbContext.RecordTasks
            .Include(item => item.RecordSession)
+            .ThenInclude(item => item!.RecordTasks)
            .Include(item => item.Result)
            .Where(item => item.OutputFormat == RecordOutputFormat.Mp4 &&
                           (item.Status == RecordTaskStatus.Processing ||
-                             item.Status == RecordTaskStatus.Completed))
+                            item.Status == RecordTaskStatus.Completed ||
+                            item.Status == RecordTaskStatus.Failed &&
+                            item.ErrorMessage == FfprobeUnreadableArtifactError))
+            // Deployment-paused work must not be starved by a large backlog of
+            // older legacy failures that may no longer have local source files.
+            .OrderBy(static item => item.Status == RecordTaskStatus.Processing
+                ? 0
+                : item.Status == RecordTaskStatus.Completed
+                    ? 1
+                    : 2)
+            .ThenBy(static item => item.UpdatedAt)
+            .Take(100)
            .ToListAsync(cancellationToken);

-        candidates = candidates
-            .OrderBy(static item => item.UpdatedAt)
-            .Take(100)
-            .ToList();
-
        var queuedTaskIds = new List<Guid>();
+        var recoveredValidatedTaskIds = new List<Guid>();
        var repairedInterruptedTasks = 0;
+        var repairedLegacyShutdownFailures = 0;
        var now = DateTimeOffset.UtcNow;
        foreach (var candidate in candidates)
        {
@@ -645,6 +920,58 @@ public sealed partial class FfmpegService : IFfmpegService
                continue;
            }

+            if (candidate.Status == RecordTaskStatus.Failed)
+            {
+                var recoverySources = candidate.RecordSession is null
+                    ? Array.Empty<string>()
+                    : ResolveManualFinalizeSourcePaths(candidate, candidate.RecordSession);
+                if (recoverySources.Count > 0)
+                {
+                    candidate.MarkProcessing(
+                        "A deployment-interrupted MP4 finalization was recovered and queued after restart.",
+                        now);
+                    candidate.RecordSession!.MarkStopped(
+                        now,
+                        "A deployment-interrupted MP4 finalization is continuing in the background.");
+                    queuedTaskIds.Add(candidate.Id);
+                    repairedLegacyShutdownFailures++;
+                    continue;
+                }
+
+                var existingOutputPath = candidate.Result?.FilePath ?? candidate.OutputFilePath;
+                var mediaValidation = await ValidateMediaArtifactAsync(
+                    existingOutputPath,
+                    candidate.OutputFormat,
+                    cancellationToken);
+                if (!mediaValidation.IsValid)
+                {
+                    continue;
+                }
+
+                candidate.MarkCompleted(now, mediaValidation.DurationSeconds);
+                if (candidate.RecordSession is not null &&
+                    candidate.RecordSession.RecordTasks.All(static task =>
+                        task.Status is RecordTaskStatus.Completed or RecordTaskStatus.Stopped))
+                {
+                    candidate.RecordSession.MarkCompleted(now);
+                }
+
+                await UpsertRecordResultAsync(
+                    candidate,
+                    dbContext,
+                    existingOutputPath,
+                    CalculateFileSize(existingOutputPath),
+                    mediaValidation.DurationSeconds,
+                    candidate.Result?.DanmakuFilePath,
+                    candidate.Result?.DanmakuMessageCount ?? 0,
+                    now,
+                    mediaValidatedForDispatch: true,
+                    cancellationToken: cancellationToken);
+                recoveredValidatedTaskIds.Add(candidate.Id);
+                repairedLegacyShutdownFailures++;
+                continue;
+            }
+
            if (!NeedsInterruptedMp4Finalization(candidate))
            {
                continue;
@@ -655,13 +982,24 @@ public sealed partial class FfmpegService : IFfmpegService
            repairedInterruptedTasks++;
        }

-        if (repairedInterruptedTasks > 0)
+        if (repairedInterruptedTasks > 0 || repairedLegacyShutdownFailures > 0)
        {
            await dbContext.SaveChangesAsync(cancellationToken);
        }

+        if (recoveredValidatedTaskIds.Count > 0)
+        {
+            var completionDispatchService = scope.ServiceProvider.GetRequiredService<CompletionDispatchService>();
+            foreach (var taskId in recoveredValidatedTaskIds)
+            {
+                await completionDispatchService.TryDispatchTaskAsync(taskId, cancellationToken);
+            }
+        }
+
        var started = 0;
-        foreach (var taskId in queuedTaskIds.Take(20))
+        foreach (var taskId in queuedTaskIds
+                     .Where(taskId => !IsTaskUnderPostProcessing(taskId))
+                     .Take(20))
        {
            if (await StartManualFinalizeTaskAsync(taskId, cancellationToken))
            {
@@ -669,13 +1007,13 @@ public sealed partial class FfmpegService : IFfmpegService
            }
        }

-        if (started > 0)
+        if (started > 0 || repairedLegacyShutdownFailures > 0)
        {
            await logService.WriteAsync(
                SystemLogLevel.Info,
                "Storage",
-                "Storage is available. Resumed paused MP4 finalization tasks.",
-                $"count={started}; repairedInterrupted={repairedInterruptedTasks}; {storageCheck.Message}",
+                "Resumed paused MP4 finalizations and repaired deployment-interrupted recording results.",
+                $"count={started}; repairedInterrupted={repairedInterruptedTasks}; repairedLegacyShutdown={repairedLegacyShutdownFailures}; {storageCheck.Message}",
                cancellationToken: cancellationToken);
        }

@@ -699,12 +1037,23 @@ public sealed partial class FfmpegService : IFfmpegService
            return false;
        }

-        var recorderOutputPath = ResolveManualFinalizeSourcePath(recordTask, recordTask.RecordSession);
-        return File.Exists(recorderOutputPath);
+        return ResolveManualFinalizeSourcePaths(recordTask, recordTask.RecordSession).Count > 0;
    }

    private static string ResolveManualFinalizeSourcePath(RecordTask recordTask, RecordSession recordSession)
+        => ResolveManualFinalizeSourcePaths(recordTask, recordSession).FirstOrDefault() ?? string.Empty;
+
+    private static IReadOnlyList<string> ResolveManualFinalizeSourcePaths(RecordTask recordTask, RecordSession recordSession)
    {
+        if (!string.IsNullOrWhiteSpace(recordTask.OutputFilePath))
+        {
+            var manifestPaths = ReadRecorderSegmentsManifest(recordTask.OutputFilePath);
+            if (manifestPaths.Count > 0)
+            {
+                return manifestPaths;
+            }
+        }
+
        var resultPath = recordTask.Result?.FilePath;
        if (!string.IsNullOrWhiteSpace(resultPath) &&
            resultPath.EndsWith(".ts", StringComparison.OrdinalIgnoreCase))
@@ -712,15 +1061,16 @@ public sealed partial class FfmpegService : IFfmpegService
            var normalizedResultPath = NormalizeAbsolutePath(resultPath);
            if (File.Exists(normalizedResultPath))
            {
-                return normalizedResultPath;
+                return [normalizedResultPath];
            }
        }

-        return NormalizeAbsolutePath(
+        var defaultPath = NormalizeAbsolutePath(
            GetRecorderOutputPath(
                recordTask.OutputFilePath ?? resultPath ?? string.Empty,
                recordSession.OutputFormat,
                recordSession.SaveMode));
+        return File.Exists(defaultPath) ? [defaultPath] : [];
    }

    private async Task PersistStartupProfileAsync(SessionProcessRuntime runtime, CancellationToken cancellationToken)
@@ -731,7 +1081,7 @@ public sealed partial class FfmpegService : IFfmpegService
            SystemLogLevel.Info,
            "FFmpeg",
            $"ffmpeg input profile={runtime.InputOptionProfile}.",
-            detail: $"stream={runtime.SelectedProtocol}:{runtime.SelectedQuality}; videoCodec={runtime.SelectedVideoCodec ?? "unknown"}; compatibilityRetry={runtime.HasRetriedWithCompatibilityProfile}; refreshRetry={runtime.HasRetriedWithRefreshedStream}",
+            detail: $"stream={runtime.SelectedProtocol}:{runtime.SelectedQuality}; videoCodec={runtime.SelectedVideoCodec ?? "unknown"}; recoveryEncoder={runtime.RecoveryVideoEncoder.Kind}; recoveryDevice={runtime.RecoveryVideoEncoder.DevicePath ?? "default"}; attempt={runtime.RetryAttemptCount}/{MaxInSessionRetryAttempts}; compatibilityRetry={runtime.HasRetriedWithCompatibilityProfile}; refreshRetry={runtime.HasRetriedWithRefreshedStream}; alternateProtocolRetry={runtime.HasRetriedWithAlternateProtocol}; softwareEncoderRetry={runtime.RecoveryContext.HasRetriedWithSoftwareEncoder}",
            liveRoomId: runtime.LiveRoomId,
            recordSessionId: runtime.RecordSessionId,
            recordTaskId: runtime.CurrentTaskId,
@@ -756,14 +1106,25 @@ public sealed partial class FfmpegService : IFfmpegService
        return false;
    }

-    private async Task RequestStopAsync(Guid recordSessionId, bool markAsCompletedOnExit, CancellationToken cancellationToken)
+    private async Task RequestStopAsync(
+        Guid recordSessionId,
+        bool markAsCompletedOnExit,
+        CancellationToken cancellationToken,
+        bool shutdownRequested = false)
    {
        if (!_processes.TryGetValue(recordSessionId, out var runtime))
        {
            return;
        }

-        runtime.MarkStopRequested(markAsCompletedOnExit);
+        if (shutdownRequested)
+        {
+            runtime.MarkShutdownRequested();
+        }
+        else
+        {
+            runtime.MarkStopRequested(markAsCompletedOnExit);
+        }
        var process = runtime.Process;
        if (process is null)
        {
@@ -919,6 +1280,41 @@ public sealed partial class FfmpegService : IFfmpegService
        }
    }

+    private TimeSpan RecordRuntimeFailure(Guid liveRoomId)
+    {
+        var now = DateTimeOffset.UtcNow;
+        var state = _roomRuntimeFailureStates.AddOrUpdate(
+            liveRoomId,
+            _ => new RoomStartupFailureState { ConsecutiveFailures = 1, FirstFailureAt = now, LastFailureAt = now },
+            (_, existing) =>
+            {
+                existing.ConsecutiveFailures++;
+                existing.LastFailureAt = now;
+                return existing;
+            });
+
+        var backoffSeconds = RuntimeFailureBaseBackoff.TotalSeconds *
+                             Math.Pow(2, Math.Min(state.ConsecutiveFailures - 1, 5));
+        var backoff = TimeSpan.FromSeconds(Math.Min(backoffSeconds, RuntimeFailureMaxBackoff.TotalSeconds));
+        _logger.LogWarning(
+            "Short runtime failure backoff for room {LiveRoomId}: {ConsecutiveFailures} consecutive failures, next poll delayed by {BackoffSeconds:F0}s",
+            liveRoomId,
+            state.ConsecutiveFailures,
+            backoff.TotalSeconds);
+        return backoff;
+    }
+
+    private void ResetRuntimeFailureBackoff(Guid liveRoomId)
+    {
+        if (_roomRuntimeFailureStates.TryRemove(liveRoomId, out var state) && state.ConsecutiveFailures > 1)
+        {
+            _logger.LogInformation(
+                "Short runtime failure backoff reset for room {LiveRoomId} after {ConsecutiveFailures} failures",
+                liveRoomId,
+                state.ConsecutiveFailures);
+        }
+    }
+
    /// <summary>
    /// Returns true if the failure notification for this room should be throttled
    /// (i.e., at most one notification per <see cref="StartupFailureNotificationCooldown"/>).