feat: improve recording recovery and upload workflow

2026-08-10 11:14:34 +08:00
parent e0d3969e46
commit 19d358b312
58 changed files with 5979 additions and 586 deletions
@@ -21,6 +21,7 @@ namespace LiveRecorder.Infrastructure.Services;

 public sealed partial class FfmpegService : IFfmpegService
 {
+    private const string ArtifactRepairMarker = "[artifact-repair]";
    private static readonly Regex SegmentOpeningRegex = new(
        """Opening '([^']+)' for writing""",
        RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
@@ -32,6 +33,7 @@ public sealed partial class FfmpegService : IFfmpegService
    private static readonly TimeSpan RuntimeFailureBaseBackoff = TimeSpan.FromSeconds(15);

    private readonly ConcurrentDictionary<Guid, SessionProcessRuntime> _processes = new();
+    private readonly ConcurrentDictionary<Guid, SessionTransitionRuntime> _sessionTransitions = new();
    private readonly ConcurrentDictionary<Guid, PostProcessRuntimeEntry> _postProcessStates = new();
    private readonly ConcurrentDictionary<Guid, Process> _postProcessProcesses = new();
    private readonly ConcurrentDictionary<Guid, RoomStartupFailureState> _roomStartupFailureStates = new();
@@ -62,7 +64,16 @@ public sealed partial class FfmpegService : IFfmpegService
        _logger = logger;
    }

-    public bool IsRunning(Guid recordSessionId) => _processes.ContainsKey(recordSessionId);
+    public bool IsRunning(Guid recordSessionId) =>
+        IsSessionRuntimeActive(
+            _processes.ContainsKey(recordSessionId),
+            _sessionTransitions.ContainsKey(recordSessionId));
+
+    internal static bool IsSessionRuntimeActive(bool hasProcess, bool hasTransition) =>
+        hasProcess || hasTransition;
+
+    internal static bool ShouldReuseRecoveryTask(bool hasMedia, RecordTaskStatus status) =>
+        !hasMedia && status is RecordTaskStatus.Pending or RecordTaskStatus.Starting or RecordTaskStatus.Running or RecordTaskStatus.Stopping;

    public IReadOnlyDictionary<Guid, RecordTaskRuntimeState> GetTaskRuntimeStates(IReadOnlyCollection<Guid> recordTaskIds)
    {
@@ -83,6 +94,120 @@ public sealed partial class FfmpegService : IFfmpegService
        return snapshot;
    }

+    public async Task<int> ResumeRecoveringSessionsAsync(CancellationToken cancellationToken = default)
+    {
+        using var lookupScope = _serviceScopeFactory.CreateScope();
+        var lookupDb = lookupScope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
+        var candidates = await lookupDb.RecordSessions
+            .AsNoTracking()
+            .Where(item => item.Status == RecordSessionStatus.Starting &&
+                           item.ErrorMessage != null &&
+                           item.ErrorMessage.StartsWith("[runtime-recovery]"))
+            .OrderBy(item => item.UpdatedAt)
+            .Select(item => item.Id)
+            .Take(20)
+            .ToListAsync(cancellationToken);
+
+        var resumed = 0;
+        foreach (var sessionId in candidates)
+        {
+            if (IsRunning(sessionId))
+            {
+                continue;
+            }
+
+            var transition = new SessionTransitionRuntime(sessionId);
+            if (!_sessionTransitions.TryAdd(sessionId, transition))
+            {
+                transition.Dispose();
+                continue;
+            }
+
+            try
+            {
+                using var scope = _serviceScopeFactory.CreateScope();
+                var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
+                var session = await dbContext.RecordSessions
+                    .Include(item => item.LiveRoom)
+                    .Include(item => item.RecordTasks)
+                    .ThenInclude(item => item.Result)
+                    .FirstOrDefaultAsync(item => item.Id == sessionId, cancellationToken);
+                if (session?.LiveRoom is null || session.Status != RecordSessionStatus.Starting)
+                {
+                    continue;
+                }
+
+                if (string.IsNullOrWhiteSpace(session.OutputPathPattern))
+                {
+                    _logger.LogWarning("Recovering session {RecordSessionId} has no output path pattern and cannot be resumed.", session.Id);
+                    continue;
+                }
+
+                var adapterFactory = scope.ServiceProvider.GetRequiredService<ILivePlatformAdapterFactory>();
+                var adapter = adapterFactory.GetByPlatform(session.LiveRoom.Platform);
+                var liveStatus = await adapter.GetLiveStatusAsync(session.LiveRoom.RoomId, cancellationToken);
+                if (!liveStatus.IsLive)
+                {
+                    continue;
+                }
+
+                var stream = await adapter.GetStreamUrlAsync(session.LiveRoom.RoomId, session.PreferredQuality, cancellationToken);
+                var latest = session.RecordTasks
+                    .OrderByDescending(item => item.SegmentIndex)
+                    .ThenByDescending(item => item.CreatedAt)
+                    .FirstOrDefault();
+                var hasMedia = latest?.Result is { DurationSeconds: > 0 } result && File.Exists(result.FilePath);
+                RecordTask task;
+                if (latest is not null && ShouldReuseRecoveryTask(hasMedia, latest.Status))
+                {
+                    task = latest;
+                }
+                else
+                {
+                    var nextIndex = Math.Max(1, (latest?.SegmentIndex ?? 0) + 1);
+                    task = new RecordTask(session.LiveRoomId, session.Id, nextIndex, session.PreferredQuality, session.OutputFormat, DateTimeOffset.UtcNow);
+                    var output = NormalizeAbsolutePath(ResolveSegmentOutputPath(session.OutputPathPattern, session.SaveMode, nextIndex));
+                    task.MarkStarting(stream.SelectedUrl, output, DateTimeOffset.UtcNow);
+                    await dbContext.RecordTasks.AddAsync(task, cancellationToken);
+                }
+
+                var settingsResolver = scope.ServiceProvider.GetRequiredService<LiveRoomRecordingSettingsResolver>();
+                var recordingSettings = await settingsResolver.ResolveAsync(session.LiveRoom, cancellationToken);
+                var taskOutputPath = string.IsNullOrWhiteSpace(task.OutputFilePath)
+                    ? NormalizeAbsolutePath(ResolveSegmentOutputPath(session.OutputPathPattern, session.SaveMode, task.SegmentIndex))
+                    : task.OutputFilePath;
+                session.MarkStarting(stream.SelectedUrl, session.OutputPathPattern, DateTimeOffset.UtcNow);
+                session.ActivateSegment(task.SegmentIndex, DateTimeOffset.UtcNow);
+                task.MarkStarting(stream.SelectedUrl, taskOutputPath, DateTimeOffset.UtcNow);
+                await dbContext.SaveChangesAsync(cancellationToken);
+
+                await StartInternalAsync(
+                    session,
+                    task,
+                    stream,
+                    recordingSettings,
+                    InitialRecoveryContext with { AttemptCount = 1, HasRetriedWithRefreshedStream = true },
+                    cancellationToken);
+                session.MarkRunning(DateTimeOffset.UtcNow);
+                task.MarkRunning(DateTimeOffset.UtcNow);
+                await dbContext.SaveChangesAsync(cancellationToken);
+                resumed++;
+            }
+            catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
+            {
+                _logger.LogWarning(ex, "Failed to resume persisted recorder recovery for session {RecordSessionId}", sessionId);
+            }
+            finally
+            {
+                _sessionTransitions.TryRemove(sessionId, out _);
+                transition.Completion.TrySetResult(true);
+                transition.Dispose();
+            }
+        }
+
+        return resumed;
+    }
+
    public Task CompleteAsync(Guid recordSessionId, CancellationToken cancellationToken = default) =>
        RequestStopAsync(recordSessionId, markAsCompletedOnExit: true, cancellationToken);

@@ -229,6 +354,7 @@ public sealed partial class FfmpegService : IFfmpegService
            {
                if (!string.IsNullOrWhiteSpace(args.Data))
                {
+                    runtime.RememberCurlErrorLine(args.Data);
                    _logger.LogDebug("curl[{SessionId}] {Line}", recordSession.Id, args.Data);
                }
            };
@@ -283,6 +409,12 @@ public sealed partial class FfmpegService : IFfmpegService
    {
        if (!_processes.TryGetValue(recordSessionId, out var runtime))
        {
+            if (_sessionTransitions.TryGetValue(recordSessionId, out var transition))
+            {
+                transition.RequestStop(markAsCompletedOnExit);
+                return await WaitForTransitionAsync(transition, timeout, cancellationToken);
+            }
+
            return true;
        }

@@ -297,6 +429,12 @@ public sealed partial class FfmpegService : IFfmpegService
    {
        if (!_processes.TryGetValue(recordSessionId, out var runtime))
        {
+            if (_sessionTransitions.TryGetValue(recordSessionId, out var transition))
+            {
+                transition.RequestStop(markAsCompletedOnExit: false);
+                return await WaitForTransitionAsync(transition, timeout, cancellationToken);
+            }
+
            return true;
        }

@@ -340,6 +478,7 @@ public sealed partial class FfmpegService : IFfmpegService
        CancellationToken cancellationToken = default)
    {
        var runtimes = _processes.Values.ToArray();
+        var transitions = _sessionTransitions.Values.ToArray();
        foreach (var runtime in runtimes)
        {
            // Mark the captured runtime before looking it up again. The process may exit
@@ -355,7 +494,13 @@ public sealed partial class FfmpegService : IFfmpegService
                cancellationToken,
                shutdownRequested: true)));

+        foreach (var transition in transitions)
+        {
+            transition.RequestStop(markAsCompletedOnExit: true);
+        }
+
        await WaitForRuntimeCompletionsAsync(runtimes, gracefulTimeout, cancellationToken);
+        await WaitForTransitionCompletionsAsync(transitions, gracefulTimeout, cancellationToken);

        foreach (var runtime in runtimes.Where(static runtime => !runtime.ExitCompletion.Task.IsCompleted))
        {
@@ -391,7 +536,23 @@ public sealed partial class FfmpegService : IFfmpegService
        }

        await WaitForAllProcessesAsync(forceKillTimeout, CancellationToken.None);
-        return runtimes.Length;
+        return runtimes.Length + transitions.Length;
+    }
+
+    private static async Task WaitForTransitionCompletionsAsync(
+        IReadOnlyCollection<SessionTransitionRuntime> transitions,
+        TimeSpan timeout,
+        CancellationToken cancellationToken)
+    {
+        if (transitions.Count == 0 || transitions.All(static item => item.Completion.Task.IsCompleted)) return;
+        var completion = Task.WhenAll(transitions.Select(static item => item.Completion.Task));
+        using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+        var delay = Task.Delay(timeout, timeoutCts.Token);
+        if (await Task.WhenAny(completion, delay) == completion)
+        {
+            timeoutCts.Cancel();
+            await completion;
+        }
    }

    private static async Task WaitForRuntimeCompletionsAsync(
@@ -417,7 +578,7 @@ public sealed partial class FfmpegService : IFfmpegService
    private async Task WaitForAllProcessesAsync(TimeSpan timeout, CancellationToken cancellationToken)
    {
        var deadline = DateTimeOffset.UtcNow + timeout;
-        while ((!_processes.IsEmpty || !_postProcessProcesses.IsEmpty || !_postProcessStates.IsEmpty) &&
+        while ((!_processes.IsEmpty || !_sessionTransitions.IsEmpty || !_postProcessProcesses.IsEmpty || !_postProcessStates.IsEmpty) &&
               DateTimeOffset.UtcNow < deadline)
        {
            await Task.Delay(TimeSpan.FromMilliseconds(200), cancellationToken);
@@ -1020,6 +1181,215 @@ public sealed partial class FfmpegService : IFfmpegService
        return started;
    }

+    public async Task<bool> StartArtifactRepairAsync(
+        Guid recordTaskId,
+        CancellationToken cancellationToken = default)
+    {
+        if (_postProcessStates.ContainsKey(recordTaskId))
+        {
+            return false;
+        }
+
+        using var scope = _serviceScopeFactory.CreateScope();
+        var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
+        var settingsService = scope.ServiceProvider.GetRequiredService<ISystemSettingsService>();
+        var recordTask = await dbContext.RecordTasks
+            .Include(static item => item.RecordSession)
+            .Include(static item => item.Result)
+            .FirstOrDefaultAsync(item => item.Id == recordTaskId, cancellationToken);
+        if (recordTask?.Result is null || recordTask.RecordSession is null ||
+            recordTask.Status is not (RecordTaskStatus.Failed or RecordTaskStatus.Processing) ||
+            recordTask.Status == RecordTaskStatus.Processing &&
+            recordTask.ErrorMessage?.StartsWith(ArtifactRepairMarker, StringComparison.Ordinal) != true)
+        {
+            return false;
+        }
+
+        var sourcePath = NormalizeAbsolutePath(recordTask.Result.FilePath);
+        if (!File.Exists(sourcePath) || new FileInfo(sourcePath).Length <= 0)
+        {
+            return false;
+        }
+
+        var settings = await settingsService.GetAsync(cancellationToken);
+        var storage = _storageGuardService.CheckCanStartOrResume(settings);
+        if (!storage.IsAvailable || storage.AvailableBytes < Math.Max(storage.RequiredBytes, new FileInfo(sourcePath).Length))
+        {
+            return false;
+        }
+
+        var targetPath = BuildRecoveredArtifactPath(sourcePath, recordTaskId);
+        var now = DateTimeOffset.UtcNow;
+        if (!_postProcessStates.TryAdd(
+                recordTask.Id,
+                new PostProcessRuntimeEntry(
+                    recordTask.RecordSessionId,
+                    new RecordTaskRuntimeState(
+                        RecordTaskStatus.Processing,
+                        "Queued",
+                        0,
+                        $"Waiting to repair {Path.GetFileName(sourcePath)}"))))
+        {
+            return false;
+        }
+
+        try
+        {
+            recordTask.MarkProcessing(
+                $"{ArtifactRepairMarker} 正在生成非破坏性恢复文件：{Path.GetFileName(targetPath)}",
+                now);
+            await dbContext.SaveChangesAsync(cancellationToken);
+        }
+        catch
+        {
+            ClearPostProcessState(recordTask.Id);
+            throw;
+        }
+
+        _ = Task.Run(
+            async () => await RunArtifactRepairAsync(recordTaskId),
+            CancellationToken.None);
+        return true;
+    }
+
+    public async Task<int> ResumeArtifactRepairsAsync(CancellationToken cancellationToken = default)
+    {
+        using var scope = _serviceScopeFactory.CreateScope();
+        var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
+        var taskIds = await dbContext.RecordTasks
+            .AsNoTracking()
+            .Where(item => item.Status == RecordTaskStatus.Processing &&
+                           item.ErrorMessage != null &&
+                           item.ErrorMessage.StartsWith(ArtifactRepairMarker))
+            .OrderBy(static item => item.UpdatedAt)
+            .Select(static item => item.Id)
+            .Take(20)
+            .ToArrayAsync(cancellationToken);
+
+        var started = 0;
+        foreach (var taskId in taskIds)
+        {
+            if (await StartArtifactRepairAsync(taskId, cancellationToken))
+            {
+                started++;
+            }
+        }
+
+        return started;
+    }
+
+    private async Task RunArtifactRepairAsync(Guid recordTaskId)
+    {
+        try
+        {
+            using var scope = _serviceScopeFactory.CreateScope();
+            var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
+            var settingsService = scope.ServiceProvider.GetRequiredService<ISystemSettingsService>();
+            var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
+            var recordTask = await dbContext.RecordTasks
+                .Include(static item => item.RecordSession)
+                .ThenInclude(static item => item!.RecordTasks)
+                .Include(static item => item.Result)
+                .FirstOrDefaultAsync(item => item.Id == recordTaskId);
+            if (recordTask?.Result is null || recordTask.RecordSession is null)
+            {
+                return;
+            }
+
+            var sourcePath = NormalizeAbsolutePath(recordTask.Result.FilePath);
+            var targetPath = BuildRecoveredArtifactPath(sourcePath, recordTaskId);
+            var originalError = recordTask.ErrorMessage;
+            var settings = await settingsService.GetAsync();
+            SetPostProcessState(
+                recordTask.RecordSessionId,
+                recordTask.Id,
+                "Repairing",
+                null,
+                $"Repairing {Path.GetFileName(sourcePath)} without replacing the source");
+
+            var repair = await TryRepairArtifactFileAsync(
+                settings.FfmpegPath,
+                settings.MaxConcurrentFfmpegTranscodeTasks,
+                settings.Mp4FinalizeTimeoutMinutes,
+                recordTask.Id,
+                sourcePath,
+                targetPath,
+                _shutdownCts.Token);
+            if (_shutdownCts.IsCancellationRequested)
+            {
+                return;
+            }
+
+            var now = DateTimeOffset.UtcNow;
+            var metadata = string.IsNullOrWhiteSpace(repair.ErrorMessage)
+                ? await scope.ServiceProvider.GetRequiredService<IVideoMetadataService>()
+                    .ExtractMetadataAsync(targetPath)
+                : null;
+            if (metadata?.DurationSeconds is > 0 && !string.IsNullOrWhiteSpace(metadata.VideoCodec))
+            {
+                recordTask.MarkCompleted(now, metadata.DurationSeconds);
+                recordTask.Result.Update(
+                    targetPath,
+                    new FileInfo(targetPath).Length,
+                    metadata.DurationSeconds,
+                    recordTask.Result.DanmakuFilePath,
+                    recordTask.Result.DanmakuMessageCount,
+                    RecordTaskStatus.Completed,
+                    null);
+                recordTask.Result.ResetUploadForRecoveredArtifact();
+                if (recordTask.RecordSession.RecordTasks.All(static task =>
+                        task.Status is RecordTaskStatus.Completed or RecordTaskStatus.Stopped))
+                {
+                    recordTask.RecordSession.MarkCompleted(now);
+                }
+
+                await dbContext.SaveChangesAsync();
+                await logService.WriteAsync(
+                    SystemLogLevel.Info,
+                    "Recovery",
+                    "录制失败产物已修复，原文件已保留。",
+                    $"source={sourcePath}; recovered={targetPath}; originalError={originalError}",
+                    recordTask.LiveRoomId,
+                    recordTask.RecordSessionId,
+                    recordTask.Id);
+                SetPostProcessState(recordTask.RecordSessionId, recordTask.Id, "Completed", 100, "Recovered file is ready for manual upload");
+            }
+            else
+            {
+                var error = repair.ErrorMessage ?? "恢复输出仍无法识别有效视频流。";
+                recordTask.MarkFailed($"录制产物修复失败：{error}", now, recordTask.DurationSeconds);
+                await dbContext.SaveChangesAsync();
+                await logService.WriteAsync(
+                    SystemLogLevel.Warning,
+                    "Recovery",
+                    "录制失败产物修复未成功，原文件保持不变。",
+                    $"source={sourcePath}; error={error}",
+                    recordTask.LiveRoomId,
+                    recordTask.RecordSessionId,
+                    recordTask.Id);
+            }
+        }
+        catch (OperationCanceledException) when (_shutdownCts.IsCancellationRequested)
+        {
+            // The persisted Processing marker is intentionally kept for restart recovery.
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Artifact repair failed for task {RecordTaskId}", recordTaskId);
+        }
+        finally
+        {
+            ClearPostProcessState(recordTaskId);
+        }
+    }
+
+    private static string BuildRecoveredArtifactPath(string sourcePath, Guid recordTaskId)
+    {
+        var directory = Path.GetDirectoryName(sourcePath) ?? AppContext.BaseDirectory;
+        var stem = Path.GetFileNameWithoutExtension(sourcePath);
+        return Path.Combine(directory, $"{stem}.{recordTaskId.ToString("N")[..8]}.recovered.mp4");
+    }
+
    private static bool NeedsInterruptedMp4Finalization(RecordTask recordTask)
    {
        if (recordTask.Status != RecordTaskStatus.Completed ||
@@ -1081,7 +1451,7 @@ public sealed partial class FfmpegService : IFfmpegService
            SystemLogLevel.Info,
            "FFmpeg",
            $"ffmpeg input profile={runtime.InputOptionProfile}.",
-            detail: $"stream={runtime.SelectedProtocol}:{runtime.SelectedQuality}; videoCodec={runtime.SelectedVideoCodec ?? "unknown"}; recoveryEncoder={runtime.RecoveryVideoEncoder.Kind}; recoveryDevice={runtime.RecoveryVideoEncoder.DevicePath ?? "default"}; attempt={runtime.RetryAttemptCount}/{MaxInSessionRetryAttempts}; compatibilityRetry={runtime.HasRetriedWithCompatibilityProfile}; refreshRetry={runtime.HasRetriedWithRefreshedStream}; alternateProtocolRetry={runtime.HasRetriedWithAlternateProtocol}; softwareEncoderRetry={runtime.RecoveryContext.HasRetriedWithSoftwareEncoder}",
+            detail: $"stream={runtime.SelectedProtocol}:{runtime.SelectedQuality}; videoCodec={runtime.SelectedVideoCodec ?? "unknown"}; recoveryEncoder={runtime.RecoveryVideoEncoder.Kind}; recoveryDevice={runtime.RecoveryVideoEncoder.DevicePath ?? "default"}; attempt={runtime.RetryAttemptCount}; compatibilityRetry={runtime.HasRetriedWithCompatibilityProfile}; refreshRetry={runtime.HasRetriedWithRefreshedStream}; alternateProtocolRetry={runtime.HasRetriedWithAlternateProtocol}; softwareEncoderRetry={runtime.RecoveryContext.HasRetriedWithSoftwareEncoder}",
            liveRoomId: runtime.LiveRoomId,
            recordSessionId: runtime.RecordSessionId,
            recordTaskId: runtime.CurrentTaskId,
@@ -1106,6 +1476,24 @@ public sealed partial class FfmpegService : IFfmpegService
        return false;
    }

+    private static async Task<bool> WaitForTransitionAsync(
+        SessionTransitionRuntime transition,
+        TimeSpan timeout,
+        CancellationToken cancellationToken)
+    {
+        using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+        var delayTask = Task.Delay(timeout, timeoutCts.Token);
+        var completed = await Task.WhenAny(transition.Completion.Task, delayTask);
+        if (completed != transition.Completion.Task)
+        {
+            return false;
+        }
+
+        timeoutCts.Cancel();
+        await transition.Completion.Task;
+        return true;
+    }
+
    private async Task RequestStopAsync(
        Guid recordSessionId,
        bool markAsCompletedOnExit,
@@ -1114,6 +1502,11 @@ public sealed partial class FfmpegService : IFfmpegService
    {
        if (!_processes.TryGetValue(recordSessionId, out var runtime))
        {
+            if (_sessionTransitions.TryGetValue(recordSessionId, out var transition))
+            {
+                transition.RequestStop(markAsCompletedOnExit || shutdownRequested);
+            }
+
            return;
        }