feat: improve recording automation and task workflows

2026-04-23 23:18:11 +08:00
parent 1c892259a9
commit 23ead56781
88 changed files with 9579 additions and 1581 deletions
@@ -6,6 +6,8 @@ using LiveRecorder.Application.Abstractions.Notifications;
 using LiveRecorder.Application.Abstractions.Platforms;
 using LiveRecorder.Application.Abstractions.Recording;
 using LiveRecorder.Application.Abstractions.Settings;
+using LiveRecorder.Application.Abstractions.Storage;
+using LiveRecorder.Application.Services;
 using LiveRecorder.Domain.Entities;
 using LiveRecorder.Domain.Enums;
 using LiveRecorder.Infrastructure.Persistence;
@@ -22,19 +24,46 @@ public sealed partial class FfmpegService : IFfmpegService
        RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);

    private readonly ConcurrentDictionary<Guid, SessionProcessRuntime> _processes = new();
+    private readonly ConcurrentDictionary<Guid, PostProcessRuntimeEntry> _postProcessStates = new();
+    private readonly object _transcodeConcurrencyLock = new();
+    private readonly Queue<TaskCompletionSource<IDisposable>> _transcodeWaiters = new();
+    private int _activeTranscodeTasks;
+    private int _maxConcurrentTranscodeTasks = 1;
    private readonly IServiceScopeFactory _serviceScopeFactory;
+    private readonly IStorageGuardService _storageGuardService;
    private readonly ILogger<FfmpegService> _logger;

    public FfmpegService(
        IServiceScopeFactory serviceScopeFactory,
+        IStorageGuardService storageGuardService,
        ILogger<FfmpegService> logger)
    {
        _serviceScopeFactory = serviceScopeFactory;
+        _storageGuardService = storageGuardService;
        _logger = logger;
    }

    public bool IsRunning(Guid recordSessionId) => _processes.ContainsKey(recordSessionId);

+    public IReadOnlyDictionary<Guid, RecordTaskRuntimeState> GetTaskRuntimeStates(IReadOnlyCollection<Guid> recordTaskIds)
+    {
+        if (recordTaskIds.Count == 0)
+        {
+            return new Dictionary<Guid, RecordTaskRuntimeState>();
+        }
+
+        var snapshot = new Dictionary<Guid, RecordTaskRuntimeState>();
+        foreach (var taskId in recordTaskIds)
+        {
+            if (_postProcessStates.TryGetValue(taskId, out var state))
+            {
+                snapshot[taskId] = state.State;
+            }
+        }
+
+        return snapshot;
+    }
+
    public Task CompleteAsync(Guid recordSessionId, CancellationToken cancellationToken = default) =>
        RequestStopAsync(recordSessionId, markAsCompletedOnExit: true, cancellationToken);

@@ -42,11 +71,13 @@ public sealed partial class FfmpegService : IFfmpegService
        RecordSession recordSession,
        RecordTask initialTask,
        StreamUrlResult streamUrlResult,
+        RecordingExecutionSettings recordingSettings,
        CancellationToken cancellationToken = default) =>
        StartInternalAsync(
            recordSession,
            initialTask,
            streamUrlResult,
+            recordingSettings,
            inputOptionProfile: FfmpegInputOptionProfile.Baseline,
            hasRetriedWithCompatibilityProfile: false,
            hasRetriedWithRefreshedStream: false,
@@ -57,6 +88,7 @@ public sealed partial class FfmpegService : IFfmpegService
        RecordSession recordSession,
        RecordTask initialTask,
        StreamUrlResult streamUrlResult,
+        RecordingExecutionSettings recordingSettings,
        FfmpegInputOptionProfile inputOptionProfile,
        bool hasRetriedWithCompatibilityProfile,
        bool hasRetriedWithRefreshedStream,
@@ -66,6 +98,7 @@ public sealed partial class FfmpegService : IFfmpegService
        ArgumentNullException.ThrowIfNull(recordSession);
        ArgumentNullException.ThrowIfNull(initialTask);
        ArgumentNullException.ThrowIfNull(streamUrlResult);
+        ArgumentNullException.ThrowIfNull(recordingSettings);

        if (string.IsNullOrWhiteSpace(streamUrlResult.SelectedUrl) || string.IsNullOrWhiteSpace(recordSession.OutputPathPattern))
        {
@@ -106,10 +139,12 @@ public sealed partial class FfmpegService : IFfmpegService
            recordSession.SaveMode,
            initialTask.Id,
            Math.Max(1, initialTask.SegmentIndex),
-            initialTask.OutputFilePath ?? outputPathPattern,
+            ResolveRecorderSegmentOutputPath(outputPathPattern, recordSession.OutputFormat, recordSession.SaveMode, Math.Max(1, initialTask.SegmentIndex)),
            streamUrlResult.SelectedQuality,
            streamUrlResult.SelectedProtocol,
+            streamUrlResult.SelectedVideoCodec,
            streamUrlResult.InputHeaders,
+            recordingSettings,
            inputOptionProfile,
            hasRetriedWithCompatibilityProfile,
            hasRetriedWithRefreshedStream,
@@ -120,12 +155,14 @@ public sealed partial class FfmpegService : IFfmpegService
                     recorderOutputPath,
                     recordSession.OutputFormat,
                     recordSession.SaveMode,
-                     settings.RecordingTemplate,
-                     settings.EnableAutoReconnect,
-                     settings.ReconnectDelayMaxSeconds,
-                     settings.ReadWriteTimeoutMilliseconds,
-                     settings.SegmentDurationMinutes,
+                     recordingSettings.RecordingTemplate,
+                     recordingSettings.EnableAutoReconnect,
+                     recordingSettings.ReconnectDelayMaxSeconds,
+                     recordingSettings.ReadWriteTimeoutMilliseconds,
+                     recordingSettings.SegmentDurationMinutes,
                     streamUrlResult.InputHeaders,
+                     streamUrlResult.SelectedProtocol,
+                     streamUrlResult.SelectedVideoCodec,
                     inputOptionProfile))
        {
            process.StartInfo.ArgumentList.Add(argument);
@@ -191,6 +228,7 @@ public sealed partial class FfmpegService : IFfmpegService
            {
                if (!process.HasExited)
                {
+                    runtime.MarkForceKilled();
                    process.Kill(true);
                }
            }
@@ -204,7 +242,7 @@ public sealed partial class FfmpegService : IFfmpegService

    public async Task<bool> TryReconcileInactiveSessionAsync(Guid recordSessionId, CancellationToken cancellationToken = default)
    {
-        if (IsRunning(recordSessionId))
+        if (IsRunning(recordSessionId) || IsSessionUnderPostProcessing(recordSessionId))
        {
            return false;
        }
@@ -232,37 +270,39 @@ public sealed partial class FfmpegService : IFfmpegService
            .ToList();

        var anyUsableOutput = false;
-        string? finalizationError = null;
+        string? sessionFinalizationError = null;

        foreach (var task in tasks.Where(item => IsActiveTaskStatus(item.Status)))
        {
-            var effectiveOutputPath = task.OutputFilePath ?? string.Empty;
-            if (recordSession.SaveMode == RecordSaveMode.SingleFile &&
-                recordSession.OutputFormat == RecordOutputFormat.Mp4 &&
-                !string.IsNullOrWhiteSpace(recordSession.OutputPathPattern))
+            var durationSeconds = task.StartedAt.HasValue
+                ? (double?)Math.Max(0, (endedAt - task.StartedAt.Value).TotalSeconds)
+                : null;
+            var finalizationResult = await TryFinalizeTaskOutputAsync(
+                settings.FfmpegPath,
+                settings.MaxConcurrentFfmpegTranscodeTasks,
+                settings.Mp4FinalizeTimeoutMinutes,
+                recordSession,
+                task,
+                durationSeconds,
+                cancellationToken);
+            var effectiveOutputPath = finalizationResult.OutputPath;
+            var taskFinalizationError = finalizationResult.ErrorMessage;
+            if (!IsLowStoragePauseError(taskFinalizationError))
            {
-                var finalOutputPath = Path.IsPathRooted(recordSession.OutputPathPattern)
-                    ? recordSession.OutputPathPattern
-                    : Path.GetFullPath(recordSession.OutputPathPattern, AppContext.BaseDirectory);
-                var recorderOutputPath = GetRecorderOutputPath(finalOutputPath, recordSession.OutputFormat, recordSession.SaveMode);
-                if (File.Exists(recorderOutputPath))
-                {
-                    var finalizationResult = await TryFinalizeMp4Async(settings.FfmpegPath, recorderOutputPath, finalOutputPath);
-                    effectiveOutputPath = finalizationResult.OutputPath;
-                    finalizationError ??= finalizationResult.ErrorMessage;
-                }
+                sessionFinalizationError ??= taskFinalizationError;
            }

            var fileSize = CalculateFileSize(effectiveOutputPath);
            var danmakuPath = task.Result?.DanmakuFilePath ?? GuessDanmakuPath(task.OutputFilePath);
            var danmakuCount = CountDanmakuMessages(danmakuPath);
-            var durationSeconds = task.StartedAt.HasValue
-                ? (double?)Math.Max(0, (endedAt - task.StartedAt.Value).TotalSeconds)
-                : null;

-            if (!string.IsNullOrWhiteSpace(finalizationError))
+            if (IsLowStoragePauseError(taskFinalizationError))
            {
-                task.MarkFailed(finalizationError, endedAt);
+                task.MarkProcessing(taskFinalizationError, endedAt);
+            }
+            else if (!string.IsNullOrWhiteSpace(taskFinalizationError))
+            {
+                task.MarkFailed(taskFinalizationError, endedAt);
            }
            else if (HasUsableOutput(effectiveOutputPath, fileSize))
            {
@@ -274,13 +314,14 @@ public sealed partial class FfmpegService : IFfmpegService
                task.MarkStopped(endedAt, durationSeconds, "Recording process was no longer running when the session was reconciled.");
            }

-            UpsertRecordResult(task, dbContext, effectiveOutputPath, fileSize, durationSeconds, danmakuPath, danmakuCount, endedAt);
+            await UpsertRecordResultAsync(task, dbContext, effectiveOutputPath, fileSize, durationSeconds, danmakuPath, danmakuCount, endedAt, cancellationToken);
+            ClearPostProcessState(task.Id);
        }

        recordSession.SyncSegmentCount(tasks.Count, endedAt);
-        if (!string.IsNullOrWhiteSpace(finalizationError))
+        if (!string.IsNullOrWhiteSpace(sessionFinalizationError))
        {
-            recordSession.MarkFailed(finalizationError, endedAt);
+            recordSession.MarkFailed(sessionFinalizationError, endedAt);
        }
        else if (anyUsableOutput)
        {
@@ -295,6 +336,176 @@ public sealed partial class FfmpegService : IFfmpegService
        return true;
    }

+    public async Task<bool> StartManualFinalizeTaskAsync(Guid recordTaskId, CancellationToken cancellationToken = default)
+    {
+        if (_postProcessStates.ContainsKey(recordTaskId))
+        {
+            return false;
+        }
+
+        using var scope = _serviceScopeFactory.CreateScope();
+        var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
+        var recordTask = await dbContext.RecordTasks
+            .Include(item => item.RecordSession)
+            .Include(item => item.Result)
+            .FirstOrDefaultAsync(item => item.Id == recordTaskId, cancellationToken);
+
+        if (recordTask?.RecordSession is null)
+        {
+            return false;
+        }
+
+        if (recordTask.OutputFormat != RecordOutputFormat.Mp4 ||
+            IsActiveTaskStatus(recordTask.Status) ||
+            IsRunning(recordTask.RecordSessionId))
+        {
+            return false;
+        }
+
+        if (string.IsNullOrWhiteSpace(recordTask.OutputFilePath))
+        {
+            return false;
+        }
+
+        var recorderOutputPath = ResolveManualFinalizeSourcePath(recordTask, recordTask.RecordSession);
+        if (!File.Exists(recorderOutputPath))
+        {
+            return false;
+        }
+
+        SetPostProcessState(
+            recordTask.RecordSessionId,
+            recordTask.Id,
+            "Queued",
+            null,
+            $"Manual MP4 finalization queued for {Path.GetFileName(recordTask.OutputFilePath)}");
+
+        _ = Task.Run(
+            async () => await RunManualFinalizeTaskAsync(recordTask.Id),
+            CancellationToken.None);
+
+        return true;
+    }
+
+    public async Task<int> ResumePausedFinalizationsAsync(CancellationToken cancellationToken = default)
+    {
+        using var scope = _serviceScopeFactory.CreateScope();
+        var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
+        var settingsService = scope.ServiceProvider.GetRequiredService<ISystemSettingsService>();
+        var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
+        var settings = await settingsService.GetAsync(cancellationToken);
+
+        var storageCheck = _storageGuardService.CheckCanStartOrResume(settings);
+        if (!storageCheck.HasEnoughSpace)
+        {
+            await logService.WriteAsync(
+                SystemLogLevel.Warning,
+                "Storage",
+                "Paused MP4 finalization remains blocked because storage is below threshold.",
+                storageCheck.Message,
+                cancellationToken: cancellationToken);
+            return 0;
+        }
+
+        var candidates = await dbContext.RecordTasks
+            .Include(item => item.RecordSession)
+            .Include(item => item.Result)
+            .Where(item => item.OutputFormat == RecordOutputFormat.Mp4 &&
+                           (item.Status == RecordTaskStatus.Processing ||
+                            item.Status == RecordTaskStatus.Completed))
+            .OrderBy(static item => item.UpdatedAt)
+            .Take(100)
+            .ToListAsync(cancellationToken);
+
+        var queuedTaskIds = new List<Guid>();
+        var repairedInterruptedTasks = 0;
+        var now = DateTimeOffset.UtcNow;
+        foreach (var candidate in candidates)
+        {
+            if (candidate.Status == RecordTaskStatus.Processing)
+            {
+                queuedTaskIds.Add(candidate.Id);
+                continue;
+            }
+
+            if (!NeedsInterruptedMp4Finalization(candidate))
+            {
+                continue;
+            }
+
+            candidate.MarkProcessing("MP4 finalization was interrupted before completion. Re-queued after restart.", now);
+            queuedTaskIds.Add(candidate.Id);
+            repairedInterruptedTasks++;
+        }
+
+        if (repairedInterruptedTasks > 0)
+        {
+            await dbContext.SaveChangesAsync(cancellationToken);
+        }
+
+        var started = 0;
+        foreach (var taskId in queuedTaskIds.Take(20))
+        {
+            if (await StartManualFinalizeTaskAsync(taskId, cancellationToken))
+            {
+                started++;
+            }
+        }
+
+        if (started > 0)
+        {
+            await logService.WriteAsync(
+                SystemLogLevel.Info,
+                "Storage",
+                "Storage is available. Resumed paused MP4 finalization tasks.",
+                $"count={started}; repairedInterrupted={repairedInterruptedTasks}; {storageCheck.Message}",
+                cancellationToken: cancellationToken);
+        }
+
+        return started;
+    }
+
+    private static bool NeedsInterruptedMp4Finalization(RecordTask recordTask)
+    {
+        if (recordTask.Status != RecordTaskStatus.Completed ||
+            recordTask.RecordSession is null ||
+            recordTask.OutputFormat != RecordOutputFormat.Mp4 ||
+            recordTask.RecordSession.OutputFormat != RecordOutputFormat.Mp4 ||
+            string.IsNullOrWhiteSpace(recordTask.OutputFilePath))
+        {
+            return false;
+        }
+
+        var finalOutputPath = NormalizeAbsolutePath(recordTask.OutputFilePath);
+        if (HasUsableOutput(finalOutputPath, CalculateFileSize(finalOutputPath)))
+        {
+            return false;
+        }
+
+        var recorderOutputPath = ResolveManualFinalizeSourcePath(recordTask, recordTask.RecordSession);
+        return File.Exists(recorderOutputPath);
+    }
+
+    private static string ResolveManualFinalizeSourcePath(RecordTask recordTask, RecordSession recordSession)
+    {
+        var resultPath = recordTask.Result?.FilePath;
+        if (!string.IsNullOrWhiteSpace(resultPath) &&
+            resultPath.EndsWith(".ts", StringComparison.OrdinalIgnoreCase))
+        {
+            var normalizedResultPath = NormalizeAbsolutePath(resultPath);
+            if (File.Exists(normalizedResultPath))
+            {
+                return normalizedResultPath;
+            }
+        }
+
+        return NormalizeAbsolutePath(
+            GetRecorderOutputPath(
+                recordTask.OutputFilePath ?? resultPath ?? string.Empty,
+                recordSession.OutputFormat,
+                recordSession.SaveMode));
+    }
+
    private async Task PersistStartupProfileAsync(SessionProcessRuntime runtime, CancellationToken cancellationToken)
    {
        using var scope = _serviceScopeFactory.CreateScope();
@@ -303,7 +514,7 @@ public sealed partial class FfmpegService : IFfmpegService
            SystemLogLevel.Info,
            "FFmpeg",
            $"ffmpeg input profile={runtime.InputOptionProfile}.",
-            detail: $"stream={runtime.SelectedProtocol}:{runtime.SelectedQuality}; compatibilityRetry={runtime.HasRetriedWithCompatibilityProfile}; refreshRetry={runtime.HasRetriedWithRefreshedStream}",
+            detail: $"stream={runtime.SelectedProtocol}:{runtime.SelectedQuality}; videoCodec={runtime.SelectedVideoCodec ?? "unknown"}; compatibilityRetry={runtime.HasRetriedWithCompatibilityProfile}; refreshRetry={runtime.HasRetriedWithRefreshedStream}",
            liveRoomId: runtime.LiveRoomId,
            recordSessionId: runtime.RecordSessionId,
            recordTaskId: runtime.CurrentTaskId,
@@ -351,29 +562,107 @@ public sealed partial class FfmpegService : IFfmpegService

            await process.StandardInput.WriteLineAsync("q");
            await process.StandardInput.FlushAsync();
-
-            using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
-            timeoutCts.CancelAfter(TimeSpan.FromSeconds(12));
-
-            try
-            {
-                await process.WaitForExitAsync(timeoutCts.Token);
-            }
-            catch (OperationCanceledException)
-            {
-                if (!process.HasExited)
-                {
-                    process.Kill(true);
-                }
-            }
        }
        catch (Exception ex)
        {
            _logger.LogWarning(ex, "Stop ffmpeg process failed for session {RecordSessionId}", recordSessionId);
+        }
+    }

-            if (!process.HasExited)
+    private void SetPostProcessState(
+        Guid recordSessionId,
+        Guid recordTaskId,
+        string stage,
+        double? progressPercent,
+        string? detail = null)
+    {
+        var normalizedProgress = progressPercent.HasValue
+            ? Math.Clamp(progressPercent.Value, 0d, 100d)
+            : (double?)null;
+        _postProcessStates[recordTaskId] = new PostProcessRuntimeEntry(
+            recordSessionId,
+            new RecordTaskRuntimeState(
+                RecordTaskStatus.Processing,
+                stage,
+                normalizedProgress,
+                detail));
+    }
+
+    private void ClearPostProcessState(Guid recordTaskId) =>
+        _postProcessStates.TryRemove(recordTaskId, out _);
+
+    private bool IsSessionUnderPostProcessing(Guid recordSessionId) =>
+        _postProcessStates.Values.Any(entry => entry.RecordSessionId == recordSessionId);
+
+    private async Task<IDisposable> AcquireTranscodeSlotAsync(int maxConcurrentTasks, CancellationToken cancellationToken)
+    {
+        var normalizedMaxConcurrentTasks = Math.Clamp(maxConcurrentTasks, 1, 16);
+        TaskCompletionSource<IDisposable> waiter;
+
+        lock (_transcodeConcurrencyLock)
+        {
+            _maxConcurrentTranscodeTasks = normalizedMaxConcurrentTasks;
+            if (_transcodeWaiters.Count == 0 && _activeTranscodeTasks < _maxConcurrentTranscodeTasks)
            {
-                process.Kill(true);
+                _activeTranscodeTasks++;
+                return new TranscodeSlotLease(this);
+            }
+
+            waiter = new TaskCompletionSource<IDisposable>(TaskCreationOptions.RunContinuationsAsynchronously);
+            _transcodeWaiters.Enqueue(waiter);
+        }
+
+        using var cancellationRegistration = cancellationToken.Register(
+            static state => ((TaskCompletionSource<IDisposable>)state!).TrySetCanceled(),
+            waiter);
+        return await waiter.Task.ConfigureAwait(false);
+    }
+
+    private void ReleaseTranscodeSlot()
+    {
+        lock (_transcodeConcurrencyLock)
+        {
+            if (_activeTranscodeTasks > 0)
+            {
+                _activeTranscodeTasks--;
+            }
+
+            while (_activeTranscodeTasks < _maxConcurrentTranscodeTasks && _transcodeWaiters.Count > 0)
+            {
+                var waiter = _transcodeWaiters.Dequeue();
+                if (waiter.Task.IsCompleted)
+                {
+                    continue;
+                }
+
+                _activeTranscodeTasks++;
+                if (waiter.TrySetResult(new TranscodeSlotLease(this)))
+                {
+                    return;
+                }
+
+                _activeTranscodeTasks--;
+            }
+        }
+    }
+
+    private sealed record PostProcessRuntimeEntry(Guid RecordSessionId, RecordTaskRuntimeState State);
+
+    private sealed class TranscodeSlotLease : IDisposable
+    {
+        private readonly FfmpegService _owner;
+        private int _disposed;
+
+        public TranscodeSlotLease(FfmpegService owner)
+        {
+            _owner = owner;
+        }
+
+        public void Dispose()
+        {
+            if (Interlocked.Exchange(ref _disposed, 1) == 0)
+            {
+                _owner.ReleaseTranscodeSlot();
            }
        }
    }