fix: harden recording recovery and dashboard metrics

src/LiveRecorder.Infrastructure/Services/FfmpegService.cs

@@ -22,6 +22,9 @@ namespace LiveRecorder.Infrastructure.Services;
 public sealed partial class FfmpegService : IFfmpegService
 {
     private const string ArtifactRepairMarker = "[artifact-repair]";
+    private const string StalePendingStartupFailure =
+        "Recording startup was interrupted before stream initialization.";
+    private static readonly TimeSpan StalePendingThreshold = TimeSpan.FromMinutes(10);
     private static readonly Regex SegmentOpeningRegex = new(
         """Opening '([^']+)' for writing""",
         RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
@@ -644,6 +647,120 @@ public sealed partial class FfmpegService : IFfmpegService
         }
     }
 
+    public async Task<int> RecoverStalePendingSessionsAsync(CancellationToken cancellationToken = default)
+    {
+        if (!await _orphanRecoveryGate.WaitAsync(0, cancellationToken))
+        {
+            return 0;
+        }
+
+        try
+        {
+            using var scope = _serviceScopeFactory.CreateScope();
+            var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
+            var cutoff = DateTimeOffset.UtcNow - StalePendingThreshold;
+            var candidateIds = await dbContext.RecordSessions
+                .AsNoTracking()
+                .Where(session =>
+                    session.Status == RecordSessionStatus.Pending &&
+                    session.UpdatedAt <= cutoff)
+                .OrderBy(session => session.UpdatedAt)
+                .Select(session => session.Id)
+                .Take(50)
+                .ToListAsync(cancellationToken);
+
+            var recovered = 0;
+            foreach (var candidateId in candidateIds)
+            {
+                if (IsRunning(candidateId))
+                {
+                    continue;
+                }
+
+                using var candidateScope = _serviceScopeFactory.CreateScope();
+                var candidateDbContext = candidateScope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
+                var session = await candidateDbContext.RecordSessions
+                    .Include(item => item.RecordTasks)
+                    .ThenInclude(item => item.Result)
+                    .FirstOrDefaultAsync(item => item.Id == candidateId, cancellationToken);
+                if (session is null ||
+                    session.Status != RecordSessionStatus.Pending ||
+                    session.UpdatedAt > cutoff ||
+                    IsRunning(candidateId))
+                {
+                    continue;
+                }
+
+                if (HasRecoverablePendingMedia(session))
+                {
+                    if (await ReconcileInactiveSessionAsync(candidateId, allowTerminalSession: false, cancellationToken))
+                    {
+                        recovered++;
+                    }
+
+                    continue;
+                }
+
+                var failedAt = DateTimeOffset.UtcNow;
+                foreach (var task in session.RecordTasks.Where(task => task.Status == RecordTaskStatus.Pending))
+                {
+                    task.MarkFailed(StalePendingStartupFailure, failedAt);
+                }
+
+                session.MarkFailed(StalePendingStartupFailure, failedAt);
+                await candidateDbContext.SaveChangesAsync(cancellationToken);
+
+                var logService = candidateScope.ServiceProvider.GetRequiredService<ISystemLogService>();
+                await logService.WriteAsync(
+                    SystemLogLevel.Warning,
+                    "RecordSession",
+                    "A stale pending recording session was marked failed.",
+                    StalePendingStartupFailure,
+                    session.LiveRoomId,
+                    session.Id,
+                    session.RecordTasks.OrderBy(task => task.SegmentIndex).FirstOrDefault()?.Id,
+                    cancellationToken);
+                recovered++;
+            }
+
+            return recovered;
+        }
+        finally
+        {
+            _orphanRecoveryGate.Release();
+        }
+    }
+
+    private static bool HasRecoverablePendingMedia(RecordSession session)
+    {
+        if (DiscoverRecoverableSegments(session.OutputPathPattern, session.OutputFormat, session.SaveMode).Count > 0)
+        {
+            return true;
+        }
+
+        foreach (var task in session.RecordTasks)
+        {
+            var candidates = new[]
+            {
+                task.Result?.FilePath,
+                task.OutputFilePath,
+                string.IsNullOrWhiteSpace(task.OutputFilePath)
+                    ? null
+                    : GetRecorderOutputPath(task.OutputFilePath, session.OutputFormat, session.SaveMode)
+            };
+
+            if (candidates.Any(path =>
+                    !string.IsNullOrWhiteSpace(path) &&
+                    File.Exists(path) &&
+                    new FileInfo(path).Length > 0))
+            {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
     public async Task<bool> TryRecoverOrphanedTerminalTaskAsync(
         Guid recordTaskId,
         CancellationToken cancellationToken = default)