fix: retry segmented sessions and restore settings text

2026-05-08 23:27:01 +08:00
parent 93e39b2c1c
commit ea41d3e7b4
4 changed files with 339 additions and 109 deletions
@@ -17,6 +17,7 @@ namespace LiveRecorder.Infrastructure.Services;

 public sealed partial class FfmpegService
 {
+    private const int MaxInSessionRetryAttempts = 3;
    private static readonly TimeSpan RuntimeSourceFailureWindow = TimeSpan.FromSeconds(20);
    private static readonly TimeSpan RuntimeOfflineVerificationStopTimeout = TimeSpan.FromSeconds(20);
    private static readonly TimeSpan RuntimeOfflineVerificationKillTimeout = TimeSpan.FromSeconds(8);
@@ -472,6 +473,11 @@ public sealed partial class FfmpegService
                return;
            }

+            if (await TryRecoverUnexpectedExitAsync(runtime, activeDanmakuSummary))
+            {
+                return;
+            }
+
            await FinalizeExitedSessionAsync(runtime, process, activeDanmakuSummary);
        }
        catch (Exception ex)
@@ -522,6 +528,11 @@ public sealed partial class FfmpegService
                return false;
            }

+            if (runtime.RetryAttemptCount >= MaxInSessionRetryAttempts)
+            {
+                return false;
+            }
+
            var observedAt = DateTimeOffset.UtcNow;
            if (runtime.StartupFailureKind == StartupFailureKind.InputOptionCompatibility)
            {
@@ -689,6 +700,182 @@ public sealed partial class FfmpegService
        }
    }

+    private async Task<bool> TryRecoverUnexpectedExitAsync(
+        SessionProcessRuntime runtime,
+        SessionDanmakuXmlRecorder.DanmakuSegmentSummary? activeDanmakuSummary)
+    {
+        if (!runtime.HasOpenedFirstSegment ||
+            runtime.StopRequested ||
+            runtime.CompletionRequested ||
+            runtime.SaveMode != RecordSaveMode.Segmented ||
+            runtime.RetryAttemptCount >= MaxInSessionRetryAttempts)
+        {
+            return false;
+        }
+
+        try
+        {
+            using var scope = _serviceScopeFactory.CreateScope();
+            var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
+            var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
+            var adapterFactory = scope.ServiceProvider.GetRequiredService<ILivePlatformAdapterFactory>();
+            var liveRoomStatusService = scope.ServiceProvider.GetRequiredService<LiveRoomStatusService>();
+            var eventScriptService = scope.ServiceProvider.GetRequiredService<IEventScriptService>();
+            var recordUploadService = scope.ServiceProvider.GetRequiredService<RecordUploadService>();
+
+            var session = await dbContext.RecordSessions
+                .Include(item => item.LiveRoom)
+                .Include(item => item.RecordTasks)
+                .ThenInclude(item => item.Result)
+                .FirstOrDefaultAsync(item => item.Id == runtime.RecordSessionId);
+
+            if (session?.LiveRoom is null || !IsActiveSessionStatus(session.Status))
+            {
+                return false;
+            }
+
+            var currentTask = session.RecordTasks
+                .OrderBy(item => item.SegmentIndex)
+                .ThenBy(item => item.CreatedAt)
+                .FirstOrDefault(item => item.Id == runtime.CurrentTaskId);
+
+            if (currentTask is null || !IsActiveTaskStatus(currentTask.Status))
+            {
+                return false;
+            }
+
+            var observedAt = DateTimeOffset.UtcNow;
+            var adapter = adapterFactory.GetByPlatform(session.LiveRoom.Platform);
+            var liveStatus = await adapter.GetLiveStatusAsync(session.LiveRoom.RoomId);
+            await liveRoomStatusService.ApplySnapshotAsync(session.LiveRoom, liveStatus, observedAt);
+            await dbContext.SaveChangesAsync();
+
+            if (!liveStatus.IsLive)
+            {
+                return false;
+            }
+
+            var retryAttempt = runtime.RetryAttemptCount + 1;
+            var refreshedStream = await adapter.GetStreamUrlAsync(session.LiveRoom.RoomId, session.PreferredQuality);
+            var retryStream = SelectRetryStreamForCurrentSession(refreshedStream, runtime);
+            var nextSegmentIndex = Math.Max(currentTask.SegmentIndex + 1, session.ActiveSegmentIndex + 1);
+            var outputPathPattern = session.OutputPathPattern ?? runtime.OutputPathPattern;
+            var retryOutputPath = NormalizeAbsolutePath(
+                ResolveSegmentOutputPath(outputPathPattern, session.SaveMode, nextSegmentIndex));
+
+            var retryTask = new RecordTask(
+                session.LiveRoomId,
+                session.Id,
+                nextSegmentIndex,
+                session.PreferredQuality,
+                session.OutputFormat,
+                observedAt);
+            retryTask.MarkStarting(retryStream.SelectedUrl, retryOutputPath, observedAt);
+            await dbContext.RecordTasks.AddAsync(retryTask);
+            await dbContext.SaveChangesAsync();
+
+            try
+            {
+                await StartInternalAsync(
+                    session,
+                    retryTask,
+                    retryStream,
+                    runtime.RecordingSettings,
+                    runtime.InputOptionProfile,
+                    hasRetriedWithCompatibilityProfile: false,
+                    hasRetriedWithRefreshedStream: false,
+                    retryAttempt);
+            }
+            catch
+            {
+                dbContext.RecordTasks.Remove(retryTask);
+                await dbContext.SaveChangesAsync();
+                throw;
+            }
+
+            var restartedAt = DateTimeOffset.UtcNow;
+            var previousTaskId = currentTask.Id;
+            var previousDurationSeconds = currentTask.StartedAt.HasValue
+                ? Math.Max(0, (restartedAt - currentTask.StartedAt.Value).TotalSeconds)
+                : (double?)null;
+            var previousEffectiveOutputPath = currentTask.OutputFilePath ??
+                NormalizeAbsolutePath(ResolveSegmentOutputPath(outputPathPattern, session.SaveMode, currentTask.SegmentIndex));
+            var previousRecorderSegmentPaths = runtime.CaptureCurrentRecorderSegments();
+
+            if (session.OutputFormat == RecordOutputFormat.Mp4)
+            {
+                currentTask.MarkProcessing("Recorder exited unexpectedly. Segment finalization was queued before retry.", restartedAt);
+                SetPostProcessState(
+                    session.Id,
+                    previousTaskId,
+                    "Queued",
+                    0,
+                    $"Waiting for {Path.GetFileName(GetRecorderOutputPath(previousEffectiveOutputPath, session.OutputFormat, session.SaveMode))} to finish writing");
+            }
+            else
+            {
+                currentTask.MarkCompleted(restartedAt, previousDurationSeconds);
+            }
+
+            currentTask.DetachProcess(restartedAt);
+            session.MarkRunning(restartedAt);
+            session.ActivateSegment(nextSegmentIndex, restartedAt);
+            retryTask.MarkRunning(restartedAt);
+            await dbContext.SaveChangesAsync();
+
+            if (session.OutputFormat == RecordOutputFormat.Mp4)
+            {
+                _ = FinalizeCompletedSegmentAsync(
+                    session.Id,
+                    previousTaskId,
+                    previousDurationSeconds,
+                    restartedAt,
+                    activeDanmakuSummary,
+                    previousRecorderSegmentPaths);
+            }
+            else
+            {
+                await UpsertRecordResultAsync(
+                    currentTask,
+                    dbContext,
+                    previousEffectiveOutputPath,
+                    CalculateFileSize(previousEffectiveOutputPath),
+                    currentTask.DurationSeconds,
+                    activeDanmakuSummary?.FilePath,
+                    activeDanmakuSummary?.MessageCount ?? 0,
+                    restartedAt);
+
+                await dbContext.SaveChangesAsync();
+                ClearPostProcessState(currentTask.Id);
+                var previousResult = await LoadRecordResultAsync(dbContext, currentTask.Id);
+                await eventScriptService.RunSegmentCompletedAsync(
+                    session.LiveRoom,
+                    session,
+                    currentTask,
+                    previousResult,
+                    previousEffectiveOutputPath,
+                    restartedAt);
+                await recordUploadService.TryAutoUploadTaskAsync(currentTask.Id, CancellationToken.None);
+            }
+
+            await logService.WriteAsync(
+                SystemLogLevel.Warning,
+                "FFmpeg",
+                $"ffmpeg exited unexpectedly. Retrying within the current session ({retryAttempt}/{MaxInSessionRetryAttempts}).",
+                runtime.GetRecentOutputSummary(),
+                session.LiveRoomId,
+                session.Id,
+                retryTask.Id);
+
+            return true;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "In-session ffmpeg retry failed for session {RecordSessionId}", runtime.RecordSessionId);
+            return false;
+        }
+    }
+
    private async Task<bool> TryRetryWithAlternateProtocolAsync(
        SessionProcessRuntime runtime,
        RecordSession session,
@@ -790,6 +977,38 @@ public sealed partial class FfmpegService
        }
    }

+    private static StreamUrlResult SelectRetryStreamForCurrentSession(
+        StreamUrlResult refreshedStream,
+        SessionProcessRuntime runtime)
+    {
+        var matchedOption = refreshedStream.AvailableQualities
+            .Where(item => item.Protocol.Equals(runtime.SelectedProtocol, StringComparison.OrdinalIgnoreCase) &&
+                           item.QualityKey.Equals(runtime.SelectedQuality, StringComparison.OrdinalIgnoreCase))
+            .OrderByDescending(item => item.Rank)
+            .FirstOrDefault();
+
+        if (matchedOption is null)
+        {
+            matchedOption = refreshedStream.AvailableQualities
+                .Where(item => item.Protocol.Equals(runtime.SelectedProtocol, StringComparison.OrdinalIgnoreCase))
+                .OrderByDescending(item => item.Rank)
+                .FirstOrDefault();
+        }
+
+        if (matchedOption is null)
+        {
+            return refreshedStream;
+        }
+
+        return new StreamUrlResult(
+            matchedOption.QualityKey,
+            matchedOption.Protocol,
+            matchedOption.Url,
+            refreshedStream.InputHeaders,
+            refreshedStream.AvailableQualities,
+            refreshedStream.SelectedVideoCodec);
+    }
+
    private async Task FinalizeExitedSessionAsync(
        SessionProcessRuntime runtime,
        Process process,
@@ -1327,11 +1546,15 @@ public sealed partial class FfmpegService

    private static bool IsRetryableStartupFailureLine(string line) =>
        line.Contains("Error reading HTTP response", StringComparison.OrdinalIgnoreCase) ||
+        line.Contains("overlong headers", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("unexpected EOF", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("Connection reset", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("Connection refused", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("I/O error", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("Invalid data found when processing input", StringComparison.OrdinalIgnoreCase) ||
+        (line.Contains("Invalid argument", StringComparison.OrdinalIgnoreCase) &&
+         (line.Contains("http://", StringComparison.OrdinalIgnoreCase) ||
+          line.Contains("https://", StringComparison.OrdinalIgnoreCase))) ||
        line.Contains("Server returned 4", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("Server returned 5", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("HTTP error 4", StringComparison.OrdinalIgnoreCase) ||
@@ -645,6 +645,7 @@ public sealed partial class FfmpegService
        string outputFilePath,
        RecordOutputFormat outputFormat,
        RecordSaveMode saveMode,
+        int startingSegmentIndex,
        RecordingTemplateType recordingTemplate,
        bool enableReconnect,
        int reconnectDelayMaxSeconds,
@@ -708,7 +709,7 @@ public sealed partial class FfmpegService
            arguments.AddRange(
            [
                "-f", "segment",
-                "-segment_start_number", "1",
+                "-segment_start_number", Math.Max(1, startingSegmentIndex).ToString(),
                "-segment_time", Math.Max(60, segmentDurationMinutes * 60).ToString(),
                "-break_non_keyframes", "0",
                "-reset_timestamps", "1",
@@ -159,6 +159,7 @@ public sealed partial class FfmpegService : IFfmpegService
                     recorderOutputPath,
                     recordSession.OutputFormat,
                     recordSession.SaveMode,
+                     Math.Max(1, initialTask.SegmentIndex),
                     recordingSettings.RecordingTemplate,
                     recordingSettings.EnableAutoReconnect,
                     recordingSettings.ReconnectDelayMaxSeconds,