live_recorder/src/LiveRecorder.Infrastructure/Services/FfmpegService.Runtime.cs

using System.Diagnostics;
using System.Globalization;
using LiveRecorder.Application.Abstractions.Logging;
using LiveRecorder.Application.Abstractions.Notifications;
using LiveRecorder.Application.Abstractions.Platforms;
using LiveRecorder.Application.Abstractions.Recording;
using LiveRecorder.Application.Abstractions.Scripting;
using LiveRecorder.Application.Abstractions.Settings;
using LiveRecorder.Application.Services;
using LiveRecorder.Domain.Entities;
using LiveRecorder.Domain.Enums;
using LiveRecorder.Infrastructure.Persistence;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;

namespace LiveRecorder.Infrastructure.Services;

public sealed partial class FfmpegService
{
    private const int MaxInSessionRetryAttempts = 3;
    private const string RuntimeRecoveryMarker = "[runtime-recovery]";
    private const string ShortUnexpectedExitArtifactError =
        "Unexpected recorder exit produced only a short fragment. The file was kept locally and excluded from automatic upload.";
    internal const string FfprobeUnreadableArtifactError = "ffprobe could not read the recorded media file.";
    private const string ShutdownFinalizationDeferredMessage =
        "Application shutdown interrupted MP4 finalization. The intermediate recording was kept and will be finalized automatically after restart.";
    private static readonly TimeSpan MinimumUnexpectedExitArtifactDuration = TimeSpan.FromSeconds(5);
    private static readonly TimeSpan StableRuntimeResetThreshold = TimeSpan.FromMinutes(1);
    private static readonly TimeSpan RuntimeSourceFailureWindow = TimeSpan.FromSeconds(20);
    private static readonly TimeSpan RuntimeOfflineVerificationStopTimeout = TimeSpan.FromSeconds(20);
    private static readonly TimeSpan RuntimeOfflineVerificationKillTimeout = TimeSpan.FromSeconds(8);
    private static readonly TimeSpan OfflineConfirmationDelay = TimeSpan.FromSeconds(10);
    private static readonly TimeSpan[] RuntimeRecoveryBackoff =
    [
        TimeSpan.FromSeconds(5),
        TimeSpan.FromSeconds(15),
        TimeSpan.FromSeconds(30),
        TimeSpan.FromSeconds(60),
        TimeSpan.FromSeconds(120)
    ];

    private async Task HandleProcessOutputAsync(SessionProcessRuntime runtime, string? line, bool isError)
    {
        if (string.IsNullOrWhiteSpace(line))
        {
            return;
        }

        runtime.RememberOutputLine(line, isError);
        if (line.Contains("overlong headers", StringComparison.OrdinalIgnoreCase))
        {
            runtime.MarkHlsOverlongHeadersFailure();
        }

        if (IsTimestampDiscontinuityFailureLine(line))
        {
            runtime.MarkTimestampDiscontinuityFailure();
        }
        else if (runtime.InputOptionProfile == FfmpegInputOptionProfile.TimestampRepair &&
                 IsTimestampMuxerFailureLine(line))
        {
            runtime.MarkTimestampMuxerFailure();
        }

        _logger.LogDebug("ffmpeg[{SessionId}] {Line}", runtime.RecordSessionId, line);
        if (TryParseSegmentOpenPath(line, out var openedPath))
        {
            await HandleSegmentOpenedAsync(runtime, openedPath);
            return;
        }

        if (!runtime.HasOpenedFirstSegment &&
            runtime.InputOptionProfile == FfmpegInputOptionProfile.TimestampTranscode &&
            runtime.RecoveryVideoEncoder.Kind != RecoveryVideoEncoderKind.Software &&
            IsHardwareEncoderFailureLine(line))
        {
            runtime.MarkStartupFailure(StartupFailureKind.HardwareEncoderUnavailable, line);
        }
        else if (!runtime.HasOpenedFirstSegment && IsOptionCompatibilityFailureLine(line))
        {
            runtime.MarkStartupFailure(StartupFailureKind.InputOptionCompatibility, line);
        }
        else if (!runtime.HasOpenedFirstSegment && IsRetryableStartupFailureLine(line))
        {
            runtime.MarkStartupFailure(StartupFailureKind.StreamHandshake, line);
        }

        if (TryClassifyPersistedFfmpegLine(line, isError, out var level))
        {
            await PersistFfmpegLineAsync(runtime, line, level);
        }

        if (IsRuntimeSourceFailureLine(line) &&
            runtime.RegisterRuntimeSourceFailure(DateTimeOffset.UtcNow, RuntimeSourceFailureWindow))
        {
            _ = ValidateRuntimeSourceFailureAsync(runtime, line);
        }

        TryUpdateBandwidthFromProgressLine(runtime, line);

        // Flush bandwidth sample periodically
        _ = runtime.FlushBandwidthSampleIfNeededAsync(WriteBandwidthSampleAsync, CancellationToken.None);
    }

    private async Task WriteBandwidthSampleAsync(
        Guid liveRoomId,
        Guid recordSessionId,
        Guid recordTaskId,
        string detail,
        CancellationToken cancellationToken)
    {
        try
        {
            using var scope = _serviceScopeFactory.CreateScope();
            var systemLogService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
            await systemLogService.WriteAsync(
                SystemLogLevel.Info,
                "Bandwidth",
                "bandwidth_sample",
                detail,
                liveRoomId: liveRoomId,
                recordSessionId: recordSessionId,
                recordTaskId: recordTaskId,
                cancellationToken: cancellationToken);
        }
        catch
        {
            // Silently ignore bandwidth logging failures
        }
    }

    private void TryUpdateBandwidthFromProgressLine(SessionProcessRuntime runtime, string line)
    {
        if (line.StartsWith("total_size=", StringComparison.Ordinal))
        {
            if (long.TryParse(line.AsSpan("total_size=".Length), out var totalSize))
            {
                runtime.UpdateBandwidthTotalSize(totalSize);
            }
        }
        else if (line.StartsWith("bitrate=", StringComparison.Ordinal))
        {
            // bitrate format: "1234.5kbits/s"
            var bitrateStr = line.AsSpan("bitrate=".Length).Trim();
            if (bitrateStr.EndsWith("kbits/s", StringComparison.OrdinalIgnoreCase))
            {
                bitrateStr = bitrateStr.Slice(0, bitrateStr.Length - "kbits/s".Length).Trim();
            }
            if (double.TryParse(bitrateStr, NumberStyles.Float, CultureInfo.InvariantCulture, out var bitrate))
            {
                runtime.UpdateBandwidthBitrate(bitrate);
            }
        }
        else if (line.StartsWith("speed=", StringComparison.Ordinal))
        {
            var speedStr = line.AsSpan("speed=".Length).TrimEnd('x').Trim();
            if (double.TryParse(speedStr, NumberStyles.Float, CultureInfo.InvariantCulture, out var speed))
            {
                runtime.UpdateBandwidthSpeed(speed);
            }
        }
    }

    internal static bool TryClassifyPersistedFfmpegLine(string line, bool isError, out SystemLogLevel level)
    {
        if (IsRecoverableFfmpegWarningLine(line))
        {
            level = SystemLogLevel.Warning;
            return true;
        }

        if (line.Contains("error", StringComparison.OrdinalIgnoreCase) ||
            line.Contains("fail", StringComparison.OrdinalIgnoreCase) ||
            line.Contains("timed out", StringComparison.OrdinalIgnoreCase))
        {
            level = isError ? SystemLogLevel.Error : SystemLogLevel.Warning;
            return true;
        }

        level = default;
        return false;
    }

    internal static bool IsRecoverableFfmpegWarningLine(string line)
    {
        return line.Contains("Will reconnect at", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("keepalive request failed", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("retrying with new connection", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("IO error: Connection reset by peer", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("Error in the push function", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("Network is unreachable", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("HTTP error 404", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("404 Not Found", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("Failed to open segment", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("Failed to resolve hostname", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("error=Connection reset by peer", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("error=End of file", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("Concatenated FLV detected", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("might fail to demux, decode and seek", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("Failed to parse temporal unit", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("Failed to read unit", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("Invalid data found when processing input", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("Error opening input file pipe:0", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("Error writing trailer", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("Error muxing a packet", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("Conversion failed", StringComparison.OrdinalIgnoreCase);
    }

    internal static bool IsTimestampDiscontinuityFailureLine(string line) =>
        line.Contains("non monotonically increasing dts", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("non-monotonous dts", StringComparison.OrdinalIgnoreCase);

    internal static bool IsTimestampMuxerFailureLine(string line) =>
        line.Contains("Error submitting a packet to the muxer", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("Error muxing a packet", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("Task finished with error code: -22", StringComparison.OrdinalIgnoreCase);

    internal static bool IsHardwareEncoderFailureLine(string line) =>
        line.Contains("Unknown encoder", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("No capable devices found", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("Cannot load libcuda", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("Device creation failed", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("No VA display found", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("Failed to initialise VAAPI connection", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("Error initializing an internal MFX session", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("Error initializing output stream", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("Impossible to convert between the formats", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("A hardware device reference is required", StringComparison.OrdinalIgnoreCase);

    internal static bool IsMeaningfulUnexpectedExitArtifact(double? durationSeconds, long? fileSizeBytes) =>
        durationSeconds >= MinimumUnexpectedExitArtifactDuration.TotalSeconds;

    internal static bool CanResetInSessionRetryBudget(DateTimeOffset? processStartedAt, DateTimeOffset observedAt) =>
        processStartedAt.HasValue && observedAt - processStartedAt.Value >= StableRuntimeResetThreshold;

    private static bool IsRuntimeSourceFailureLine(string line)
    {
        if (line.Contains("Will reconnect at", StringComparison.OrdinalIgnoreCase) ||
            line.Contains("retrying with new connection", StringComparison.OrdinalIgnoreCase))
        {
            return false;
        }

        return line.Contains("HTTP error 404", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("404 Not Found", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("Input/output error", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("Failed to open segment", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("Network is unreachable", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("Failed to resolve hostname", StringComparison.OrdinalIgnoreCase) ||
               line.Contains("Connection to tcp://", StringComparison.OrdinalIgnoreCase) &&
               line.Contains("failed", StringComparison.OrdinalIgnoreCase);
    }

    private async Task PersistFfmpegLineAsync(SessionProcessRuntime runtime, string line, SystemLogLevel level)
    {
        try
        {
            using var scope = _serviceScopeFactory.CreateScope();
            var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
            await logService.WriteAsync(
                level,
                "FFmpeg",
                "ffmpeg reported a warning or error line.",
                line,
                runtime.LiveRoomId,
                runtime.RecordSessionId,
                runtime.CurrentTaskId);
        }
        catch (Exception ex)
        {
            _logger.LogWarning(ex, "Persist ffmpeg output failed for session {RecordSessionId}", runtime.RecordSessionId);
        }
    }

    private async Task ValidateRuntimeSourceFailureAsync(SessionProcessRuntime runtime, string failureLine)
    {
        try
        {
            using var scope = _serviceScopeFactory.CreateScope();
            var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
            var adapterFactory = scope.ServiceProvider.GetRequiredService<ILivePlatformAdapterFactory>();
            var liveRoomStatusService = scope.ServiceProvider.GetRequiredService<LiveRoomStatusService>();
            var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();

            var session = await dbContext.RecordSessions
                .Include(item => item.LiveRoom)
                .Include(item => item.RecordTasks)
                .FirstOrDefaultAsync(item => item.Id == runtime.RecordSessionId);
            if (session?.LiveRoom is null || !IsActiveSessionStatus(session.Status) || !_processes.ContainsKey(session.Id))
            {
                return;
            }

            var observedAt = DateTimeOffset.UtcNow;
            var adapter = adapterFactory.GetByPlatform(session.LiveRoom!.Platform);
            var liveStatus = await adapter.GetLiveStatusAsync(session.LiveRoom.RoomId);
            await liveRoomStatusService.ApplySnapshotAsync(session.LiveRoom, liveStatus, observedAt);
            await dbContext.SaveChangesAsync();

            if (liveStatus.IsLive)
            {
                await logService.WriteAsync(
                    SystemLogLevel.Warning,
                    "FFmpeg",
                    "Repeated stream errors were detected, but the live room is still online.",
                    failureLine,
                    session.LiveRoomId,
                    session.Id,
                    runtime.CurrentTaskId);
                return;
            }

            await logService.WriteAsync(
                SystemLogLevel.Warning,
                "FFmpeg",
                "Repeated stream errors indicate the live room is offline. Completing the active recording session.",
                failureLine,
                session.LiveRoomId,
                session.Id,
                runtime.CurrentTaskId);

            var stopped = await StopAndWaitAsync(
                session.Id,
                markAsCompletedOnExit: true,
                RuntimeOfflineVerificationStopTimeout,
                CancellationToken.None);
            if (!stopped)
            {
                await logService.WriteAsync(
                    SystemLogLevel.Warning,
                    "FFmpeg",
                    "The recorder did not stop gracefully after offline verification. Force killing the ffmpeg process.",
                    failureLine,
                    session.LiveRoomId,
                    session.Id,
                    runtime.CurrentTaskId);

                var killed = await KillAndWaitAsync(
                    session.Id,
                    RuntimeOfflineVerificationKillTimeout,
                    CancellationToken.None);
                if (!killed)
                {
                    await logService.WriteAsync(
                        SystemLogLevel.Warning,
                        "FFmpeg",
                        "The recorder did not exit before the forced-stop wait expired; the session will be finalized when the process exits.",
                        failureLine,
                        session.LiveRoomId,
                        session.Id,
                        runtime.CurrentTaskId);
                    return;
                }
            }

            await TryReconcileInactiveSessionAsync(session.Id, CancellationToken.None);
        }
        catch (Exception ex)
        {
            _logger.LogWarning(ex, "Runtime source failure verification failed for session {RecordSessionId}", runtime.RecordSessionId);
        }
        finally
        {
            runtime.CompleteRuntimeSourceFailureVerification();
        }
    }

    private async Task HandleSegmentOpenedAsync(SessionProcessRuntime runtime, string openedPath)
    {
        await runtime.Gate.WaitAsync();
        try
        {
            var normalizedOpenedPath = NormalizeAbsolutePath(openedPath);
            using var scope = _serviceScopeFactory.CreateScope();
            var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
            var session = await dbContext.RecordSessions
                .Include(item => item.LiveRoom)
                .Include(item => item.RecordTasks)
                .ThenInclude(item => item.Result)
                .FirstOrDefaultAsync(item => item.Id == runtime.RecordSessionId);

            if (session is null)
            {
                return;
            }

            var now = DateTimeOffset.UtcNow;
            var currentTask = session.RecordTasks.FirstOrDefault(item => item.Id == runtime.CurrentTaskId);
            if (currentTask is null)
            {
                return;
            }

            if (runtime.SaveMode == RecordSaveMode.SingleFile)
            {
                runtime.HasOpenedFirstSegment = true;
                ResetStartupFailureBackoff(runtime.LiveRoomId);
                currentTask.AttachProcess(runtime.ProcessId, now);
                currentTask.MarkRunning(now);
                session.AttachProcess(runtime.ProcessId, now);
                session.MarkRunning(now);
                session.ActivateSegment(1, now);
                runtime.ResetCurrentRecorderSegmentPaths(normalizedOpenedPath);
                if (!runtime.HasInitializedDanmaku)
                {
                    await EnsureDanmakuSegmentAsync(runtime, currentTask, now);
                    runtime.HasInitializedDanmaku = true;
                }

                await dbContext.SaveChangesAsync();
                return;
            }

            var detectedSegmentIndex = ExtractSegmentIndex(openedPath);
            var segmentIndex = !runtime.HasOpenedFirstSegment
                ? Math.Max(1, detectedSegmentIndex ?? runtime.CurrentSegmentIndex)
                : Math.Max(
                    1,
                    detectedSegmentIndex ??
                    (string.Equals(
                        NormalizeAbsolutePath(openedPath),
                        NormalizeAbsolutePath(runtime.CurrentOutputFilePath),
                        StringComparison.OrdinalIgnoreCase)
                        ? runtime.CurrentSegmentIndex
                        : runtime.CurrentSegmentIndex + 1));
            if (!runtime.HasInitializedDanmaku)
            {
                await EnsureDanmakuSegmentAsync(runtime, currentTask, now);
                runtime.HasInitializedDanmaku = true;
            }

            if (segmentIndex == runtime.CurrentSegmentIndex)
            {
                runtime.HasOpenedFirstSegment = true;
                ResetStartupFailureBackoff(runtime.LiveRoomId);
                var finalOutputPath = NormalizeAbsolutePath(
                    currentTask.OutputFilePath ??
                    ResolveSegmentOutputPath(session.OutputPathPattern ?? runtime.OutputPathPattern, session.SaveMode, segmentIndex));
                currentTask.MarkStarting(runtime.StreamUrl, finalOutputPath, currentTask.StartedAt ?? now);
                currentTask.AttachProcess(runtime.ProcessId, now);
                currentTask.MarkRunning(now);
                session.AttachProcess(runtime.ProcessId, now);
                session.MarkRunning(now);
                session.ActivateSegment(segmentIndex, now);
                runtime.CurrentOutputFilePath = normalizedOpenedPath;
                runtime.TrackCurrentRecorderSegment(normalizedOpenedPath);
                await dbContext.SaveChangesAsync();
                return;
            }

            var previousTask = currentTask;
            var previousTaskId = previousTask.Id;
            var previousDurationSeconds = previousTask.StartedAt.HasValue
                ? Math.Max(0, (now - previousTask.StartedAt.Value).TotalSeconds)
                : (double?)null;
            var previousEffectiveOutputPath = previousTask.OutputFilePath ??
                NormalizeAbsolutePath(ResolveSegmentOutputPath(session.OutputPathPattern ?? runtime.OutputPathPattern, session.SaveMode, previousTask.SegmentIndex));

            if (ShouldTreatAsUnexpectedFragmentRollover(runtime, previousDurationSeconds))
            {
                runtime.CurrentOutputFilePath = normalizedOpenedPath;
                runtime.TrackCurrentRecorderSegment(normalizedOpenedPath);
                currentTask.AttachProcess(runtime.ProcessId, now);
                currentTask.MarkRunning(now);
                session.AttachProcess(runtime.ProcessId, now);
                session.MarkRunning(now);
                session.ActivateSegment(runtime.CurrentSegmentIndex, now);
                await dbContext.SaveChangesAsync();
                return;
            }

            var previousRecorderSegmentPaths = runtime.CaptureCurrentRecorderSegments();
            MediaArtifactValidation? previousMediaValidation = null;

            var newTask = new RecordTask(
                session.LiveRoomId,
                session.Id,
                segmentIndex,
                session.PreferredQuality,
                session.OutputFormat,
                now);
            var newTaskOutputPath = NormalizeAbsolutePath(
                ResolveSegmentOutputPath(session.OutputPathPattern ?? runtime.OutputPathPattern, session.SaveMode, segmentIndex));
            newTask.MarkStarting(runtime.StreamUrl, newTaskOutputPath, now);
            newTask.AttachProcess(runtime.ProcessId, now);
            newTask.MarkRunning(now);
            await dbContext.RecordTasks.AddAsync(newTask);

            if (session.OutputFormat == RecordOutputFormat.Mp4)
            {
                previousTask.MarkProcessing("MP4 finalization queued.", now);
            }
            else
            {
                previousMediaValidation = await ValidateMediaArtifactAsync(
                    previousEffectiveOutputPath,
                    session.OutputFormat,
                    CancellationToken.None);
                previousDurationSeconds = previousMediaValidation.DurationSeconds;
                if (previousMediaValidation.IsValid)
                {
                    previousTask.MarkCompleted(now, previousDurationSeconds);
                }
                else
                {
                    previousTask.MarkFailed(previousMediaValidation.ErrorMessage!, now, previousDurationSeconds);
                }
            }

            previousTask.DetachProcess(now);

            session.AttachProcess(runtime.ProcessId, now);
            session.MarkRunning(now);
            session.ActivateSegment(segmentIndex, now);

            await dbContext.SaveChangesAsync();

            runtime.CurrentTaskId = newTask.Id;
            runtime.CurrentSegmentIndex = segmentIndex;
            runtime.CurrentOutputFilePath = normalizedOpenedPath;
            runtime.ResetCurrentRecorderSegmentPaths(normalizedOpenedPath);
            runtime.HasOpenedFirstSegment = true;
            ResetStartupFailureBackoff(runtime.LiveRoomId);

            if (runtime.DanmakuRecorder is not null)
            {
                await runtime.DanmakuRecorder.StartSegmentAsync(
                    newTask.Id,
                    segmentIndex,
                    NormalizeAbsolutePath(newTask.OutputFilePath ?? newTaskOutputPath),
                    now);
            }

            var previousDanmakuSummary = runtime.DanmakuRecorder?.TakeSummary(previousTaskId);
            if (session.OutputFormat == RecordOutputFormat.Mp4)
            {
                SetPostProcessState(
                    session.Id,
                    previousTaskId,
                    "Queued",
                    0,
                    $"Waiting for {Path.GetFileName(GetRecorderOutputPath(previousEffectiveOutputPath, session.OutputFormat, session.SaveMode))} to finish writing");
                _ = FinalizeCompletedSegmentAsync(
                    session.Id,
                    previousTaskId,
                    previousDurationSeconds,
                    now,
                    previousDanmakuSummary,
                    previousRecorderSegmentPaths);
            }
            else
            {
                await UpsertRecordResultAsync(
                    previousTask,
                    dbContext,
                    previousEffectiveOutputPath,
                    CalculateFileSize(previousEffectiveOutputPath),
                    previousTask.DurationSeconds,
                    previousDanmakuSummary?.FilePath,
                    previousDanmakuSummary?.MessageCount ?? 0,
                    now,
                    mediaValidatedForDispatch: previousMediaValidation?.IsValid == true);

                await dbContext.SaveChangesAsync();
                ClearPostProcessState(previousTask.Id);
                var completionDispatchService = scope.ServiceProvider.GetRequiredService<CompletionDispatchService>();
                await completionDispatchService.TryDispatchTaskAsync(previousTask.Id, CancellationToken.None);
            }
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Handle segment open failed for session {RecordSessionId}", runtime.RecordSessionId);
        }
        finally
        {
            runtime.Gate.Release();
        }
    }

    private async Task HandleProcessExitedAsync(SessionProcessRuntime runtime, Process process)
    {
        var transition = new SessionTransitionRuntime(runtime.RecordSessionId);
        if (!_sessionTransitions.TryAdd(runtime.RecordSessionId, transition))
        {
            transition.Dispose();
            if (_sessionTransitions.TryGetValue(runtime.RecordSessionId, out var activeTransition))
            {
                await activeTransition.Completion.Task;
            }

            await HandleProcessExitedAsync(runtime, process);
            return;
        }

        _processes.TryRemove(runtime.RecordSessionId, out _);

        try
        {
            runtime.DanmakuCancellation.Cancel();
            if (runtime.DanmakuPumpTask is not null)
            {
                try
                {
                    await runtime.DanmakuPumpTask;
                }
                catch (OperationCanceledException)
                {
                }
                catch (Exception ex)
                {
                    _logger.LogWarning(ex, "Danmaku pump ended with error for session {RecordSessionId}", runtime.RecordSessionId);
                }
            }

            SessionDanmakuXmlRecorder.DanmakuSegmentSummary? activeDanmakuSummary = null;
            if (runtime.DanmakuRecorder is not null)
            {
                activeDanmakuSummary = await runtime.DanmakuRecorder.CompleteActiveSegmentAsync();
                await runtime.DanmakuRecorder.DisposeAsync();
            }

            if (runtime.DanmakuConnection is not null)
            {
                await runtime.DanmakuConnection.DisposeAsync();
            }

            if (await TryRecoverStartupFailureAsync(runtime))
            {
                return;
            }

            if (!runtime.HasOpenedFirstSegment &&
                !runtime.StopRequested &&
                !runtime.CompletionRequested &&
                !runtime.ShutdownRequested &&
                await TryRecoverStartupUntilAvailableAsync(runtime, transition))
            {
                return;
            }

            if (await TryRecoverUnexpectedExitAsync(runtime, activeDanmakuSummary, transition))
            {
                return;
            }

            await FinalizeExitedSessionAsync(runtime, process, activeDanmakuSummary);
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Handle ffmpeg exit failed for session {RecordSessionId}", runtime.RecordSessionId);
        }
        finally
        {
            _sessionTransitions.TryRemove(runtime.RecordSessionId, out _);
            transition.Completion.TrySetResult(true);
            transition.Dispose();
            runtime.ExitCompletion.TrySetResult(true);
            runtime.Dispose();
            process.Dispose();
        }
    }

    private async Task<bool> TryRecoverStartupUntilAvailableAsync(
        SessionProcessRuntime runtime,
        SessionTransitionRuntime transition)
    {
        var attempt = transition.NextAttempt(Math.Max(1, runtime.RetryAttemptCount + 1));
        while (!transition.Token.IsCancellationRequested)
        {
            try
            {
                using var scope = _serviceScopeFactory.CreateScope();
                var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
                var session = await dbContext.RecordSessions
                    .Include(item => item.LiveRoom)
                    .Include(item => item.RecordTasks)
                    .ThenInclude(item => item.Result)
                    .FirstOrDefaultAsync(item => item.Id == runtime.RecordSessionId, transition.Token);
                if (session?.LiveRoom is null || !IsActiveSessionStatus(session.Status))
                {
                    return false;
                }

                var currentTask = session.RecordTasks.FirstOrDefault(item => item.Id == runtime.CurrentTaskId)
                    ?? session.RecordTasks.OrderByDescending(item => item.SegmentIndex).FirstOrDefault();
                if (currentTask is null)
                {
                    return false;
                }

                var delay = GetRuntimeRecoveryDelay(attempt);
                session.MarkRecovering(
                    $"{RuntimeRecoveryMarker} attempt={attempt}; startup did not open a media segment",
                    DateTimeOffset.UtcNow);
                currentTask.MarkStarting(runtime.StreamUrl, currentTask.OutputFilePath ?? runtime.CurrentOutputFilePath, DateTimeOffset.UtcNow);
                await dbContext.SaveChangesAsync(transition.Token);

                var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
                await logService.WriteAsync(
                    SystemLogLevel.Warning,
                    "FFmpeg",
                    $"Recorder startup is recovering in the same session (attempt {attempt}).",
                    $"delaySeconds={delay.TotalSeconds:0}; ffmpegExit={runtime.Process?.ExitCode}; curlExit={runtime.CurlExitCode?.ToString() ?? "unknown"}; output={runtime.GetRecentOutputSummary()}; curl={runtime.GetRecentCurlErrorSummary()}",
                    session.LiveRoomId,
                    session.Id,
                    currentTask.Id,
                    transition.Token);

                await Task.Delay(delay, transition.Token);
                var adapterFactory = scope.ServiceProvider.GetRequiredService<ILivePlatformAdapterFactory>();
                var adapter = adapterFactory.GetByPlatform(session.LiveRoom.Platform);
                var liveStatus = await adapter.GetLiveStatusAsync(session.LiveRoom.RoomId, transition.Token);
                if (!liveStatus.IsLive)
                {
                    await Task.Delay(OfflineConfirmationDelay, transition.Token);
                    liveStatus = await adapter.GetLiveStatusAsync(session.LiveRoom.RoomId, transition.Token);
                }

                if (!liveStatus.IsLive)
                {
                    runtime.MarkStopRequested(markAsCompletedOnExit: true);
                    return false;
                }

                var stream = await adapter.GetStreamUrlAsync(session.LiveRoom.RoomId, session.PreferredQuality, transition.Token);
                var selected = SelectRetryStreamForCurrentSession(stream, runtime);
                var context = runtime.RecoveryContext with
                {
                    AttemptCount = attempt,
                    HasRetriedWithRefreshedStream = true
                };
                session.MarkStarting(selected.SelectedUrl, session.OutputPathPattern ?? runtime.OutputPathPattern, DateTimeOffset.UtcNow);
                currentTask.MarkStarting(selected.SelectedUrl, currentTask.OutputFilePath ?? runtime.CurrentOutputFilePath, DateTimeOffset.UtcNow);
                await dbContext.SaveChangesAsync(transition.Token);

                await StartInternalAsync(session, currentTask, selected, runtime.RecordingSettings, context, transition.Token);
                var restartedAt = DateTimeOffset.UtcNow;
                session.MarkRunning(restartedAt);
                currentTask.MarkRunning(restartedAt);
                await dbContext.SaveChangesAsync(transition.Token);
                return true;
            }
            catch (OperationCanceledException) when (transition.Token.IsCancellationRequested)
            {
                runtime.MarkStopRequested(transition.MarkAsCompletedOnExit);
                return false;
            }
            catch (Exception ex)
            {
                _logger.LogWarning(ex, "Recorder startup recovery attempt {Attempt} failed for session {RecordSessionId}", attempt, runtime.RecordSessionId);
                attempt = transition.NextAttempt(attempt + 1);
            }
        }

        return false;
    }

    private async Task<bool> TryRecoverStartupFailureAsync(SessionProcessRuntime runtime)
    {
        if (runtime.ShutdownRequested ||
            runtime.HasOpenedFirstSegment ||
            runtime.StartupFailureKind == StartupFailureKind.None ||
            string.IsNullOrWhiteSpace(runtime.LastStartupFailureLine))
        {
            return false;
        }

        try
        {
            using var scope = _serviceScopeFactory.CreateScope();
            var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
            var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();

            var session = await dbContext.RecordSessions
                .Include(item => item.LiveRoom)
                .Include(item => item.RecordTasks)
                .ThenInclude(item => item.Result)
                .FirstOrDefaultAsync(item => item.Id == runtime.RecordSessionId);

            if (session?.LiveRoom is null || !IsActiveSessionStatus(session.Status))
            {
                return false;
            }

            var currentTask = session.RecordTasks
                .OrderBy(item => item.SegmentIndex)
                .ThenBy(item => item.CreatedAt)
                .FirstOrDefault(item => item.Id == runtime.CurrentTaskId);

            if (currentTask is null || !IsActiveTaskStatus(currentTask.Status))
            {
                return false;
            }

            var observedAt = DateTimeOffset.UtcNow;
            if (runtime.StartupFailureKind == StartupFailureKind.HardwareEncoderUnavailable)
            {
                if (runtime.RecoveryContext.HasRetriedWithSoftwareEncoder)
                {
                    return false;
                }

                await logService.WriteAsync(
                    SystemLogLevel.Warning,
                    "FFmpeg",
                    $"Hardware recovery encoder {runtime.RecoveryVideoEncoder.Kind} failed to initialize. Retrying with libx264.",
                    runtime.LastStartupFailureLine,
                    session.LiveRoomId,
                    session.Id,
                    currentTask.Id);

                DisableRecoveryVideoEncoder(runtime.RecoveryVideoEncoder);

                var retryStream = new StreamUrlResult(
                    runtime.SelectedQuality,
                    runtime.SelectedProtocol,
                    runtime.StreamUrl,
                    runtime.InputHeaders,
                    Array.Empty<StreamQualityOption>(),
                    runtime.SelectedVideoCodec);
                var softwareContext = runtime.RecoveryContext with
                {
                    ForceSoftwareEncoder = true,
                    HasRetriedWithSoftwareEncoder = true
                };

                session.MarkStarting(retryStream.SelectedUrl, session.OutputPathPattern ?? runtime.OutputPathPattern, observedAt);
                session.ActivateSegment(Math.Max(1, runtime.CurrentSegmentIndex), observedAt);
                currentTask.MarkStarting(retryStream.SelectedUrl, currentTask.OutputFilePath ?? runtime.CurrentOutputFilePath, observedAt);
                await dbContext.SaveChangesAsync();

                await StartInternalAsync(
                    session,
                    currentTask,
                    retryStream,
                    runtime.RecordingSettings,
                    softwareContext);

                var restartedAt = DateTimeOffset.UtcNow;
                session.MarkRunning(restartedAt);
                currentTask.MarkRunning(restartedAt);
                await dbContext.SaveChangesAsync();
                return true;
            }

            if (runtime.RetryAttemptCount >= MaxInSessionRetryAttempts)
            {
                return false;
            }

            if (runtime.StartupFailureKind == StartupFailureKind.InputOptionCompatibility)
            {
                if (runtime.HasRetriedWithCompatibilityProfile)
                {
                    return false;
                }

                await logService.WriteAsync(
                    SystemLogLevel.Warning,
                    "FFmpeg",
                    "Initial ffmpeg input profile was rejected. Retrying once with minimal compatibility options.",
                    runtime.LastStartupFailureLine,
                    session.LiveRoomId,
                    session.Id,
                    currentTask.Id);

                var retryStream = new StreamUrlResult(
                    runtime.SelectedQuality,
                    runtime.SelectedProtocol,
                    runtime.StreamUrl,
                    runtime.InputHeaders,
                    Array.Empty<StreamQualityOption>(),
                    runtime.SelectedVideoCodec);

                session.MarkStarting(retryStream.SelectedUrl, session.OutputPathPattern ?? runtime.OutputPathPattern, observedAt);
                session.ActivateSegment(Math.Max(1, runtime.CurrentSegmentIndex), observedAt);
                currentTask.MarkStarting(retryStream.SelectedUrl, currentTask.OutputFilePath ?? runtime.CurrentOutputFilePath, observedAt);
                await dbContext.SaveChangesAsync();

                await StartInternalAsync(
                    session,
                    currentTask,
                    retryStream,
                    runtime.RecordingSettings,
                    runtime.RecoveryContext with
                    {
                        InputOptionProfile = FfmpegInputOptionProfile.Minimal,
                        AttemptCount = runtime.RetryAttemptCount + 1,
                        HasRetriedWithCompatibilityProfile = true
                    });

                var restartedAt = DateTimeOffset.UtcNow;
                session.MarkRunning(restartedAt);
                currentTask.MarkRunning(restartedAt);
                await dbContext.SaveChangesAsync();

                await logService.WriteAsync(
                    SystemLogLevel.Info,
                    "FFmpeg",
                    "ffmpeg startup retry succeeded with minimal compatibility options.",
                    liveRoomId: session.LiveRoomId,
                    recordSessionId: session.Id,
                    recordTaskId: currentTask.Id);

                return true;
            }

            if (runtime.StartupFailureKind != StartupFailureKind.StreamHandshake)
            {
                return false;
            }

            if (ShouldImmediatelyFallbackFromHls(runtime.SelectedProtocol, runtime.HasHlsOverlongHeadersFailure))
            {
                return await TryRetryWithAlternateProtocolAsync(
                    runtime,
                    session,
                    currentTask,
                    observedAt,
                    scope,
                    "HLS input exceeded the native FFmpeg header limit; switching directly to FLV without refreshing HLS.");
            }

            if (runtime.HasRetriedWithRefreshedStream)
            {
                if (runtime.HasRetriedWithAlternateProtocol)
                {
                    return false;
                }

                return await TryRetryWithAlternateProtocolAsync(runtime, session, currentTask, observedAt, scope);
            }

            var adapterFactory = scope.ServiceProvider.GetRequiredService<ILivePlatformAdapterFactory>();
            var liveRoomStatusService = scope.ServiceProvider.GetRequiredService<LiveRoomStatusService>();
            var adapter = adapterFactory.GetByPlatform(session.LiveRoom!.Platform);
            var liveStatus = await adapter.GetLiveStatusAsync(session.LiveRoom.RoomId);
            await liveRoomStatusService.ApplySnapshotAsync(session.LiveRoom, liveStatus, observedAt);
            await dbContext.SaveChangesAsync();

            if (!liveStatus.IsLive)
            {
                return false;
            }

            await logService.WriteAsync(
                SystemLogLevel.Warning,
                "FFmpeg",
                "Initial ffmpeg input handshake failed. Refreshing stream URL and retrying once.",
                runtime.LastStartupFailureLine,
                session.LiveRoomId,
                session.Id,
                currentTask.Id);

            var refreshedStream = await adapter.GetStreamUrlAsync(session.LiveRoom.RoomId, session.PreferredQuality);

            var refreshedOption = refreshedStream.AvailableQualities
                .Where(option => option.Protocol.Equals(runtime.SelectedProtocol, StringComparison.OrdinalIgnoreCase) &&
                                 option.QualityKey.Equals(runtime.SelectedQuality, StringComparison.OrdinalIgnoreCase))
                .OrderByDescending(option => option.Rank)
                .FirstOrDefault()
                ?? refreshedStream.AvailableQualities
                    .Where(option => option.Protocol.Equals(runtime.SelectedProtocol, StringComparison.OrdinalIgnoreCase))
                    .OrderByDescending(option => option.Rank)
                    .FirstOrDefault();

            var selectedUrl = refreshedOption?.Url ?? refreshedStream.SelectedUrl;
            var selectedProtocol = refreshedOption?.Protocol ?? refreshedStream.SelectedProtocol;
            var selectedQuality = refreshedOption?.QualityKey ?? refreshedStream.SelectedQuality;

            var streamForRetry = new StreamUrlResult(
                selectedQuality,
                selectedProtocol,
                selectedUrl,
                refreshedStream.InputHeaders,
                refreshedStream.AvailableQualities,
                refreshedStream.SelectedVideoCodec);

            session.MarkStarting(streamForRetry.SelectedUrl, session.OutputPathPattern ?? runtime.OutputPathPattern, observedAt);
            session.ActivateSegment(Math.Max(1, runtime.CurrentSegmentIndex), observedAt);
            currentTask.MarkStarting(streamForRetry.SelectedUrl, currentTask.OutputFilePath ?? runtime.CurrentOutputFilePath, observedAt);
            await dbContext.SaveChangesAsync();

            await StartInternalAsync(
                session,
                currentTask,
                streamForRetry,
                runtime.RecordingSettings,
                AdvanceRecoveryContext(
                    runtime.RecoveryContext,
                    runtime.InputOptionProfile,
                    runtime.SelectedProtocol,
                    streamForRetry.SelectedProtocol,
                    refreshedStream: true));

            var refreshedRetryStartedAt = DateTimeOffset.UtcNow;
            session.MarkRunning(refreshedRetryStartedAt);
            currentTask.MarkRunning(refreshedRetryStartedAt);
            await dbContext.SaveChangesAsync();

            await logService.WriteAsync(
                SystemLogLevel.Info,
                "FFmpeg",
                "ffmpeg startup retry succeeded with a refreshed stream URL.",
                liveRoomId: session.LiveRoomId,
                recordSessionId: session.Id,
                recordTaskId: currentTask.Id);

            return true;
        }
        catch (Exception ex)
        {
            _logger.LogWarning(ex, "Recover ffmpeg startup failure failed for session {RecordSessionId}", runtime.RecordSessionId);
            return false;
        }
    }

    private async Task<bool> TryRecoverUnexpectedExitAsync(
        SessionProcessRuntime runtime,
        SessionDanmakuXmlRecorder.DanmakuSegmentSummary? activeDanmakuSummary,
        SessionTransitionRuntime transition)
    {
        if (!runtime.HasOpenedFirstSegment ||
            runtime.StopRequested ||
            runtime.CompletionRequested ||
            runtime.ShutdownRequested ||
            runtime.SaveMode != RecordSaveMode.Segmented)
        {
            return false;
        }

        try
        {
            using var scope = _serviceScopeFactory.CreateScope();
            var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
            var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
            var adapterFactory = scope.ServiceProvider.GetRequiredService<ILivePlatformAdapterFactory>();
            var liveRoomStatusService = scope.ServiceProvider.GetRequiredService<LiveRoomStatusService>();

            var session = await dbContext.RecordSessions
                .Include(item => item.LiveRoom)
                .Include(item => item.RecordTasks)
                .ThenInclude(item => item.Result)
                .FirstOrDefaultAsync(item => item.Id == runtime.RecordSessionId);

            if (session?.LiveRoom is null || !IsActiveSessionStatus(session.Status))
            {
                return false;
            }

            var currentTask = session.RecordTasks
                .OrderBy(item => item.SegmentIndex)
                .ThenBy(item => item.CreatedAt)
                .FirstOrDefault(item => item.Id == runtime.CurrentTaskId);

            if (currentTask is null || !IsActiveTaskStatus(currentTask.Status))
            {
                return false;
            }

            var observedAt = DateTimeOffset.UtcNow;
            var adapter = adapterFactory.GetByPlatform(session.LiveRoom.Platform);
            var liveStatus = await adapter.GetLiveStatusAsync(session.LiveRoom.RoomId);
            if (!liveStatus.IsLive)
            {
                await Task.Delay(OfflineConfirmationDelay, transition.Token);
                liveStatus = await adapter.GetLiveStatusAsync(session.LiveRoom.RoomId, transition.Token);
            }
            await liveRoomStatusService.ApplySnapshotAsync(session.LiveRoom, liveStatus, observedAt);
            await dbContext.SaveChangesAsync();

            if (!liveStatus.IsLive)
            {
                runtime.MarkStopRequested(markAsCompletedOnExit: true);
                return false;
            }

            var refreshedStream = await adapter.GetStreamUrlAsync(session.LiveRoom.RoomId, session.PreferredQuality);
            var retryStream = SelectRetryStreamForCurrentSession(refreshedStream, runtime);
            var nextSegmentIndex = Math.Max(currentTask.SegmentIndex + 1, session.ActiveSegmentIndex + 1);
            var outputPathPattern = session.OutputPathPattern ?? runtime.OutputPathPattern;
            var retryOutputPath = NormalizeAbsolutePath(
                ResolveSegmentOutputPath(outputPathPattern, session.SaveMode, nextSegmentIndex));

            var retryTask = new RecordTask(
                session.LiveRoomId,
                session.Id,
                nextSegmentIndex,
                session.PreferredQuality,
                session.OutputFormat,
                observedAt);
            retryTask.MarkStarting(retryStream.SelectedUrl, retryOutputPath, observedAt);
            await dbContext.RecordTasks.AddAsync(retryTask);
            await dbContext.SaveChangesAsync();

            var retryInputOptionProfile = ResolveRetryInputOptionProfile(runtime);
            var stableBeforeExit = runtime.ProcessStartedAt.HasValue &&
                                   observedAt - runtime.ProcessStartedAt.Value >= StableRuntimeResetThreshold;
            var recoveryBase = stableBeforeExit
                ? InitialRecoveryContext
                : runtime.RecoveryContext;
            var recoveryContext = AdvanceRecoveryContext(
                recoveryBase,
                retryInputOptionProfile,
                runtime.SelectedProtocol,
                retryStream.SelectedProtocol);
            var retryAttempt = recoveryContext.AttemptCount;
            try
            {
                await StartInternalAsync(
                    session,
                    retryTask,
                    retryStream,
                    runtime.RecordingSettings,
                    recoveryContext);
            }
            catch
            {
                dbContext.RecordTasks.Remove(retryTask);
                await dbContext.SaveChangesAsync();
                throw;
            }

            var restartedAt = DateTimeOffset.UtcNow;
            var previousTaskId = currentTask.Id;
            var previousDurationSeconds = currentTask.StartedAt.HasValue
                ? Math.Max(0, (restartedAt - currentTask.StartedAt.Value).TotalSeconds)
                : (double?)null;
            var previousProcessRuntime = runtime.ProcessStartedAt.HasValue
                ? restartedAt - runtime.ProcessStartedAt.Value
                : TimeSpan.Zero;
            if (previousProcessRuntime >= StableRuntimeResetThreshold)
            {
                ResetRuntimeFailureBackoff(session.LiveRoomId);
            }
            var previousEffectiveOutputPath = currentTask.OutputFilePath ??
                NormalizeAbsolutePath(ResolveSegmentOutputPath(outputPathPattern, session.SaveMode, currentTask.SegmentIndex));
            var previousRecorderSegmentPaths = runtime.CaptureCurrentRecorderSegments();
            long? previousFileSize = null;
            MediaArtifactValidation? previousMediaValidation = null;

            if (session.OutputFormat == RecordOutputFormat.Mp4)
            {
                currentTask.MarkProcessing("Recorder exited unexpectedly. Segment finalization was queued before retry.", restartedAt);
                SetPostProcessState(
                    session.Id,
                    previousTaskId,
                    "Queued",
                    0,
                    $"Waiting for {Path.GetFileName(GetRecorderOutputPath(previousEffectiveOutputPath, session.OutputFormat, session.SaveMode))} to finish writing");
            }
            else
            {
                previousFileSize = CalculateFileSize(previousEffectiveOutputPath);
                previousMediaValidation = await ValidateMediaArtifactAsync(
                    previousEffectiveOutputPath,
                    session.OutputFormat,
                    CancellationToken.None);
                previousDurationSeconds = previousMediaValidation.DurationSeconds;
                if (previousMediaValidation.IsValid)
                {
                    currentTask.MarkCompleted(restartedAt, previousDurationSeconds);
                }
                else
                {
                    currentTask.MarkFailed(previousMediaValidation.ErrorMessage!, restartedAt, previousDurationSeconds);
                }
            }

            currentTask.DetachProcess(restartedAt);
            session.MarkRunning(restartedAt);
            session.ActivateSegment(nextSegmentIndex, restartedAt);
            retryTask.MarkRunning(restartedAt);
            await dbContext.SaveChangesAsync();

            if (session.OutputFormat == RecordOutputFormat.Mp4)
            {
                _ = FinalizeCompletedSegmentAsync(
                    session.Id,
                    previousTaskId,
                    previousDurationSeconds,
                    restartedAt,
                    activeDanmakuSummary,
                    previousRecorderSegmentPaths,
                    unexpectedExit: true);
            }
            else
            {
                await UpsertRecordResultAsync(
                    currentTask,
                    dbContext,
                    previousEffectiveOutputPath,
                    previousFileSize,
                    currentTask.DurationSeconds,
                    activeDanmakuSummary?.FilePath,
                    activeDanmakuSummary?.MessageCount ?? 0,
                    restartedAt,
                    mediaValidatedForDispatch: previousMediaValidation?.IsValid == true);

                await dbContext.SaveChangesAsync();
                ClearPostProcessState(currentTask.Id);
                var completionDispatchService = scope.ServiceProvider.GetRequiredService<CompletionDispatchService>();
                await completionDispatchService.TryDispatchTaskAsync(currentTask.Id, CancellationToken.None);
            }

            await logService.WriteAsync(
                SystemLogLevel.Warning,
                "FFmpeg",
                $"ffmpeg exited unexpectedly. Retrying within the current session (attempt {retryAttempt}).",
                $"transition={runtime.InputOptionProfile}/{runtime.SelectedProtocol} -> {retryInputOptionProfile}/{retryStream.SelectedProtocol}; output={runtime.GetRecentOutputSummary()}",
                session.LiveRoomId,
                session.Id,
                retryTask.Id);

            return true;
        }
        catch (Exception ex)
        {
            _logger.LogWarning(ex, "In-session ffmpeg retry failed for session {RecordSessionId}", runtime.RecordSessionId);
            return await WaitAndRetryRecoveringSessionAsync(runtime, activeDanmakuSummary, transition, ex);
        }
    }

    private async Task<bool> WaitAndRetryRecoveringSessionAsync(
        SessionProcessRuntime runtime,
        SessionDanmakuXmlRecorder.DanmakuSegmentSummary? activeDanmakuSummary,
        SessionTransitionRuntime transition,
        Exception failure)
    {
        var attempt = transition.NextAttempt(Math.Max(1, runtime.RetryAttemptCount + 1));
        while (!transition.Token.IsCancellationRequested && !runtime.ShutdownRequested)
        {
            try
            {
                using (var scope = _serviceScopeFactory.CreateScope())
                {
                    var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
                    var session = await dbContext.RecordSessions.FirstOrDefaultAsync(
                        item => item.Id == runtime.RecordSessionId,
                        CancellationToken.None);
                    if (session is null || !IsActiveSessionStatus(session.Status))
                    {
                        return false;
                    }

                    var reason = $"{RuntimeRecoveryMarker} attempt={attempt}; {failure.GetBaseException().Message}";
                    session.MarkRecovering(reason, DateTimeOffset.UtcNow);
                    await dbContext.SaveChangesAsync(CancellationToken.None);

                    var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
                    await logService.WriteAsync(
                        SystemLogLevel.Warning,
                        "FFmpeg",
                        $"Recorder recovery is waiting before attempt {attempt}.",
                        $"delaySeconds={GetRuntimeRecoveryDelay(attempt).TotalSeconds:0}; failure={failure}",
                        runtime.LiveRoomId,
                        runtime.RecordSessionId,
                        runtime.CurrentTaskId,
                        CancellationToken.None);
                }

                await Task.Delay(GetRuntimeRecoveryDelay(attempt), transition.Token);
                if (transition.MarkAsCompletedOnExit)
                {
                    runtime.MarkStopRequested(markAsCompletedOnExit: true);
                    return false;
                }

                if (transition.StopRequested)
                {
                    runtime.MarkStopRequested(markAsCompletedOnExit: false);
                    return false;
                }

                // Re-enter the normal recovery path with a fresh platform status and stream URL.
                return await TryRecoverUnexpectedExitAsync(runtime, activeDanmakuSummary, transition);
            }
            catch (OperationCanceledException) when (transition.Token.IsCancellationRequested)
            {
                runtime.MarkStopRequested(transition.MarkAsCompletedOnExit);
                return false;
            }
            catch (Exception ex)
            {
                failure = ex;
                attempt = transition.NextAttempt(attempt + 1);
            }
        }

        return false;
    }

    internal static TimeSpan GetRuntimeRecoveryDelay(int attempt) =>
        RuntimeRecoveryBackoff[Math.Clamp(attempt - 1, 0, RuntimeRecoveryBackoff.Length - 1)];

    private async Task<bool> TryRetryWithAlternateProtocolAsync(
        SessionProcessRuntime runtime,
        RecordSession session,
        RecordTask currentTask,
        DateTimeOffset observedAt,
        IServiceScope scope,
        string? transitionReason = null)
    {
        try
        {
            var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
            var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
            var adapterFactory = scope.ServiceProvider.GetRequiredService<ILivePlatformAdapterFactory>();
            var liveRoomStatusService = scope.ServiceProvider.GetRequiredService<LiveRoomStatusService>();
            var adapter = adapterFactory.GetByPlatform(session.LiveRoom!.Platform);
            var liveStatus = await adapter.GetLiveStatusAsync(session.LiveRoom.RoomId);
            await liveRoomStatusService.ApplySnapshotAsync(session.LiveRoom, liveStatus, observedAt);
            await dbContext.SaveChangesAsync();

            if (!liveStatus.IsLive)
            {
                return false;
            }

            var refreshedStream = await adapter.GetStreamUrlAsync(session.LiveRoom.RoomId, session.PreferredQuality);
            var alternateProtocol = runtime.SelectedProtocol.Equals("flv", StringComparison.OrdinalIgnoreCase)
                ? "hls"
                : "flv";

            var alternateOption = refreshedStream.AvailableQualities
                .Where(o => o.Protocol.Equals(alternateProtocol, StringComparison.OrdinalIgnoreCase))
                .OrderByDescending(o => o.Rank)
                .FirstOrDefault();

            if (alternateOption is null)
            {
                await logService.WriteAsync(
                    SystemLogLevel.Warning,
                    "FFmpeg",
                    $"No {alternateProtocol.ToUpperInvariant()} stream option available for alternate protocol fallback.",
                    liveRoomId: session.LiveRoomId,
                    recordSessionId: session.Id,
                    recordTaskId: currentTask.Id);
                return false;
            }

            await logService.WriteAsync(
                SystemLogLevel.Warning,
                "FFmpeg",
                transitionReason ??
                $"Refreshed stream URL also failed. Trying alternate protocol {alternateProtocol.ToUpperInvariant()} as last fallback.",
                runtime.LastStartupFailureLine,
                session.LiveRoomId,
                session.Id,
                currentTask.Id);

            var retryStream = new StreamUrlResult(
                alternateOption.QualityKey,
                alternateOption.Protocol,
                alternateOption.Url,
                refreshedStream.InputHeaders,
                refreshedStream.AvailableQualities,
                refreshedStream.SelectedVideoCodec);

            session.MarkStarting(retryStream.SelectedUrl, session.OutputPathPattern ?? runtime.OutputPathPattern, observedAt);
            session.ActivateSegment(Math.Max(1, runtime.CurrentSegmentIndex), observedAt);
            currentTask.MarkStarting(retryStream.SelectedUrl, currentTask.OutputFilePath ?? runtime.CurrentOutputFilePath, observedAt);
            await dbContext.SaveChangesAsync();

            await StartInternalAsync(
                session,
                currentTask,
                retryStream,
                runtime.RecordingSettings,
                AdvanceRecoveryContext(
                    runtime.RecoveryContext,
                    runtime.InputOptionProfile,
                    runtime.SelectedProtocol,
                    retryStream.SelectedProtocol,
                    refreshedStream: runtime.HasRetriedWithRefreshedStream));

            var retryStartedAt = DateTimeOffset.UtcNow;
            session.MarkRunning(retryStartedAt);
            currentTask.MarkRunning(retryStartedAt);
            await dbContext.SaveChangesAsync();

            await logService.WriteAsync(
                SystemLogLevel.Info,
                "FFmpeg",
                $"ffmpeg alternate protocol ({alternateProtocol.ToUpperInvariant()}) retry started.",
                liveRoomId: session.LiveRoomId,
                recordSessionId: session.Id,
                recordTaskId: currentTask.Id);

            return true;
        }
        catch (Exception ex)
        {
            _logger.LogWarning(ex, "Alternate protocol fallback failed for session {RecordSessionId}", runtime.RecordSessionId);
            return false;
        }
    }

    private static StreamUrlResult SelectRetryStreamForCurrentSession(
        StreamUrlResult refreshedStream,
        SessionProcessRuntime runtime)
    {
        if (runtime.HasTimestampDiscontinuityFailure &&
            runtime.InputOptionProfile == FfmpegInputOptionProfile.Baseline)
        {
            var alternateProtocol = runtime.SelectedProtocol.Equals("flv", StringComparison.OrdinalIgnoreCase)
                ? "hls"
                : "flv";
            var alternateSameQuality = refreshedStream.AvailableQualities
                .Where(item => item.Protocol.Equals(alternateProtocol, StringComparison.OrdinalIgnoreCase) &&
                               item.QualityKey.Equals(runtime.SelectedQuality, StringComparison.OrdinalIgnoreCase))
                .OrderByDescending(item => item.Rank)
                .FirstOrDefault();
            if (alternateSameQuality is not null)
            {
                return new StreamUrlResult(
                    alternateSameQuality.QualityKey,
                    alternateSameQuality.Protocol,
                    alternateSameQuality.Url,
                    refreshedStream.InputHeaders,
                    refreshedStream.AvailableQualities,
                    refreshedStream.SelectedVideoCodec);
            }
        }

        var matchedOption = refreshedStream.AvailableQualities
            .Where(item => item.Protocol.Equals(runtime.SelectedProtocol, StringComparison.OrdinalIgnoreCase) &&
                           item.QualityKey.Equals(runtime.SelectedQuality, StringComparison.OrdinalIgnoreCase))
            .OrderByDescending(item => item.Rank)
            .FirstOrDefault();

        if (matchedOption is null)
        {
            matchedOption = refreshedStream.AvailableQualities
                .Where(item => item.Protocol.Equals(runtime.SelectedProtocol, StringComparison.OrdinalIgnoreCase))
                .OrderByDescending(item => item.Rank)
                .FirstOrDefault();
        }

        if (matchedOption is null)
        {
            return refreshedStream;
        }

        return new StreamUrlResult(
            matchedOption.QualityKey,
            matchedOption.Protocol,
            matchedOption.Url,
            refreshedStream.InputHeaders,
            refreshedStream.AvailableQualities,
            refreshedStream.SelectedVideoCodec);
    }

    private static FfmpegInputOptionProfile ResolveRetryInputOptionProfile(SessionProcessRuntime runtime)
        => ResolveRetryInputOptionProfile(
            runtime.InputOptionProfile,
            runtime.HasTimestampDiscontinuityFailure,
            runtime.HasTimestampMuxerFailure);

    internal static FfmpegInputOptionProfile ResolveRetryInputOptionProfile(
        FfmpegInputOptionProfile inputOptionProfile,
        bool hasTimestampDiscontinuityFailure,
        bool hasTimestampMuxerFailure)
    {
        if (inputOptionProfile == FfmpegInputOptionProfile.TimestampRepair &&
            (hasTimestampDiscontinuityFailure || hasTimestampMuxerFailure))
        {
            return FfmpegInputOptionProfile.TimestampTranscode;
        }

        if (inputOptionProfile == FfmpegInputOptionProfile.Baseline &&
            hasTimestampDiscontinuityFailure)
        {
            return FfmpegInputOptionProfile.TimestampRepair;
        }

        return inputOptionProfile;
    }

    private async Task FinalizeExitedSessionAsync(
        SessionProcessRuntime runtime,
        Process process,
        SessionDanmakuXmlRecorder.DanmakuSegmentSummary? activeDanmakuSummary)
    {
        using var scope = _serviceScopeFactory.CreateScope();
        var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
        var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
        var settingsService = scope.ServiceProvider.GetRequiredService<ISystemSettingsService>();
        var emailNotificationService = scope.ServiceProvider.GetRequiredService<IEmailNotificationService>();
        var webhookNotificationService = scope.ServiceProvider.GetRequiredService<IWebhookNotificationService>();
        var settings = await settingsService.GetAsync();

        var session = await dbContext.RecordSessions
            .Include(item => item.LiveRoom)
            .Include(item => item.RecordTasks)
            .ThenInclude(item => item.Result)
            .FirstOrDefaultAsync(item => item.Id == runtime.RecordSessionId);

        if (session is null)
        {
            return;
        }

        var currentTask = session.RecordTasks.FirstOrDefault(item => item.Id == runtime.CurrentTaskId)
                          ?? session.RecordTasks.OrderByDescending(static item => item.SegmentIndex).FirstOrDefault();
        if (currentTask is null)
        {
            return;
        }

        var endedAt = DateTimeOffset.UtcNow;
        string? finalizationError = null;
        var effectiveOutputPath = currentTask.OutputFilePath ?? runtime.CurrentOutputFilePath;
        var durationSeconds = currentTask.StartedAt.HasValue
            ? (double?)Math.Max(0, (endedAt - currentTask.StartedAt.Value).TotalSeconds)
            : null;

        var shutdownFinalizationDeferred = false;
        if (session.OutputFormat == RecordOutputFormat.Mp4)
        {
            var recorderOutputPath = NormalizeAbsolutePath(GetRecorderOutputPath(
                currentTask.OutputFilePath ?? runtime.CurrentOutputFilePath,
                session.OutputFormat,
                session.SaveMode));

            if (runtime.ShutdownRequested)
            {
                var recorderSegmentPaths = runtime.CaptureCurrentRecorderSegments()
                    .Where(static path => !string.IsNullOrWhiteSpace(path))
                    .Select(NormalizeAbsolutePath)
                    .Distinct(StringComparer.OrdinalIgnoreCase)
                    .Where(File.Exists)
                    .ToArray();
                if (recorderSegmentPaths.Length > 1 && !string.IsNullOrWhiteSpace(currentTask.OutputFilePath))
                {
                    await WriteRecorderSegmentsManifestAsync(
                        GetRecorderSegmentsManifestPath(currentTask.OutputFilePath),
                        currentTask.OutputFilePath,
                        recorderSegmentPaths,
                        CancellationToken.None);
                }

                effectiveOutputPath = recorderSegmentPaths.FirstOrDefault() ??
                    (File.Exists(recorderOutputPath) ? recorderOutputPath : effectiveOutputPath);
                shutdownFinalizationDeferred = recorderSegmentPaths.Length > 0 || File.Exists(recorderOutputPath);
                finalizationError = shutdownFinalizationDeferred
                    ? ShutdownFinalizationDeferredMessage
                    : "Application shutdown completed, but no recoverable recording file was found.";
            }
            else if (runtime.StopRequested && runtime.WasForceKilled)
            {
                effectiveOutputPath = File.Exists(recorderOutputPath)
                    ? recorderOutputPath
                    : effectiveOutputPath;
                finalizationError = "The active segment did not close gracefully before the recorder process was terminated. The intermediate TS file was kept without MP4 finalization.";
            }
            else
            {
                var recorderSegmentPaths = runtime.CaptureCurrentRecorderSegments();
                await WaitForFileToStabilizeAsync(recorderOutputPath, CancellationToken.None);
                var finalizationResult = await TryFinalizeTaskOutputAsync(
                    settings.FfmpegPath,
                    settings.MaxConcurrentFfmpegTranscodeTasks,
                    settings.Mp4FinalizeTimeoutMinutes,
                    session,
                    currentTask,
                    durationSeconds,
                    recorderSegmentPaths);
                effectiveOutputPath = finalizationResult.OutputPath;
                finalizationError = finalizationResult.ErrorMessage;
            }
        }

        var fileSize = CalculateFileSize(effectiveOutputPath);
        var hasUsableOutput = HasUsableOutput(effectiveOutputPath, fileSize);
        var mediaValidation = await ValidateMediaArtifactAsync(
            effectiveOutputPath,
            session.OutputFormat,
            CancellationToken.None);
        durationSeconds = mediaValidation.DurationSeconds ?? durationSeconds;
        var toleratedNonZeroExit = false;
        var danmakuPath = activeDanmakuSummary?.FilePath ?? currentTask.Result?.DanmakuFilePath ?? GuessDanmakuPath(effectiveOutputPath);
        var danmakuMessageCount = activeDanmakuSummary?.MessageCount ?? CountDanmakuMessages(danmakuPath);

        var disposition = ClassifyExitedRecording(
            runtime.ShutdownRequested,
            runtime.StopRequested,
            IsLowStoragePauseError(finalizationError) || IsShutdownFinalizationPauseError(finalizationError),
            !string.IsNullOrWhiteSpace(finalizationError),
            mediaValidation.IsValid,
            process.ExitCode,
            runtime.CompletionRequested,
            hasUsableOutput,
            shutdownFinalizationDeferred || ResolveManualFinalizeSourcePaths(currentTask, session).Count > 0);

        if (disposition == ExitedRecordingDisposition.Processing)
        {
            currentTask.MarkProcessing(finalizationError, endedAt);
            session.MarkStopped(endedAt, finalizationError);
        }
        else if (disposition == ExitedRecordingDisposition.Stopped)
        {
            currentTask.MarkStopped(endedAt, durationSeconds, finalizationError);
            session.MarkStopped(endedAt, finalizationError);
        }
        else if (disposition == ExitedRecordingDisposition.Completed)
        {
            toleratedNonZeroExit = process.ExitCode != 0;
            currentTask.MarkCompleted(endedAt, durationSeconds);
            session.MarkCompleted(endedAt);
        }
        else
        {
            var errorMessage = finalizationError ?? mediaValidation.ErrorMessage ?? $"ffmpeg exit code: {process.ExitCode}";
            currentTask.MarkFailed(errorMessage, endedAt, durationSeconds);
            session.MarkFailed(errorMessage, endedAt);
        }

        currentTask.DetachProcess(endedAt);
        session.SyncSegmentCount(session.RecordTasks.Count, endedAt);
        await UpsertRecordResultAsync(
            currentTask,
            dbContext,
            effectiveOutputPath,
            fileSize,
            durationSeconds,
            danmakuPath,
            danmakuMessageCount,
            endedAt,
            mediaValidatedForDispatch: mediaValidation.IsValid);

        await dbContext.SaveChangesAsync();
        ClearPostProcessState(currentTask.Id);

        if (currentTask.Status == RecordTaskStatus.Completed)
        {
            var completionDispatchService = scope.ServiceProvider.GetRequiredService<CompletionDispatchService>();
            await completionDispatchService.TryDispatchTaskAsync(currentTask.Id, CancellationToken.None);
        }

        await logService.WriteAsync(
            session.Status == RecordSessionStatus.Completed ? SystemLogLevel.Info :
            runtime.ShutdownRequested ? SystemLogLevel.Warning :
            session.Status == RecordSessionStatus.Stopped ? SystemLogLevel.Warning :
            SystemLogLevel.Error,
            "FFmpeg",
            $"Recording session exited with status={session.Status}.",
            BuildExitLogDetail(runtime, process.ExitCode, effectiveOutputPath, toleratedNonZeroExit, finalizationError),
            session.LiveRoomId,
            session.Id,
            currentTask.Id);

        if (!runtime.StopRequested && !runtime.ShutdownRequested && session.LiveRoomId != Guid.Empty)
        {
            var isStartupFailure = !runtime.HasOpenedFirstSegment &&
                                   runtime.StartupFailureKind != StartupFailureKind.None;
            var processRuntime = runtime.ProcessStartedAt.HasValue
                ? endedAt - runtime.ProcessStartedAt.Value
                : TimeSpan.Zero;
            TimeSpan pollDelay;
            if (isStartupFailure)
            {
                pollDelay = RecordStartupFailure(session.LiveRoomId);
            }
            else if (ShouldApplyRuntimeFailureBackoff(session.Status, processRuntime))
            {
                pollDelay = RecordRuntimeFailure(session.LiveRoomId);
            }
            else
            {
                ResetRuntimeFailureBackoff(session.LiveRoomId);
                pollDelay = TimeSpan.FromSeconds(2);
            }

            _liveRoomPollingSignal.RequestImmediatePoll(session.LiveRoomId, pollDelay);
        }

        if (!runtime.ShutdownRequested && session.Status == RecordSessionStatus.Failed && session.LiveRoom is not null)
        {
            var isStartupFailure = !runtime.HasOpenedFirstSegment &&
                                   runtime.StartupFailureKind != StartupFailureKind.None;
            if (isStartupFailure && ShouldThrottleStartupFailureNotification(session.LiveRoomId))
            {
                // Notification suppressed — the failure has already been logged above.
            }
            else
            {
                var failureDetail = BuildExitLogDetail(runtime, process.ExitCode, effectiveOutputPath, toleratedNonZeroExit: false, finalizationError);
                await emailNotificationService.SendExceptionAsync(
                    "FFmpeg",
                    "Recording session exited abnormally.",
                    failureDetail,
                    session.LiveRoom,
                    currentTask);
                await webhookNotificationService.SendExceptionAsync(
                    "FFmpeg",
                    "Recording session exited abnormally.",
                    failureDetail,
                    session.LiveRoom,
                    currentTask);
            }
        }
    }

    internal static ExitedRecordingDisposition ClassifyExitedRecording(
        bool shutdownRequested,
        bool stopRequested,
        bool finalizationPaused,
        bool hasFinalizationError,
        bool mediaValid,
        int exitCode,
        bool completionRequested,
        bool hasUsableOutput,
        bool hasRecoverableIntermediateOutput)
    {
        if (finalizationPaused)
        {
            return ExitedRecordingDisposition.Processing;
        }

        if (stopRequested)
        {
            return ExitedRecordingDisposition.Stopped;
        }

        if (shutdownRequested)
        {
            if (!hasFinalizationError && mediaValid && (exitCode == 0 || hasUsableOutput))
            {
                return ExitedRecordingDisposition.Completed;
            }

            return hasRecoverableIntermediateOutput
                ? ExitedRecordingDisposition.Processing
                : ExitedRecordingDisposition.Failed;
        }

        if (hasFinalizationError || !mediaValid)
        {
            return ExitedRecordingDisposition.Failed;
        }

        return (exitCode == 0 || completionRequested && hasUsableOutput) &&
               (completionRequested || !stopRequested)
            ? ExitedRecordingDisposition.Completed
            : ExitedRecordingDisposition.Failed;
    }

    private static string BuildExitLogDetail(
        SessionProcessRuntime runtime,
        int exitCode,
        string? effectiveOutputPath,
        bool toleratedNonZeroExit,
        string? finalizationError)
    {
        var parts = new List<string>
        {
            $"exitCode={exitCode}",
            $"output={effectiveOutputPath}",
            $"recorderOutput={runtime.RecorderOutputPath}",
            $"shutdownRequested={runtime.ShutdownRequested}",
            $"lifetimeSeconds={(runtime.ProcessStartedAt.HasValue ? Math.Max(0, (DateTimeOffset.UtcNow - runtime.ProcessStartedAt.Value).TotalSeconds).ToString("F1", CultureInfo.InvariantCulture) : "unknown")}",
            $"curlExitCode={runtime.CurlExitCode?.ToString(CultureInfo.InvariantCulture) ?? "unknown"}"
        };

        if (toleratedNonZeroExit)
        {
            parts.Add("non-zero exit was tolerated because completion was requested and the output is usable");
        }

        if (!string.IsNullOrWhiteSpace(finalizationError))
        {
            parts.Add($"finalizationError={finalizationError}");
        }

        if (!string.IsNullOrWhiteSpace(runtime.LastStartupFailureLine))
        {
            parts.Add($"startupFailure={runtime.LastStartupFailureLine}");
        }

        var recentOutput = runtime.GetRecentOutputSummary();
        if (!string.IsNullOrWhiteSpace(recentOutput))
        {
            parts.Add($"recentOutput={recentOutput}");
        }

        var recentCurlError = runtime.GetRecentCurlErrorSummary();
        if (!string.IsNullOrWhiteSpace(recentCurlError))
        {
            parts.Add($"curlStderr={recentCurlError}");
        }

        return string.Join("; ", parts);
    }

    private async Task FinalizeCompletedSegmentAsync(
        Guid recordSessionId,
        Guid recordTaskId,
        double? durationSeconds,
        DateTimeOffset endedAt,
        SessionDanmakuXmlRecorder.DanmakuSegmentSummary? danmakuSummary,
        IReadOnlyList<string>? recorderSegmentPaths,
        bool unexpectedExit = false)
    {
        try
        {
            using var scope = _serviceScopeFactory.CreateScope();
            var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
            var settingsService = scope.ServiceProvider.GetRequiredService<ISystemSettingsService>();
            var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
            var settings = await settingsService.GetAsync();

            var session = await dbContext.RecordSessions
                .Include(item => item.LiveRoom)
                .Include(item => item.RecordTasks)
                .ThenInclude(item => item.Result)
                .FirstOrDefaultAsync(item => item.Id == recordSessionId);

            if (session is null)
            {
                return;
            }

            var recordTask = session.RecordTasks.FirstOrDefault(item => item.Id == recordTaskId);
            if (recordTask is null)
            {
                return;
            }

            var effectiveOutputPath = recordTask.OutputFilePath;
            if (string.IsNullOrWhiteSpace(effectiveOutputPath))
            {
                if (string.IsNullOrWhiteSpace(session.OutputPathPattern))
                {
                    return;
                }

                effectiveOutputPath = NormalizeAbsolutePath(ResolveSegmentOutputPath(
                    session.OutputPathPattern,
                    session.SaveMode,
                    recordTask.SegmentIndex));
            }

            await WaitForFileToStabilizeAsync(
                GetRecorderOutputPath(effectiveOutputPath, session.OutputFormat, session.SaveMode),
                CancellationToken.None);

            var finalizationResult = await TryFinalizeTaskOutputAsync(
                settings.FfmpegPath,
                settings.MaxConcurrentFfmpegTranscodeTasks,
                settings.Mp4FinalizeTimeoutMinutes,
                session,
                recordTask,
                durationSeconds,
                recorderSegmentPaths);
            effectiveOutputPath = finalizationResult.OutputPath;
            var finalizationError = finalizationResult.ErrorMessage;
            var fileSize = CalculateFileSize(effectiveOutputPath);
            var mediaValidation = await ValidateMediaArtifactAsync(
                effectiveOutputPath,
                session.OutputFormat,
                CancellationToken.None);
            durationSeconds = mediaValidation.DurationSeconds;
            var mediaValidationError = !mediaValidation.IsValid
                ? mediaValidation.ErrorMessage
                : null;
            var danmakuPath = danmakuSummary?.FilePath ?? recordTask.Result?.DanmakuFilePath ?? GuessDanmakuPath(recordTask.OutputFilePath);
            var danmakuMessageCount = danmakuSummary?.MessageCount ?? recordTask.Result?.DanmakuMessageCount ?? CountDanmakuMessages(danmakuPath);

            if (IsLowStoragePauseError(finalizationError) || IsShutdownFinalizationPauseError(finalizationError))
            {
                recordTask.MarkProcessing(finalizationError, endedAt);
            }
            else if (!string.IsNullOrWhiteSpace(mediaValidationError))
            {
                recordTask.MarkFailed(mediaValidationError, endedAt, durationSeconds);
            }
            else if (string.IsNullOrWhiteSpace(finalizationError))
            {
                recordTask.MarkCompleted(endedAt, durationSeconds);
            }
            else
            {
                recordTask.MarkFailed(finalizationError, endedAt, durationSeconds);
            }

            await UpsertRecordResultAsync(
                recordTask,
                dbContext,
                effectiveOutputPath,
                fileSize,
                durationSeconds,
                danmakuPath,
                danmakuMessageCount,
                endedAt,
                mediaValidatedForDispatch: mediaValidation.IsValid);

            await dbContext.SaveChangesAsync();

            if ((recordTask.Status is RecordTaskStatus.Completed or RecordTaskStatus.Stopped) &&
                HasUsableOutput(effectiveOutputPath, fileSize))
            {
                var completionDispatchService = scope.ServiceProvider.GetRequiredService<CompletionDispatchService>();
                await completionDispatchService.TryDispatchTaskAsync(recordTask.Id, CancellationToken.None);
            }

            if (!string.IsNullOrWhiteSpace(finalizationError) || !string.IsNullOrWhiteSpace(mediaValidationError))
            {
                var warningDetail = finalizationError ?? mediaValidationError!;
                await logService.WriteAsync(
                    SystemLogLevel.Warning,
                    "FFmpeg",
                    IsLowStoragePauseError(finalizationError) || IsShutdownFinalizationPauseError(finalizationError)
                        ? "Segment MP4 finalization was paused and will resume automatically."
                        : !string.IsNullOrWhiteSpace(mediaValidationError)
                            ? "Unexpected recorder exit produced a short fragment; it was kept locally and excluded from automatic upload."
                            : "Segment MP4 finalization failed after rollover.",
                    warningDetail,
                    session.LiveRoomId,
                    session.Id,
                    recordTask.Id);
            }
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Finalize completed segment failed for session {RecordSessionId}, task {RecordTaskId}", recordSessionId, recordTaskId);

            try
            {
                using var scope = _serviceScopeFactory.CreateScope();
                var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
                var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
                var session = await dbContext.RecordSessions
                    .AsNoTracking()
                    .FirstOrDefaultAsync(item => item.Id == recordSessionId);
                await logService.WriteAsync(
                    SystemLogLevel.Error,
                    "FFmpeg",
                    "Background segment finalization failed.",
                    ex.ToString(),
                    session?.LiveRoomId,
                    recordSessionId,
                    recordTaskId);
            }
            catch (Exception logEx)
            {
                _logger.LogWarning(logEx, "Persist segment finalization failure log failed for session {RecordSessionId}, task {RecordTaskId}", recordSessionId, recordTaskId);
            }
        }
        finally
        {
            ClearPostProcessState(recordTaskId);
        }
    }

    private async Task PersistProcessBindingAsync(Guid recordSessionId, Guid recordTaskId, int processId, CancellationToken cancellationToken)
    {
        using var scope = _serviceScopeFactory.CreateScope();
        var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
        var session = await dbContext.RecordSessions.FirstOrDefaultAsync(item => item.Id == recordSessionId, cancellationToken);
        var task = await dbContext.RecordTasks.FirstOrDefaultAsync(item => item.Id == recordTaskId, cancellationToken);

        var now = DateTimeOffset.UtcNow;
        session?.AttachProcess(processId, now);
        task?.AttachProcess(processId, now);
        if (session is not null || task is not null)
        {
            await dbContext.SaveChangesAsync(cancellationToken);
        }
    }

    private async Task StartDanmakuAsync(SessionProcessRuntime runtime, RecordTask initialTask, CancellationToken cancellationToken)
    {
        try
        {
            LiveRoom? liveRoom = null;
            var includeNonChatEvents = true;
            var firstWrittenEventLogged = 0;

            using (var readScope = _serviceScopeFactory.CreateScope())
            {
                var dbContext = readScope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
                var adapterFactory = readScope.ServiceProvider.GetRequiredService<ILiveDanmakuAdapterFactory>();

                if (!runtime.RecordingSettings.EnableDanmakuRecording)
                {
                    return;
                }

                liveRoom = await dbContext.LiveRooms
                    .AsNoTracking()
                    .FirstOrDefaultAsync(item => item.Id == runtime.LiveRoomId, cancellationToken);
                if (liveRoom is null)
                {
                    return;
                }

                includeNonChatEvents = runtime.RecordingSettings.DanmakuIncludeNonChatEvents;
                var adapter = adapterFactory.TryGetByPlatform(liveRoom.Platform);
                if (adapter is null)
                {
                    await WriteDanmakuSystemLogAsync(
                        SystemLogLevel.Info,
                        $"No danmaku adapter is registered for {liveRoom.Platform}. Recording will continue without live comments.",
                        null,
                        runtime.LiveRoomId,
                        runtime.RecordSessionId,
                        initialTask.Id,
                        cancellationToken);
                    return;
                }

                runtime.DanmakuConnection = await adapter.ConnectAsync(
                    new DanmakuConnectionContext(
                        liveRoom.Id,
                        runtime.RecordSessionId,
                        liveRoom.Platform,
                        liveRoom.RoomId,
                        liveRoom.AnchorName,
                        liveRoom.Title,
                        liveRoom.SourceUrl,
                        runtime.RecordingSettings.DanmakuMinPollIntervalMilliseconds,
                        runtime.RecordingSettings.DanmakuRetryDelayMaxSeconds),
                    cancellationToken);
            }

            runtime.DanmakuRecorder = new SessionDanmakuXmlRecorder(
                liveRoom.Platform,
                liveRoom.Id,
                runtime.RecordSessionId,
                liveRoom.RoomId);
            await runtime.DanmakuRecorder.StartSegmentAsync(
                initialTask.Id,
                initialTask.SegmentIndex,
                NormalizeAbsolutePath(initialTask.OutputFilePath ?? runtime.CurrentOutputFilePath),
                initialTask.StartedAt ?? DateTimeOffset.UtcNow);
            runtime.HasInitializedDanmaku = true;

            runtime.DanmakuPumpTask = Task.Run(
                async () =>
                {
                    try
                    {
                        await runtime.DanmakuConnection.StartAsync(
                            async danmakuEvent =>
                            {
                                if (runtime.DanmakuRecorder is null)
                                {
                                    return;
                                }

                                if (!includeNonChatEvents &&
                                    !string.Equals(danmakuEvent.Type, "chat", StringComparison.OrdinalIgnoreCase))
                                {
                                    return;
                                }

                                await runtime.DanmakuRecorder.AppendAsync(danmakuEvent);
                                if (Interlocked.Exchange(ref firstWrittenEventLogged, 1) == 0)
                                {
                                    await WriteDanmakuSystemLogAsync(
                                        SystemLogLevel.Info,
                                        "First danmaku event was written to XML.",
                                        $"type={danmakuEvent.Type}; user={TruncateDanmakuPreview(danmakuEvent.User, 80)}; content={TruncateDanmakuPreview(danmakuEvent.Content, 160)}",
                                        runtime.LiveRoomId,
                                        runtime.RecordSessionId,
                                        initialTask.Id,
                                        runtime.DanmakuCancellation.Token);
                                }
                            },
                            runtime.DanmakuCancellation.Token);
                    }
                    catch (OperationCanceledException) when (runtime.DanmakuCancellation.IsCancellationRequested)
                    {
                    }
                    catch (Exception ex)
                    {
                        await WriteDanmakuSystemLogAsync(
                            SystemLogLevel.Warning,
                            "Danmaku pump stopped unexpectedly.",
                            ex.ToString(),
                            runtime.LiveRoomId,
                            runtime.RecordSessionId,
                            runtime.CurrentTaskId,
                            CancellationToken.None);
                        throw;
                    }
                },
                runtime.DanmakuCancellation.Token);

            await WriteDanmakuSystemLogAsync(
                SystemLogLevel.Info,
                "Danmaku capture started for the recording session.",
                null,
                liveRoom.Id,
                runtime.RecordSessionId,
                initialTask.Id,
                cancellationToken);
        }
        catch (Exception ex)
        {
            _logger.LogWarning(ex, "Start danmaku capture failed for session {RecordSessionId}", runtime.RecordSessionId);

            try
            {
                await WriteDanmakuSystemLogAsync(
                    SystemLogLevel.Warning,
                    "Danmaku capture startup failed. Recording will continue without live comments until retry succeeds.",
                    ex.ToString(),
                    runtime.LiveRoomId,
                    runtime.RecordSessionId,
                    runtime.CurrentTaskId,
                    cancellationToken);
            }
            catch (Exception logEx)
            {
                _logger.LogWarning(logEx, "Persist danmaku startup failure log failed for session {RecordSessionId}", runtime.RecordSessionId);
            }
        }
    }

    private async Task WriteDanmakuSystemLogAsync(
        SystemLogLevel level,
        string message,
        string? detail,
        Guid? liveRoomId,
        Guid? recordSessionId,
        Guid? recordTaskId,
        CancellationToken cancellationToken)
    {
        using var scope = _serviceScopeFactory.CreateScope();
        var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
        await logService.WriteAsync(
            level,
            "Danmaku",
            message,
            detail,
            liveRoomId,
            recordSessionId,
            recordTaskId,
            cancellationToken);
    }

    private static string TruncateDanmakuPreview(string? value, int maxLength)
    {
        if (string.IsNullOrWhiteSpace(value))
        {
            return string.Empty;
        }

        var normalized = value.Trim().Replace(Environment.NewLine, " ", StringComparison.Ordinal);
        return normalized.Length <= maxLength ? normalized : normalized[..maxLength];
    }

    private static bool IsRetryableStartupFailureLine(string line) =>
        line.Contains("Error reading HTTP response", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("overlong headers", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("unexpected EOF", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("Connection reset", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("Connection refused", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("I/O error", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("Invalid data found when processing input", StringComparison.OrdinalIgnoreCase) ||
        (line.Contains("Invalid argument", StringComparison.OrdinalIgnoreCase) &&
         (line.Contains("http://", StringComparison.OrdinalIgnoreCase) ||
          line.Contains("https://", StringComparison.OrdinalIgnoreCase))) ||
        line.Contains("Server returned 4", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("Server returned 5", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("HTTP error 4", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("HTTP error 5", StringComparison.OrdinalIgnoreCase);

    private static bool IsOptionCompatibilityFailureLine(string line) =>
        line.Contains("Option not found", StringComparison.OrdinalIgnoreCase) ||
        line.Contains("Unrecognized option", StringComparison.OrdinalIgnoreCase);

    private static bool ShouldTreatAsUnexpectedFragmentRollover(
        SessionProcessRuntime runtime,
        double? previousDurationSeconds)
    {
        if (!runtime.HasOpenedFirstSegment || !previousDurationSeconds.HasValue)
        {
            return false;
        }

        var expectedSegmentDurationSeconds = Math.Max(60, runtime.RecordingSettings.SegmentDurationMinutes * 60);
        var rolloverToleranceSeconds = Math.Min(30, Math.Max(5, expectedSegmentDurationSeconds * 0.1));
        return previousDurationSeconds.Value + rolloverToleranceSeconds < expectedSegmentDurationSeconds;
    }

    private async Task EnsureDanmakuSegmentAsync(SessionProcessRuntime runtime, RecordTask recordTask, DateTimeOffset startedAt)
    {
        if (runtime.DanmakuRecorder is null)
        {
            return;
        }

        await runtime.DanmakuRecorder.StartSegmentAsync(
            recordTask.Id,
            recordTask.SegmentIndex,
            NormalizeAbsolutePath(recordTask.OutputFilePath ?? runtime.CurrentOutputFilePath),
            startedAt);
    }

    private async Task RunManualFinalizeTaskAsync(Guid recordTaskId)
    {
        try
        {
            using var scope = _serviceScopeFactory.CreateScope();
            var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
            var settingsService = scope.ServiceProvider.GetRequiredService<ISystemSettingsService>();
            var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();

            var recordTask = await dbContext.RecordTasks
                .Include(item => item.RecordSession)
                .ThenInclude(item => item!.RecordTasks)
                .Include(item => item.LiveRoom)
                .Include(item => item.Result)
                .FirstOrDefaultAsync(item => item.Id == recordTaskId);

            if (recordTask?.RecordSession is null)
            {
                ClearPostProcessState(recordTaskId);
                return;
            }

            var session = recordTask.RecordSession;
            var finalOutputPath = recordTask.OutputFilePath;
            if (string.IsNullOrWhiteSpace(finalOutputPath))
            {
                ClearPostProcessState(recordTaskId);
                return;
            }

            var recorderOutputPaths = ResolveManualFinalizeSourcePaths(recordTask, session);
            if (recorderOutputPaths.Count == 0)
            {
                ClearPostProcessState(recordTaskId);
                await logService.WriteAsync(
                    SystemLogLevel.Warning,
                    "FFmpeg",
                    "Manual MP4 finalization was skipped because the intermediate file was missing.",
                    GetRecorderSegmentsManifestPath(finalOutputPath),
                    recordTask.LiveRoomId,
                    recordTask.RecordSessionId,
                    recordTask.Id);
                return;
            }

            foreach (var recorderOutputPath in recorderOutputPaths)
            {
                await WaitForFileToStabilizeAsync(recorderOutputPath, CancellationToken.None);
            }

            var settings = await settingsService.GetAsync();
            var endedAt = recordTask.EndedAt ?? DateTimeOffset.UtcNow;
            var durationSeconds = recordTask.DurationSeconds ?? recordTask.Result?.DurationSeconds;
            var finalizationResult = await TryFinalizeTaskOutputAsync(
                settings.FfmpegPath,
                settings.MaxConcurrentFfmpegTranscodeTasks,
                settings.Mp4FinalizeTimeoutMinutes,
                session,
                recordTask,
                durationSeconds,
                recorderOutputPaths);

            var effectiveOutputPath = finalizationResult.OutputPath;
            var finalizationError = finalizationResult.ErrorMessage;
            var fileSize = CalculateFileSize(effectiveOutputPath);
            var mediaValidation = await ValidateMediaArtifactAsync(
                effectiveOutputPath,
                session.OutputFormat,
                CancellationToken.None);
            durationSeconds = mediaValidation.DurationSeconds;
            var danmakuPath = recordTask.Result?.DanmakuFilePath ?? GuessDanmakuPath(recordTask.OutputFilePath);
            var danmakuMessageCount = recordTask.Result?.DanmakuMessageCount ?? CountDanmakuMessages(danmakuPath);

            if (IsLowStoragePauseError(finalizationError) || IsShutdownFinalizationPauseError(finalizationError))
            {
                recordTask.MarkProcessing(finalizationError, endedAt);
            }
            else if (string.IsNullOrWhiteSpace(finalizationError))
            {
                if (!mediaValidation.IsValid)
                {
                    recordTask.MarkFailed(mediaValidation.ErrorMessage!, endedAt, durationSeconds);
                }
                else if (recordTask.Status == RecordTaskStatus.Stopped)
                {
                    recordTask.MarkStopped(endedAt, durationSeconds);
                }
                else
                {
                    recordTask.MarkCompleted(endedAt, durationSeconds);
                }
            }
            else if (recordTask.Status == RecordTaskStatus.Stopped)
            {
                recordTask.MarkStopped(endedAt, durationSeconds, finalizationError);
            }
            else
            {
                recordTask.MarkFailed(finalizationError, endedAt, durationSeconds);
            }

            await UpsertRecordResultAsync(
                recordTask,
                dbContext,
                effectiveOutputPath,
                fileSize,
                durationSeconds,
                danmakuPath,
                danmakuMessageCount,
                endedAt,
                mediaValidatedForDispatch: mediaValidation.IsValid);

            if (recordTask.Status == RecordTaskStatus.Completed &&
                session.Status == RecordSessionStatus.Stopped &&
                session.RecordTasks.All(static task =>
                    task.Status is RecordTaskStatus.Completed or RecordTaskStatus.Stopped))
            {
                session.MarkCompleted(endedAt);
            }

            await dbContext.SaveChangesAsync();

            if ((recordTask.Status is RecordTaskStatus.Completed or RecordTaskStatus.Stopped) &&
                HasUsableOutput(effectiveOutputPath, fileSize))
            {
                var completionDispatchService = scope.ServiceProvider.GetRequiredService<CompletionDispatchService>();
                await completionDispatchService.TryDispatchTaskAsync(recordTask.Id, CancellationToken.None);
            }

            await logService.WriteAsync(
                string.IsNullOrWhiteSpace(finalizationError) ? SystemLogLevel.Info : SystemLogLevel.Warning,
                "FFmpeg",
                string.IsNullOrWhiteSpace(finalizationError)
                    ? "Manual MP4 finalization completed."
                    : "Manual MP4 finalization finished with warnings.",
                string.IsNullOrWhiteSpace(finalizationError)
                    ? effectiveOutputPath
                    : $"{effectiveOutputPath}; {finalizationError}",
                recordTask.LiveRoomId,
                recordTask.RecordSessionId,
                recordTask.Id);
        }
        catch (Exception ex)
        {
            _logger.LogWarning(ex, "Manual MP4 finalization failed for task {RecordTaskId}", recordTaskId);

            try
            {
                using var scope = _serviceScopeFactory.CreateScope();
                var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
                await logService.WriteAsync(
                    SystemLogLevel.Warning,
                    "FFmpeg",
                    "Manual MP4 finalization failed to start or complete.",
                    ex.ToString(),
                    recordTaskId: recordTaskId);
            }
            catch (Exception logEx)
            {
                _logger.LogWarning(logEx, "Persist manual MP4 finalization failure log failed for task {RecordTaskId}", recordTaskId);
            }
        }
        finally
        {
            ClearPostProcessState(recordTaskId);
        }
    }

    private sealed class SessionProcessRuntime : IDisposable
    {
        public SessionProcessRuntime(
            Guid recordSessionId,
            Guid liveRoomId,
            string streamUrl,
            string outputPathPattern,
            string recorderOutputPath,
            RecordOutputFormat outputFormat,
            RecordSaveMode saveMode,
            Guid currentTaskId,
            int currentSegmentIndex,
            string currentOutputFilePath,
            string selectedQuality,
            string selectedProtocol,
            string? selectedVideoCodec,
            StreamInputHeaders? inputHeaders,
            RecordingExecutionSettings recordingSettings,
            FfmpegRecoveryContext recoveryContext,
            RecoveryVideoEncoderSelection recoveryVideoEncoder)
        {
            RecordSessionId = recordSessionId;
            LiveRoomId = liveRoomId;
            StreamUrl = streamUrl;
            OutputPathPattern = outputPathPattern;
            RecorderOutputPath = recorderOutputPath;
            OutputFormat = outputFormat;
            SaveMode = saveMode;
            CurrentTaskId = currentTaskId;
            CurrentSegmentIndex = currentSegmentIndex;
            CurrentOutputFilePath = currentOutputFilePath;
            SelectedQuality = selectedQuality;
            SelectedProtocol = selectedProtocol;
            SelectedVideoCodec = selectedVideoCodec;
            InputHeaders = inputHeaders;
            RecordingSettings = recordingSettings;
            RecoveryContext = recoveryContext;
            RecoveryVideoEncoder = recoveryVideoEncoder;
        }

        public Guid RecordSessionId { get; }
        public Guid LiveRoomId { get; }
        public string StreamUrl { get; }
        public string OutputPathPattern { get; }
        public string RecorderOutputPath { get; }
        public RecordOutputFormat OutputFormat { get; }
        public RecordSaveMode SaveMode { get; }
        public Guid CurrentTaskId { get; set; }
        public int CurrentSegmentIndex { get; set; }
        public string CurrentOutputFilePath { get; set; }
        public string SelectedQuality { get; }
        public string SelectedProtocol { get; }
        public string? SelectedVideoCodec { get; }
        public StreamInputHeaders? InputHeaders { get; }
        public RecordingExecutionSettings RecordingSettings { get; }
        public FfmpegRecoveryContext RecoveryContext { get; }
        public RecoveryVideoEncoderSelection RecoveryVideoEncoder { get; }
        public FfmpegInputOptionProfile InputOptionProfile => RecoveryContext.InputOptionProfile;
        public bool HasRetriedWithCompatibilityProfile => RecoveryContext.HasRetriedWithCompatibilityProfile;
        public bool HasRetriedWithRefreshedStream => RecoveryContext.HasRetriedWithRefreshedStream;
        public bool HasRetriedWithAlternateProtocol => RecoveryContext.HasRetriedWithAlternateProtocol;
        public bool HasTimestampDiscontinuityFailure { get; private set; }
        public bool HasTimestampMuxerFailure { get; private set; }
        public bool HasHlsOverlongHeadersFailure { get; private set; }
        public Process? Process { get; private set; }
        public Process? CurlProcess { get; private set; }
        public int ProcessId => Process?.Id ?? 0;
        public bool CompletionRequested { get; private set; }
        public bool StopRequested { get; private set; }
        public bool ShutdownRequested { get; private set; }
        public bool WasForceKilled { get; private set; }
        public bool HasInitializedDanmaku { get; set; }
        public bool HasOpenedFirstSegment { get; set; }
        public int RetryAttemptCount => RecoveryContext.AttemptCount;
        public DateTimeOffset? ProcessStartedAt { get; private set; }
        public StartupFailureKind StartupFailureKind { get; private set; }
        public string? LastStartupFailureLine { get; private set; }
        public SemaphoreSlim Gate { get; } = new(1, 1);
        public CancellationTokenSource DanmakuCancellation { get; } = new();
        public TaskCompletionSource<bool> ExitCompletion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
        public SessionDanmakuXmlRecorder? DanmakuRecorder { get; set; }
        public ILiveDanmakuConnection? DanmakuConnection { get; set; }
        public Task? DanmakuPumpTask { get; set; }
        private List<string> CurrentRecorderSegmentPaths { get; } = [];
        // Bandwidth tracking fields
        private long _lastBandwidthTotalSize;
        private double? _lastBandwidthBitrate;
        private double _lastBandwidthSpeed;
        private DateTimeOffset _lastBandwidthFlushAt = DateTimeOffset.MinValue;
        private static readonly TimeSpan BandwidthFlushInterval = TimeSpan.FromSeconds(30);

        public void UpdateBandwidthTotalSize(long totalSize)
        {
            _lastBandwidthTotalSize = Math.Max(0, totalSize);
        }

        public void UpdateBandwidthBitrate(double bitrateKbps)
        {
            _lastBandwidthBitrate = Math.Max(0, bitrateKbps);
        }

        public void UpdateBandwidthSpeed(double speed)
        {
            _lastBandwidthSpeed = speed;
        }

        public async Task FlushBandwidthSampleIfNeededAsync(
            Func<Guid, Guid, Guid, string, System.Threading.CancellationToken, Task> writeLogAsync,
            System.Threading.CancellationToken cancellationToken)
        {
            var nowUtc = DateTimeOffset.UtcNow;
            if (nowUtc - _lastBandwidthFlushAt < BandwidthFlushInterval)
            {
                return;
            }

            _lastBandwidthFlushAt = nowUtc;

            if (_lastBandwidthTotalSize <= 0 && !_lastBandwidthBitrate.HasValue)
            {
                return;
            }

            var detail = $$"""{"bytesDownloaded":{{_lastBandwidthTotalSize}},"bitrateKbps":{{(_lastBandwidthBitrate?.ToString("F1", CultureInfo.InvariantCulture) ?? "null")}},"speed":{{_lastBandwidthSpeed.ToString("F2", CultureInfo.InvariantCulture)}}}""";

            await writeLogAsync(
                LiveRoomId,
                RecordSessionId,
                CurrentTaskId,
                detail,
                cancellationToken);
        }
        private object RuntimeSourceFailureSync { get; } = new();
        private object RecentOutputSync { get; } = new();
        private Queue<string> RecentOutputLines { get; } = new();
        private Queue<string> RecentCurlErrorLines { get; } = new();
        private DateTimeOffset RuntimeSourceFailureWindowStartedAt { get; set; }
        private int RuntimeSourceFailureCount { get; set; }
        private bool RuntimeSourceFailureVerificationInProgress { get; set; }

        public void AttachProcess(Process process)
        {
            Process = process;
            ProcessStartedAt = DateTimeOffset.UtcNow;
        }
        public void AttachCurlProcess(Process curlProcess) => CurlProcess = curlProcess;
        public int? CurlExitCode
        {
            get
            {
                try { return CurlProcess?.HasExited == true ? CurlProcess.ExitCode : null; }
                catch { return null; }
            }
        }
        public void MarkTimestampDiscontinuityFailure() => HasTimestampDiscontinuityFailure = true;
        public void MarkTimestampMuxerFailure() => HasTimestampMuxerFailure = true;
        public void MarkHlsOverlongHeadersFailure() => HasHlsOverlongHeadersFailure = true;

        public void MarkStopRequested(bool markAsCompletedOnExit)
        {
            ShutdownRequested = false;
            if (markAsCompletedOnExit)
            {
                CompletionRequested = true;
                StopRequested = false;
            }
            else
            {
                CompletionRequested = false;
                StopRequested = true;
            }
        }

        public void MarkShutdownRequested()
        {
            ShutdownRequested = true;
            CompletionRequested = true;
            StopRequested = false;
        }

        public void MarkStartupFailure(StartupFailureKind kind, string line)
        {
            if ((StartupFailureKind == StartupFailureKind.InputOptionCompatibility ||
                 StartupFailureKind == StartupFailureKind.HardwareEncoderUnavailable) &&
                kind is not StartupFailureKind.InputOptionCompatibility and not StartupFailureKind.HardwareEncoderUnavailable)
            {
                return;
            }

            StartupFailureKind = kind;
            LastStartupFailureLine = line;
        }

        public void MarkForceKilled() => WasForceKilled = true;

        public void RememberOutputLine(string line, bool isError)
        {
            lock (RecentOutputSync)
            {
                RecentOutputLines.Enqueue($"{(isError ? "stderr" : "stdout")}={line.Trim()}");
                while (RecentOutputLines.Count > 8)
                {
                    RecentOutputLines.Dequeue();
                }
            }
        }

        public string? GetRecentOutputSummary()
        {
            lock (RecentOutputSync)
            {
                return RecentOutputLines.Count == 0
                    ? null
                    : string.Join(" | ", RecentOutputLines);
            }
        }

        public void RememberCurlErrorLine(string line)
        {
            lock (RecentOutputSync)
            {
                RecentCurlErrorLines.Enqueue(line.Trim());
                while (RecentCurlErrorLines.Count > 8)
                {
                    RecentCurlErrorLines.Dequeue();
                }
            }
        }

        public string? GetRecentCurlErrorSummary()
        {
            lock (RecentOutputSync)
            {
                return RecentCurlErrorLines.Count == 0
                    ? null
                    : string.Join(" | ", RecentCurlErrorLines);
            }
        }

        public void ResetCurrentRecorderSegmentPaths(string openedPath)
        {
            CurrentRecorderSegmentPaths.Clear();
            CurrentRecorderSegmentPaths.Add(openedPath);
        }

        public void TrackCurrentRecorderSegment(string openedPath)
        {
            if (CurrentRecorderSegmentPaths.Count == 0 ||
                !string.Equals(CurrentRecorderSegmentPaths[^1], openedPath, StringComparison.OrdinalIgnoreCase))
            {
                CurrentRecorderSegmentPaths.Add(openedPath);
            }
        }

        public IReadOnlyList<string> CaptureCurrentRecorderSegments()
        {
            if (CurrentRecorderSegmentPaths.Count == 0)
            {
                return [CurrentOutputFilePath];
            }

            return CurrentRecorderSegmentPaths.ToArray();
        }

        public bool RegisterRuntimeSourceFailure(DateTimeOffset observedAt, TimeSpan window)
        {
            lock (RuntimeSourceFailureSync)
            {
                if (RuntimeSourceFailureVerificationInProgress)
                {
                    return false;
                }

                if (RuntimeSourceFailureWindowStartedAt == default ||
                    observedAt - RuntimeSourceFailureWindowStartedAt > window)
                {
                    RuntimeSourceFailureWindowStartedAt = observedAt;
                    RuntimeSourceFailureCount = 0;
                }

                RuntimeSourceFailureCount++;
                if (RuntimeSourceFailureCount < 3)
                {
                    return false;
                }

                RuntimeSourceFailureVerificationInProgress = true;
                RuntimeSourceFailureWindowStartedAt = observedAt;
                RuntimeSourceFailureCount = 0;
                return true;
            }
        }

        public void CompleteRuntimeSourceFailureVerification()
        {
            lock (RuntimeSourceFailureSync)
            {
                RuntimeSourceFailureVerificationInProgress = false;
                RuntimeSourceFailureWindowStartedAt = DateTimeOffset.UtcNow;
                RuntimeSourceFailureCount = 0;
            }
        }

        public void Dispose()
        {
            if (CurlProcess is not null)
            {
                try { if (!CurlProcess.HasExited) CurlProcess.Kill(true); } catch { }
                CurlProcess.Dispose();
            }
            DanmakuCancellation.Dispose();
            Gate.Dispose();
        }
    }

    private sealed class SessionTransitionRuntime : IDisposable
    {
        private readonly CancellationTokenSource _cancellation = new();

        public SessionTransitionRuntime(Guid recordSessionId) => RecordSessionId = recordSessionId;

        public Guid RecordSessionId { get; }
        public bool StopRequested { get; private set; }
        public bool MarkAsCompletedOnExit { get; private set; }
        public CancellationToken Token => _cancellation.Token;
        public TaskCompletionSource<bool> Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
        private int _attemptCount;

        public int NextAttempt(int minimum)
        {
            var next = Interlocked.Increment(ref _attemptCount);
            if (next >= minimum)
            {
                return next;
            }

            Interlocked.Exchange(ref _attemptCount, minimum);
            return minimum;
        }

        public void RequestStop(bool markAsCompletedOnExit)
        {
            StopRequested = true;
            MarkAsCompletedOnExit = markAsCompletedOnExit;
            _cancellation.Cancel();
        }

        public void Dispose() => _cancellation.Dispose();
    }
}

internal enum ExitedRecordingDisposition
{
    Processing,
    Stopped,
    Completed,
    Failed
}