fix: improve recovery and auto-start flows

This commit is contained in:
2026-04-25 00:06:25 +08:00
parent 39db8c012f
commit 9701c88c8b
5 changed files with 207 additions and 14 deletions
@@ -4,6 +4,7 @@ using LiveRecorder.Application.Abstractions.Platforms;
using LiveRecorder.Application.Abstractions.Recording;
using LiveRecorder.Application.Abstractions.Settings;
using LiveRecorder.Application.Models.LiveRooms;
using LiveRecorder.Application.Models.RecordTasks;
using LiveRecorder.Domain.Entities;
using LiveRecorder.Domain.Enums;
@@ -17,6 +18,7 @@ public sealed class LiveRoomService
private readonly LiveRoomRecordingSettingsResolver _liveRoomRecordingSettingsResolver;
private readonly ISystemSettingsService _systemSettingsService;
private readonly StoppedOrphanRecordSessionCleanupService _stoppedOrphanRecordSessionCleanupService;
private readonly RecordService _recordService;
private readonly IUnitOfWork _unitOfWork;
private readonly ISystemLogService _systemLogService;
@@ -27,6 +29,7 @@ public sealed class LiveRoomService
LiveRoomRecordingSettingsResolver liveRoomRecordingSettingsResolver,
ISystemSettingsService systemSettingsService,
StoppedOrphanRecordSessionCleanupService stoppedOrphanRecordSessionCleanupService,
RecordService recordService,
IUnitOfWork unitOfWork,
ISystemLogService systemLogService)
{
@@ -36,6 +39,7 @@ public sealed class LiveRoomService
_liveRoomRecordingSettingsResolver = liveRoomRecordingSettingsResolver;
_systemSettingsService = systemSettingsService;
_stoppedOrphanRecordSessionCleanupService = stoppedOrphanRecordSessionCleanupService;
_recordService = recordService;
_unitOfWork = unitOfWork;
_systemLogService = systemLogService;
}
@@ -214,6 +218,7 @@ public sealed class LiveRoomService
$"Live room resolved: {liveRoom.RoomId} ({liveRoom.Platform}).",
liveRoomId: liveRoom.Id,
cancellationToken: cancellationToken);
await TryAutoStartRecordingAsync(liveRoom, liveStatus, cancellationToken);
var effectiveSettings = await _liveRoomRecordingSettingsResolver.ResolveAsync(liveRoom, cancellationToken);
return (liveRoom, effectiveSettings, created);
@@ -238,6 +243,7 @@ public sealed class LiveRoomService
detail: $"status={liveStatus.StatusCode}, rawStatus={liveStatus.RawStatus}",
liveRoomId: room.Id,
cancellationToken: cancellationToken);
await TryAutoStartRecordingAsync(room, liveStatus, cancellationToken);
var effectiveSettings = await _liveRoomRecordingSettingsResolver.ResolveAsync(room, cancellationToken);
return Map(room, effectiveSettings);
@@ -405,6 +411,56 @@ public sealed class LiveRoomService
return rooms.ToDictionary(item => item.Id, item => _liveRoomRecordingSettingsResolver.Resolve(item, systemSettings));
}
private async Task TryAutoStartRecordingAsync(
LiveRoom liveRoom,
LiveStatusSnapshot liveStatus,
CancellationToken cancellationToken)
{
if (!liveStatus.IsLive || !liveRoom.IsEnabled)
{
return;
}
var settings = await _systemSettingsService.GetAsync(cancellationToken);
if (!settings.AutoStartRecordingOnLive)
{
await _systemLogService.WriteAsync(
SystemLogLevel.Info,
"LiveRoom",
"Live room is already online, but immediate auto-start is disabled by settings.",
liveRoomId: liveRoom.Id,
cancellationToken: cancellationToken);
return;
}
await _systemLogService.WriteAsync(
SystemLogLevel.Info,
"LiveRoom",
"Live room is already online. Attempting immediate auto-start after resolve/refresh.",
liveRoomId: liveRoom.Id,
cancellationToken: cancellationToken);
try
{
await _recordService.StartAsync(
new StartRecordTaskRequest
{
LiveRoomId = liveRoom.Id
},
cancellationToken);
}
catch (Exception ex)
{
await _systemLogService.WriteAsync(
ex is InvalidOperationException ? SystemLogLevel.Warning : SystemLogLevel.Error,
"LiveRoom",
"Immediate auto-start after resolve/refresh did not start a recording session.",
ex.Message,
liveRoomId: liveRoom.Id,
cancellationToken: cancellationToken);
}
}
private LiveRoomDto Map(LiveRoom room, RecordingExecutionSettings effectiveSettings) => new()
{
Id = room.Id,
@@ -80,7 +80,9 @@ public sealed partial class FfmpegService
var stderrLines = new Queue<string>();
var progressSync = new object();
double? processedSeconds = null;
var lastReportedWholePercent = -1;
double? lastReportedProgressPercent = null;
string? lastReportedStage = null;
string? lastReportedDetail = null;
var attemptStartedAt = DateTimeOffset.UtcNow;
var activeStage = "Finalizing MP4";
var maxDuration = TimeSpan.FromMinutes(Math.Clamp(mp4FinalizeTimeoutMinutes, 1, 1440));
@@ -114,19 +116,28 @@ public sealed partial class FfmpegService
detail = $"Processed {processedSeconds.Value:F1}s";
}
var wholePercent = progressPercent.HasValue ? (int)Math.Floor(progressPercent.Value) : -1;
if (stageOverride is null && wholePercent == lastReportedWholePercent)
var effectiveStage = stageOverride ?? activeStage;
var normalizedProgressPercent = progressPercent.HasValue
? Math.Round(progressPercent.Value, 1, MidpointRounding.AwayFromZero)
: (double?)null;
var effectiveDetail = detail ?? $"Optimizing MP4 index for {Path.GetFileName(targetPath)}";
if (string.Equals(effectiveStage, lastReportedStage, StringComparison.Ordinal) &&
string.Equals(effectiveDetail, lastReportedDetail, StringComparison.Ordinal) &&
Nullable.Equals(normalizedProgressPercent, lastReportedProgressPercent))
{
return;
}
lastReportedWholePercent = wholePercent;
lastReportedStage = effectiveStage;
lastReportedDetail = effectiveDetail;
lastReportedProgressPercent = normalizedProgressPercent;
SetPostProcessState(
recordSessionId,
recordTaskId,
stageOverride ?? activeStage,
progressPercent,
detail ?? $"Optimizing MP4 index for {Path.GetFileName(targetPath)}");
effectiveStage,
normalizedProgressPercent,
effectiveDetail);
}
}
@@ -159,7 +170,9 @@ public sealed partial class FfmpegService
ClearErrorDetail();
activeStage = stage;
processedSeconds = null;
lastReportedWholePercent = -1;
lastReportedProgressPercent = null;
lastReportedStage = null;
lastReportedDetail = null;
attemptStartedAt = DateTimeOffset.UtcNow;
System.Threading.Interlocked.Exchange(ref lastActivityTicks, attemptStartedAt.UtcTicks);
@@ -242,7 +242,7 @@ public sealed partial class FfmpegService : IFfmpegService
public async Task<bool> TryReconcileInactiveSessionAsync(Guid recordSessionId, CancellationToken cancellationToken = default)
{
if (IsRunning(recordSessionId) || IsSessionUnderPostProcessing(recordSessionId))
if (IsRunning(recordSessionId))
{
return false;
}
@@ -269,11 +269,23 @@ public sealed partial class FfmpegService : IFfmpegService
.ThenBy(static item => item.CreatedAt)
.ToList();
var anyUsableOutput = false;
var anyUsableOutput = tasks.Any(static item => item.Status == RecordTaskStatus.Completed);
var hasBackgroundPostProcessing = false;
string? sessionFinalizationError = null;
foreach (var task in tasks.Where(item => IsActiveTaskStatus(item.Status)))
foreach (var task in tasks)
{
if (IsTaskUnderPostProcessing(task.Id))
{
hasBackgroundPostProcessing = true;
continue;
}
if (!IsActiveTaskStatus(task.Status))
{
continue;
}
var durationSeconds = task.StartedAt.HasValue
? (double?)Math.Max(0, (endedAt - task.StartedAt.Value).TotalSeconds)
: null;
@@ -299,6 +311,7 @@ public sealed partial class FfmpegService : IFfmpegService
if (IsLowStoragePauseError(taskFinalizationError))
{
task.MarkProcessing(taskFinalizationError, endedAt);
hasBackgroundPostProcessing = true;
}
else if (!string.IsNullOrWhiteSpace(taskFinalizationError))
{
@@ -323,6 +336,12 @@ public sealed partial class FfmpegService : IFfmpegService
{
recordSession.MarkFailed(sessionFinalizationError, endedAt);
}
else if (hasBackgroundPostProcessing)
{
recordSession.MarkStopped(
endedAt,
"Recording process was no longer running when the session was reconciled. Post-processing will continue in background.");
}
else if (anyUsableOutput)
{
recordSession.MarkCompleted(endedAt);
@@ -594,8 +613,8 @@ public sealed partial class FfmpegService : IFfmpegService
private void ClearPostProcessState(Guid recordTaskId) =>
_postProcessStates.TryRemove(recordTaskId, out _);
private bool IsSessionUnderPostProcessing(Guid recordSessionId) =>
_postProcessStates.Values.Any(entry => entry.RecordSessionId == recordSessionId);
private bool IsTaskUnderPostProcessing(Guid recordTaskId) =>
_postProcessStates.ContainsKey(recordTaskId);
private async Task<IDisposable> AcquireTranscodeSlotAsync(int maxConcurrentTasks, CancellationToken cancellationToken)
{
@@ -93,7 +93,14 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
try
{
using var notificationScope = _serviceScopeFactory.CreateScope();
var logService = notificationScope.ServiceProvider.GetRequiredService<ISystemLogService>();
var emailNotificationService = notificationScope.ServiceProvider.GetRequiredService<IEmailNotificationService>();
await logService.WriteAsync(
SystemLogLevel.Error,
"Scheduler",
"Background live room polling failed.",
ex.ToString(),
cancellationToken: CancellationToken.None);
if (ShouldSendExceptionEmail(BuildExceptionEmailKey("background-loop", ex)))
{
await emailNotificationService.SendExceptionAsync(