fix: improve recovery and auto-start flows
This commit is contained in:
@@ -4,6 +4,7 @@ using LiveRecorder.Application.Abstractions.Platforms;
|
||||
using LiveRecorder.Application.Abstractions.Recording;
|
||||
using LiveRecorder.Application.Abstractions.Settings;
|
||||
using LiveRecorder.Application.Models.LiveRooms;
|
||||
using LiveRecorder.Application.Models.RecordTasks;
|
||||
using LiveRecorder.Domain.Entities;
|
||||
using LiveRecorder.Domain.Enums;
|
||||
|
||||
@@ -17,6 +18,7 @@ public sealed class LiveRoomService
|
||||
private readonly LiveRoomRecordingSettingsResolver _liveRoomRecordingSettingsResolver;
|
||||
private readonly ISystemSettingsService _systemSettingsService;
|
||||
private readonly StoppedOrphanRecordSessionCleanupService _stoppedOrphanRecordSessionCleanupService;
|
||||
private readonly RecordService _recordService;
|
||||
private readonly IUnitOfWork _unitOfWork;
|
||||
private readonly ISystemLogService _systemLogService;
|
||||
|
||||
@@ -27,6 +29,7 @@ public sealed class LiveRoomService
|
||||
LiveRoomRecordingSettingsResolver liveRoomRecordingSettingsResolver,
|
||||
ISystemSettingsService systemSettingsService,
|
||||
StoppedOrphanRecordSessionCleanupService stoppedOrphanRecordSessionCleanupService,
|
||||
RecordService recordService,
|
||||
IUnitOfWork unitOfWork,
|
||||
ISystemLogService systemLogService)
|
||||
{
|
||||
@@ -36,6 +39,7 @@ public sealed class LiveRoomService
|
||||
_liveRoomRecordingSettingsResolver = liveRoomRecordingSettingsResolver;
|
||||
_systemSettingsService = systemSettingsService;
|
||||
_stoppedOrphanRecordSessionCleanupService = stoppedOrphanRecordSessionCleanupService;
|
||||
_recordService = recordService;
|
||||
_unitOfWork = unitOfWork;
|
||||
_systemLogService = systemLogService;
|
||||
}
|
||||
@@ -214,6 +218,7 @@ public sealed class LiveRoomService
|
||||
$"Live room resolved: {liveRoom.RoomId} ({liveRoom.Platform}).",
|
||||
liveRoomId: liveRoom.Id,
|
||||
cancellationToken: cancellationToken);
|
||||
await TryAutoStartRecordingAsync(liveRoom, liveStatus, cancellationToken);
|
||||
|
||||
var effectiveSettings = await _liveRoomRecordingSettingsResolver.ResolveAsync(liveRoom, cancellationToken);
|
||||
return (liveRoom, effectiveSettings, created);
|
||||
@@ -238,6 +243,7 @@ public sealed class LiveRoomService
|
||||
detail: $"status={liveStatus.StatusCode}, rawStatus={liveStatus.RawStatus}",
|
||||
liveRoomId: room.Id,
|
||||
cancellationToken: cancellationToken);
|
||||
await TryAutoStartRecordingAsync(room, liveStatus, cancellationToken);
|
||||
|
||||
var effectiveSettings = await _liveRoomRecordingSettingsResolver.ResolveAsync(room, cancellationToken);
|
||||
return Map(room, effectiveSettings);
|
||||
@@ -405,6 +411,56 @@ public sealed class LiveRoomService
|
||||
return rooms.ToDictionary(item => item.Id, item => _liveRoomRecordingSettingsResolver.Resolve(item, systemSettings));
|
||||
}
|
||||
|
||||
private async Task TryAutoStartRecordingAsync(
|
||||
LiveRoom liveRoom,
|
||||
LiveStatusSnapshot liveStatus,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!liveStatus.IsLive || !liveRoom.IsEnabled)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var settings = await _systemSettingsService.GetAsync(cancellationToken);
|
||||
if (!settings.AutoStartRecordingOnLive)
|
||||
{
|
||||
await _systemLogService.WriteAsync(
|
||||
SystemLogLevel.Info,
|
||||
"LiveRoom",
|
||||
"Live room is already online, but immediate auto-start is disabled by settings.",
|
||||
liveRoomId: liveRoom.Id,
|
||||
cancellationToken: cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
await _systemLogService.WriteAsync(
|
||||
SystemLogLevel.Info,
|
||||
"LiveRoom",
|
||||
"Live room is already online. Attempting immediate auto-start after resolve/refresh.",
|
||||
liveRoomId: liveRoom.Id,
|
||||
cancellationToken: cancellationToken);
|
||||
|
||||
try
|
||||
{
|
||||
await _recordService.StartAsync(
|
||||
new StartRecordTaskRequest
|
||||
{
|
||||
LiveRoomId = liveRoom.Id
|
||||
},
|
||||
cancellationToken);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _systemLogService.WriteAsync(
|
||||
ex is InvalidOperationException ? SystemLogLevel.Warning : SystemLogLevel.Error,
|
||||
"LiveRoom",
|
||||
"Immediate auto-start after resolve/refresh did not start a recording session.",
|
||||
ex.Message,
|
||||
liveRoomId: liveRoom.Id,
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
private LiveRoomDto Map(LiveRoom room, RecordingExecutionSettings effectiveSettings) => new()
|
||||
{
|
||||
Id = room.Id,
|
||||
|
||||
@@ -80,7 +80,9 @@ public sealed partial class FfmpegService
|
||||
var stderrLines = new Queue<string>();
|
||||
var progressSync = new object();
|
||||
double? processedSeconds = null;
|
||||
var lastReportedWholePercent = -1;
|
||||
double? lastReportedProgressPercent = null;
|
||||
string? lastReportedStage = null;
|
||||
string? lastReportedDetail = null;
|
||||
var attemptStartedAt = DateTimeOffset.UtcNow;
|
||||
var activeStage = "Finalizing MP4";
|
||||
var maxDuration = TimeSpan.FromMinutes(Math.Clamp(mp4FinalizeTimeoutMinutes, 1, 1440));
|
||||
@@ -114,19 +116,28 @@ public sealed partial class FfmpegService
|
||||
detail = $"Processed {processedSeconds.Value:F1}s";
|
||||
}
|
||||
|
||||
var wholePercent = progressPercent.HasValue ? (int)Math.Floor(progressPercent.Value) : -1;
|
||||
if (stageOverride is null && wholePercent == lastReportedWholePercent)
|
||||
var effectiveStage = stageOverride ?? activeStage;
|
||||
var normalizedProgressPercent = progressPercent.HasValue
|
||||
? Math.Round(progressPercent.Value, 1, MidpointRounding.AwayFromZero)
|
||||
: (double?)null;
|
||||
var effectiveDetail = detail ?? $"Optimizing MP4 index for {Path.GetFileName(targetPath)}";
|
||||
|
||||
if (string.Equals(effectiveStage, lastReportedStage, StringComparison.Ordinal) &&
|
||||
string.Equals(effectiveDetail, lastReportedDetail, StringComparison.Ordinal) &&
|
||||
Nullable.Equals(normalizedProgressPercent, lastReportedProgressPercent))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
lastReportedWholePercent = wholePercent;
|
||||
lastReportedStage = effectiveStage;
|
||||
lastReportedDetail = effectiveDetail;
|
||||
lastReportedProgressPercent = normalizedProgressPercent;
|
||||
SetPostProcessState(
|
||||
recordSessionId,
|
||||
recordTaskId,
|
||||
stageOverride ?? activeStage,
|
||||
progressPercent,
|
||||
detail ?? $"Optimizing MP4 index for {Path.GetFileName(targetPath)}");
|
||||
effectiveStage,
|
||||
normalizedProgressPercent,
|
||||
effectiveDetail);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -159,7 +170,9 @@ public sealed partial class FfmpegService
|
||||
ClearErrorDetail();
|
||||
activeStage = stage;
|
||||
processedSeconds = null;
|
||||
lastReportedWholePercent = -1;
|
||||
lastReportedProgressPercent = null;
|
||||
lastReportedStage = null;
|
||||
lastReportedDetail = null;
|
||||
attemptStartedAt = DateTimeOffset.UtcNow;
|
||||
System.Threading.Interlocked.Exchange(ref lastActivityTicks, attemptStartedAt.UtcTicks);
|
||||
|
||||
|
||||
@@ -242,7 +242,7 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
|
||||
public async Task<bool> TryReconcileInactiveSessionAsync(Guid recordSessionId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (IsRunning(recordSessionId) || IsSessionUnderPostProcessing(recordSessionId))
|
||||
if (IsRunning(recordSessionId))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
@@ -269,11 +269,23 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
.ThenBy(static item => item.CreatedAt)
|
||||
.ToList();
|
||||
|
||||
var anyUsableOutput = false;
|
||||
var anyUsableOutput = tasks.Any(static item => item.Status == RecordTaskStatus.Completed);
|
||||
var hasBackgroundPostProcessing = false;
|
||||
string? sessionFinalizationError = null;
|
||||
|
||||
foreach (var task in tasks.Where(item => IsActiveTaskStatus(item.Status)))
|
||||
foreach (var task in tasks)
|
||||
{
|
||||
if (IsTaskUnderPostProcessing(task.Id))
|
||||
{
|
||||
hasBackgroundPostProcessing = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!IsActiveTaskStatus(task.Status))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var durationSeconds = task.StartedAt.HasValue
|
||||
? (double?)Math.Max(0, (endedAt - task.StartedAt.Value).TotalSeconds)
|
||||
: null;
|
||||
@@ -299,6 +311,7 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
if (IsLowStoragePauseError(taskFinalizationError))
|
||||
{
|
||||
task.MarkProcessing(taskFinalizationError, endedAt);
|
||||
hasBackgroundPostProcessing = true;
|
||||
}
|
||||
else if (!string.IsNullOrWhiteSpace(taskFinalizationError))
|
||||
{
|
||||
@@ -323,6 +336,12 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
{
|
||||
recordSession.MarkFailed(sessionFinalizationError, endedAt);
|
||||
}
|
||||
else if (hasBackgroundPostProcessing)
|
||||
{
|
||||
recordSession.MarkStopped(
|
||||
endedAt,
|
||||
"Recording process was no longer running when the session was reconciled. Post-processing will continue in background.");
|
||||
}
|
||||
else if (anyUsableOutput)
|
||||
{
|
||||
recordSession.MarkCompleted(endedAt);
|
||||
@@ -594,8 +613,8 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
private void ClearPostProcessState(Guid recordTaskId) =>
|
||||
_postProcessStates.TryRemove(recordTaskId, out _);
|
||||
|
||||
private bool IsSessionUnderPostProcessing(Guid recordSessionId) =>
|
||||
_postProcessStates.Values.Any(entry => entry.RecordSessionId == recordSessionId);
|
||||
private bool IsTaskUnderPostProcessing(Guid recordTaskId) =>
|
||||
_postProcessStates.ContainsKey(recordTaskId);
|
||||
|
||||
private async Task<IDisposable> AcquireTranscodeSlotAsync(int maxConcurrentTasks, CancellationToken cancellationToken)
|
||||
{
|
||||
|
||||
@@ -93,7 +93,14 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
try
|
||||
{
|
||||
using var notificationScope = _serviceScopeFactory.CreateScope();
|
||||
var logService = notificationScope.ServiceProvider.GetRequiredService<ISystemLogService>();
|
||||
var emailNotificationService = notificationScope.ServiceProvider.GetRequiredService<IEmailNotificationService>();
|
||||
await logService.WriteAsync(
|
||||
SystemLogLevel.Error,
|
||||
"Scheduler",
|
||||
"Background live room polling failed.",
|
||||
ex.ToString(),
|
||||
cancellationToken: CancellationToken.None);
|
||||
if (ShouldSendExceptionEmail(BuildExceptionEmailKey("background-loop", ex)))
|
||||
{
|
||||
await emailNotificationService.SendExceptionAsync(
|
||||
|
||||
Reference in New Issue
Block a user