using LiveRecorder.Application.Abstractions.Recording; using LiveRecorder.Application.Abstractions.Settings; using LiveRecorder.Application.Abstractions.Storage; using LiveRecorder.Application.Common; using LiveRecorder.Application.Models.Recovery; using LiveRecorder.Application.Models.RecordTasks; using LiveRecorder.Application.Services; using LiveRecorder.Domain.Entities; using LiveRecorder.Domain.Enums; using LiveRecorder.Infrastructure.Persistence; using Microsoft.EntityFrameworkCore; namespace LiveRecorder.Infrastructure.Services; public sealed class RecoveryService { private readonly LiveRecorderDbContext _dbContext; private readonly ISystemSettingsService _systemSettingsService; private readonly IStorageGuardService _storageGuardService; private readonly IFfmpegService _ffmpegService; private readonly RecordService _recordService; public RecoveryService( LiveRecorderDbContext dbContext, ISystemSettingsService systemSettingsService, IStorageGuardService storageGuardService, IFfmpegService ffmpegService, RecordService recordService) { _dbContext = dbContext; _systemSettingsService = systemSettingsService; _storageGuardService = storageGuardService; _ffmpegService = ffmpegService; _recordService = recordService; } public async Task GetOverviewAsync(CancellationToken cancellationToken = default) { var settings = await _systemSettingsService.GetAsync(cancellationToken); var storage = _storageGuardService.CheckCanStartOrResume(settings); var liveRooms = await ListRecoverableLiveRoomsAsync(cancellationToken); var finalizations = await ListRecoverableFinalizationsAsync(cancellationToken); return new RecoveryOverviewDto { Storage = new StorageGuardStatusDto { IsEnabled = storage.IsEnabled, HasEnoughSpace = storage.HasEnoughSpace, CheckedPath = storage.CheckedPath, AvailableBytes = storage.AvailableBytes, RequiredBytes = storage.RequiredBytes, Message = storage.Message, Tier = storage.Tier.ToString(), UsagePercent = storage.UsagePercent }, LiveRooms = liveRooms, Finalizations = finalizations }; } public async Task RetryLiveRoomAsync(Guid liveRoomId, CancellationToken cancellationToken = default) { var room = await _dbContext.LiveRooms.FirstOrDefaultAsync(item => item.Id == liveRoomId, cancellationToken); if (room is null) { return FailureResult("Live room was not found."); } if (!room.IsEnabled) { return FailureResult("Live room is disabled and cannot be retried."); } if (room.AvailabilityStatus != LiveRoomAvailabilityStatus.Live) { return FailureResult("Live room is not currently online."); } var hasActiveSession = await _dbContext.RecordSessions.AnyAsync( item => item.LiveRoomId == room.Id && (item.Status == RecordSessionStatus.Starting || item.Status == RecordSessionStatus.Running || item.Status == RecordSessionStatus.Stopping), cancellationToken); if (hasActiveSession) { room.SetLastAutoStartDecision( AutoStartDecisionCodes.SkippedActiveSession, "Auto-start skipped because an active recording session already exists.", null, DateTimeOffset.UtcNow); await _dbContext.SaveChangesAsync(cancellationToken); return FailureResult("An active recording session already exists for this live room."); } try { var task = await _recordService.StartAsync( new StartRecordTaskRequest { LiveRoomId = liveRoomId }, trackAutoStartDecision: true, cancellationToken); var started = task.Status is RecordTaskStatus.Starting or RecordTaskStatus.Running; return new RecoveryActionResultDto { RequestedCount = 1, SuccessCount = started ? 1 : 0, FailedCount = started ? 0 : 1, Messages = [ started ? $"Recording retry started for room {room.RoomId}." : $"Recording retry did not start for room {room.RoomId}. Status={task.Status}; Error={task.ErrorMessage ?? "n/a"}" ] }; } catch (Exception ex) { return FailureResult($"Recording retry failed for room {room.RoomId}: {ex.Message}"); } } public async Task RetryAllLiveRoomsAsync(CancellationToken cancellationToken = default) { var liveRooms = await ListRecoverableLiveRoomsAsync(cancellationToken); if (liveRooms.Count == 0) { return new RecoveryActionResultDto { RequestedCount = 0, SuccessCount = 0, FailedCount = 0, Messages = ["No live rooms currently require retry."] }; } var messages = new List(); var successCount = 0; foreach (var item in liveRooms) { var result = await RetryLiveRoomAsync(item.LiveRoomId, cancellationToken); successCount += result.SuccessCount; messages.AddRange(result.Messages); } return new RecoveryActionResultDto { RequestedCount = liveRooms.Count, SuccessCount = successCount, FailedCount = liveRooms.Count - successCount, Messages = messages }; } public async Task ResumeFinalizationAsync(Guid recordTaskId, CancellationToken cancellationToken = default) { var started = await _ffmpegService.StartManualFinalizeTaskAsync(recordTaskId, cancellationToken); return new RecoveryActionResultDto { RequestedCount = 1, SuccessCount = started ? 1 : 0, FailedCount = started ? 0 : 1, Messages = [ started ? $"MP4 finalization resumed for task {recordTaskId}." : $"MP4 finalization could not be resumed for task {recordTaskId}." ] }; } public async Task ResumeAllFinalizationsAsync(CancellationToken cancellationToken = default) { var finalizations = await ListRecoverableFinalizationsAsync(cancellationToken); if (finalizations.Count == 0) { return new RecoveryActionResultDto { RequestedCount = 0, SuccessCount = 0, FailedCount = 0, Messages = ["No MP4 finalization tasks currently require recovery."] }; } var successCount = 0; var messages = new List(); foreach (var item in finalizations) { var started = await _ffmpegService.StartManualFinalizeTaskAsync(item.RecordTaskId, cancellationToken); if (started) { successCount++; } messages.Add( started ? $"MP4 finalization resumed for task {item.RecordTaskId}." : $"MP4 finalization could not be resumed for task {item.RecordTaskId}."); } return new RecoveryActionResultDto { RequestedCount = finalizations.Count, SuccessCount = successCount, FailedCount = finalizations.Count - successCount, Messages = messages }; } private async Task> ListRecoverableLiveRoomsAsync(CancellationToken cancellationToken) { var activeLiveRoomIds = await _dbContext.RecordSessions .AsNoTracking() .Where(item => item.Status == RecordSessionStatus.Starting || item.Status == RecordSessionStatus.Running || item.Status == RecordSessionStatus.Stopping) .Select(item => item.LiveRoomId) .Distinct() .ToListAsync(cancellationToken); var rooms = await _dbContext.LiveRooms .AsNoTracking() .Where(item => item.IsEnabled && item.AvailabilityStatus == LiveRoomAvailabilityStatus.Live && item.LastAutoStartDecisionCode != AutoStartDecisionCodes.Started) .ToListAsync(cancellationToken); return rooms .Where(item => !activeLiveRoomIds.Contains(item.Id)) .OrderByDescending(item => item.LastAutoStartDecisionAt ?? item.LastCheckedAt ?? item.UpdatedAt) .Select(item => new RecoverableLiveRoomDto { LiveRoomId = item.Id, PlatformName = item.Platform.ToString(), RoomId = item.RoomId, Title = item.Title, AnchorName = item.AnchorName, LastAutoStartDecisionCode = item.LastAutoStartDecisionCode, LastAutoStartDecisionSummary = item.LastAutoStartDecisionSummary, LastAutoStartDecisionDetail = item.LastAutoStartDecisionDetail, LastAutoStartDecisionAt = item.LastAutoStartDecisionAt, LastCheckedAt = item.LastCheckedAt }) .ToList(); } private async Task> ListRecoverableFinalizationsAsync(CancellationToken cancellationToken) { var tasks = await _dbContext.RecordTasks .AsNoTracking() .Include(item => item.LiveRoom) .Include(item => item.RecordSession) .Include(item => item.Result) .Where(item => item.OutputFormat == RecordOutputFormat.Mp4 && item.RecordSession != null) .ToListAsync(cancellationToken); return tasks .Where(IsRecoverableFinalization) .OrderByDescending(item => item.UpdatedAt) .Select(item => new RecoverableFinalizationDto { RecordTaskId = item.Id, RecordSessionId = item.RecordSessionId, LiveRoomId = item.LiveRoomId, LiveRoomTitle = item.LiveRoom?.Title ?? item.LiveRoom?.AnchorName ?? item.LiveRoom?.RoomId ?? item.LiveRoomId.ToString(), RoomId = item.LiveRoom?.RoomId ?? "-", PlatformName = item.LiveRoom?.Platform.ToString() ?? "Unknown", SegmentIndex = item.SegmentIndex, Status = item.Status, OutputFilePath = item.OutputFilePath, Reason = BuildFinalizationReason(item), CreatedAt = item.CreatedAt, EndedAt = item.EndedAt }) .ToList(); } private static bool IsRecoverableFinalization(RecordTask recordTask) { if (recordTask.RecordSession is null || recordTask.OutputFormat != RecordOutputFormat.Mp4) { return false; } if (recordTask.Status == RecordTaskStatus.Processing) { return true; } if (recordTask.Status is RecordTaskStatus.Starting or RecordTaskStatus.Running or RecordTaskStatus.Stopping) { return false; } if (recordTask.RecordSession.Status is RecordSessionStatus.Starting or RecordSessionStatus.Running or RecordSessionStatus.Stopping) { return false; } if (string.IsNullOrWhiteSpace(recordTask.OutputFilePath)) { return false; } var recorderOutputPath = ResolveManualFinalizeSourcePath(recordTask, recordTask.RecordSession); if (!File.Exists(recorderOutputPath)) { return false; } if (recordTask.Status == RecordTaskStatus.Completed) { var finalOutputPath = NormalizeAbsolutePath(recordTask.OutputFilePath); return !HasUsableOutput(finalOutputPath, CalculateFileSize(finalOutputPath)); } return true; } private static string BuildFinalizationReason(RecordTask recordTask) { if (recordTask.Status == RecordTaskStatus.Processing) { return string.IsNullOrWhiteSpace(recordTask.ErrorMessage) ? "MP4 finalization is queued or paused and can be resumed." : recordTask.ErrorMessage!; } if (recordTask.Status == RecordTaskStatus.Completed) { return "The final MP4 output is missing, but the intermediate recording file is still available."; } return "Manual MP4 finalization can be retried from the intermediate recording file."; } private static string ResolveManualFinalizeSourcePath(RecordTask recordTask, RecordSession recordSession) { var resultPath = recordTask.Result?.FilePath; if (!string.IsNullOrWhiteSpace(resultPath) && resultPath.EndsWith(".ts", StringComparison.OrdinalIgnoreCase)) { var normalizedResultPath = NormalizeAbsolutePath(resultPath); if (File.Exists(normalizedResultPath)) { return normalizedResultPath; } } return NormalizeAbsolutePath( GetRecorderOutputPath( recordTask.OutputFilePath ?? resultPath ?? string.Empty, recordSession.OutputFormat, recordSession.SaveMode)); } private static string GetRecorderOutputPath( string finalOutputPath, RecordOutputFormat outputFormat, RecordSaveMode saveMode) { if (outputFormat != RecordOutputFormat.Mp4) { return finalOutputPath; } if (saveMode == RecordSaveMode.SingleFile) { return Path.Combine( Path.GetDirectoryName(finalOutputPath)!, $"{Path.GetFileNameWithoutExtension(finalOutputPath)}.recording.ts"); } return Path.ChangeExtension(finalOutputPath, ".ts"); } private static long? CalculateFileSize(string? outputPath) { if (string.IsNullOrWhiteSpace(outputPath)) { return null; } if (File.Exists(outputPath)) { return new FileInfo(outputPath).Length; } if (Directory.Exists(outputPath)) { return new DirectoryInfo(outputPath) .EnumerateFiles("*", SearchOption.TopDirectoryOnly) .Sum(static file => file.Length); } return null; } private static bool HasUsableOutput(string? outputPath, long? fileSize) { if (string.IsNullOrWhiteSpace(outputPath)) { return false; } if (File.Exists(outputPath)) { return fileSize.GetValueOrDefault() > 0; } if (Directory.Exists(outputPath)) { return Directory.EnumerateFiles(outputPath, "*", SearchOption.TopDirectoryOnly).Any(); } return false; } private static string NormalizeAbsolutePath(string path) => Path.IsPathRooted(path) ? path : Path.GetFullPath(path, AppContext.BaseDirectory); private static RecoveryActionResultDto FailureResult(string message) => new() { RequestedCount = 1, SuccessCount = 0, FailedCount = 1, Messages = [message] }; }