using LiveRecorder.Application.Abstractions.Recording; using LiveRecorder.Application.Abstractions.Logging; using LiveRecorder.Application.Abstractions.Settings; using LiveRecorder.Application.Abstractions.Storage; using LiveRecorder.Application.Common; using LiveRecorder.Application.Models.Recovery; using LiveRecorder.Application.Models.RecordTasks; using LiveRecorder.Application.Services; using LiveRecorder.Domain.Entities; using LiveRecorder.Domain.Enums; using LiveRecorder.Infrastructure.Persistence; using Microsoft.EntityFrameworkCore; namespace LiveRecorder.Infrastructure.Services; public sealed class RecoveryService { private readonly LiveRecorderDbContext _dbContext; private readonly ISystemSettingsService _systemSettingsService; private readonly IStorageGuardService _storageGuardService; private readonly IFfmpegService _ffmpegService; private readonly RecordService _recordService; private readonly IVideoMetadataService _videoMetadataService; private readonly ISystemLogService _systemLogService; public RecoveryService( LiveRecorderDbContext dbContext, ISystemSettingsService systemSettingsService, IStorageGuardService storageGuardService, IFfmpegService ffmpegService, RecordService recordService, IVideoMetadataService videoMetadataService, ISystemLogService systemLogService) { _dbContext = dbContext; _systemSettingsService = systemSettingsService; _storageGuardService = storageGuardService; _ffmpegService = ffmpegService; _recordService = recordService; _videoMetadataService = videoMetadataService; _systemLogService = systemLogService; } public async Task GetOverviewAsync(CancellationToken cancellationToken = default) { var settings = await _systemSettingsService.GetAsync(cancellationToken); var storage = _storageGuardService.CheckCanStartOrResume(settings); var liveRooms = await ListRecoverableLiveRoomsAsync(cancellationToken); var finalizations = await ListRecoverableFinalizationsAsync(cancellationToken); var mergedArtifacts = await ListMergedArtifactsAsync(cancellationToken); return new RecoveryOverviewDto { Storage = new StorageGuardStatusDto { IsEnabled = storage.IsEnabled, IsAvailable = storage.IsAvailable, HasEnoughSpace = storage.HasEnoughSpace, CheckedPath = storage.CheckedPath, TotalBytes = storage.TotalBytes, UsedBytes = storage.UsedBytes, AvailableBytes = storage.AvailableBytes, RequiredBytes = storage.RequiredBytes, Message = storage.Message, Tier = storage.Tier.ToString(), UsagePercent = storage.UsagePercent, FreePercent = storage.FreePercent, GreenThresholdPercent = storage.GreenThresholdPercent, RedThresholdPercent = storage.RedThresholdPercent }, LiveRooms = liveRooms, Finalizations = finalizations, MergedArtifacts = mergedArtifacts }; } private async Task> ListMergedArtifactsAsync(CancellationToken cancellationToken) { var sources = await _dbContext.RecordTasks .AsNoTracking() .Include(item => item.Result) .Where(item => item.IsHiddenArtifactSource) .OrderByDescending(item => item.UpdatedAt) .Take(100) .ToListAsync(cancellationToken); var targetIds = sources.Where(item => item.MergedIntoRecordTaskId.HasValue) .Select(item => item.MergedIntoRecordTaskId!.Value) .Distinct() .ToArray(); var targets = await _dbContext.RecordTasks .AsNoTracking() .Include(item => item.Result) .Where(item => targetIds.Contains(item.Id)) .ToDictionaryAsync(item => item.Id, cancellationToken); return sources.Select(source => { var sourcePath = source.Result?.FilePath ?? source.OutputFilePath; var parent = string.IsNullOrWhiteSpace(sourcePath) ? null : Path.GetDirectoryName(Path.GetFullPath(sourcePath)); var recoveryBase = parent is null ? null : Path.Combine(parent, ".liverecorder-recovery", source.RecordSessionId.ToString("N")); var manifest = FindRecoveryManifest(recoveryBase, source.Id); return new MergedArtifactRecordDto { SourceRecordTaskId = source.Id, RecordSessionId = source.RecordSessionId, MergedIntoRecordTaskId = source.MergedIntoRecordTaskId, SourceVideoPath = sourcePath, MergedVideoPath = source.MergedIntoRecordTaskId.HasValue && targets.TryGetValue(source.MergedIntoRecordTaskId.Value, out var target) ? target.Result?.FilePath : null, RecoveryDirectory = manifest is null ? recoveryBase : Path.GetDirectoryName(manifest), ManifestPath = manifest, SourceDurationSeconds = source.Result?.DurationSeconds ?? source.DurationSeconds, CreatedAt = source.UpdatedAt }; }).ToList(); } private static string? FindRecoveryManifest(string? recoveryBase, Guid recordTaskId) { if (string.IsNullOrWhiteSpace(recoveryBase) || !Directory.Exists(recoveryBase)) return null; try { foreach (var path in Directory.EnumerateFiles(recoveryBase, "manifest.json", SearchOption.AllDirectories).Take(100)) { if (File.ReadAllText(path).Contains(recordTaskId.ToString(), StringComparison.OrdinalIgnoreCase)) return path; } } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { } return null; } public async Task RetryLiveRoomAsync(Guid liveRoomId, CancellationToken cancellationToken = default) { var room = await _dbContext.LiveRooms.FirstOrDefaultAsync(item => item.Id == liveRoomId, cancellationToken); if (room is null) { return FailureResult("未找到该直播间。"); } if (!room.IsEnabled) { return FailureResult("该直播间已禁用,无法重试开录。"); } if (room.AvailabilityStatus != LiveRoomAvailabilityStatus.Live) { return FailureResult("该直播间当前未开播,无法重试开录。"); } var hasActiveSession = await _dbContext.RecordSessions.AnyAsync( item => item.LiveRoomId == room.Id && (item.Status == RecordSessionStatus.Starting || item.Status == RecordSessionStatus.Running || item.Status == RecordSessionStatus.Stopping), cancellationToken); if (hasActiveSession) { room.SetLastAutoStartDecision( AutoStartDecisionCodes.SkippedActiveSession, "已有活动录制会话,本次自动开录已跳过。", null, DateTimeOffset.UtcNow); await _dbContext.SaveChangesAsync(cancellationToken); return FailureResult("该直播间已经存在活动录制会话。"); } try { var task = await _recordService.StartAsync( new StartRecordTaskRequest { LiveRoomId = liveRoomId }, trackAutoStartDecision: true, cancellationToken); var started = task.Status is RecordTaskStatus.Starting or RecordTaskStatus.Running; return new RecoveryActionResultDto { RequestedCount = 1, SuccessCount = started ? 1 : 0, FailedCount = started ? 0 : 1, Messages = [ started ? $"直播间 {room.RoomId} 已开始重试录制。" : $"直播间 {room.RoomId} 未能开始录制。状态={task.Status};错误={task.ErrorMessage ?? "无"}" ] }; } catch (Exception ex) { return FailureResult($"直播间 {room.RoomId} 重试录制失败:{ex.Message}"); } } public async Task RetryAllLiveRoomsAsync(CancellationToken cancellationToken = default) { var liveRooms = await ListRecoverableLiveRoomsAsync(cancellationToken); if (liveRooms.Count == 0) { return new RecoveryActionResultDto { RequestedCount = 0, SuccessCount = 0, FailedCount = 0, Messages = ["当前没有需要重试开录的直播间。"] }; } var messages = new List(); var successCount = 0; foreach (var item in liveRooms) { var result = await RetryLiveRoomAsync(item.LiveRoomId, cancellationToken); successCount += result.SuccessCount; messages.AddRange(result.Messages); } return new RecoveryActionResultDto { RequestedCount = liveRooms.Count, SuccessCount = successCount, FailedCount = liveRooms.Count - successCount, Messages = messages }; } public async Task ResumeFinalizationAsync(Guid recordTaskId, CancellationToken cancellationToken = default) { var recoveredOrphan = await _ffmpegService.TryRecoverOrphanedTerminalTaskAsync(recordTaskId, cancellationToken); var started = recoveredOrphan || await _ffmpegService.StartManualFinalizeTaskAsync(recordTaskId, cancellationToken); return new RecoveryActionResultDto { RequestedCount = 1, SuccessCount = started ? 1 : 0, FailedCount = started ? 0 : 1, Messages = [ started ? recoveredOrphan ? $"任务 {recordTaskId} 的遗留分片已恢复,并已进入后续上传流程。" : $"任务 {recordTaskId} 已恢复 MP4 转码。" : $"任务 {recordTaskId} 无法恢复 MP4 转码。" ] }; } public async Task ResumeAllFinalizationsAsync(CancellationToken cancellationToken = default) { var finalizations = await ListRecoverableFinalizationsAsync(cancellationToken); if (finalizations.Count == 0) { return new RecoveryActionResultDto { RequestedCount = 0, SuccessCount = 0, FailedCount = 0, Messages = ["当前没有需要恢复的 MP4 转码任务。"] }; } var successCount = 0; var messages = new List(); foreach (var item in finalizations) { var recoveredOrphan = await _ffmpegService.TryRecoverOrphanedTerminalTaskAsync(item.RecordTaskId, cancellationToken); var started = recoveredOrphan || await _ffmpegService.StartManualFinalizeTaskAsync(item.RecordTaskId, cancellationToken); if (started) { successCount++; } messages.Add( started ? recoveredOrphan ? $"任务 {item.RecordTaskId} 的遗留分片已恢复。" : $"任务 {item.RecordTaskId} 已恢复 MP4 转码。" : $"任务 {item.RecordTaskId} 无法恢复 MP4 转码。"); } return new RecoveryActionResultDto { RequestedCount = finalizations.Count, SuccessCount = successCount, FailedCount = finalizations.Count - successCount, Messages = messages }; } public async Task ListRecordingFailuresAsync( string? failureKind, int skip, int take, CancellationToken cancellationToken = default) { var tasks = await _dbContext.RecordTasks .AsNoTracking() .Include(static item => item.LiveRoom) .Include(static item => item.Result) .Where(item => item.Status == RecordTaskStatus.Failed || item.Status == RecordTaskStatus.Processing && item.ErrorMessage != null && item.ErrorMessage.StartsWith("[artifact-repair]")) .OrderByDescending(static item => item.UpdatedAt) .ToListAsync(cancellationToken); var items = tasks.Select(MapRecordingFailure).AsEnumerable(); if (!string.IsNullOrWhiteSpace(failureKind)) { items = items.Where(item => item.FailureKind.Equals(failureKind.Trim(), StringComparison.OrdinalIgnoreCase)); } var materialized = items.ToList(); return new RecordingFailureListResponse { TotalCount = materialized.Count, Items = materialized .Skip(Math.Max(0, skip)) .Take(Math.Clamp(take, 1, 100)) .ToList() }; } public async Task AcceptRecordingArtifactAsync( Guid recordTaskId, bool confirmShortArtifact, CancellationToken cancellationToken = default) { var task = await _dbContext.RecordTasks .Include(static item => item.RecordSession) .ThenInclude(static item => item!.RecordTasks) .Include(static item => item.Result) .FirstOrDefaultAsync(item => item.Id == recordTaskId, cancellationToken); if (task?.Result is null || task.Status != RecordTaskStatus.Failed) { return FailureResult("该任务不是可认领的录制失败产物。"); } var path = NormalizeAbsolutePath(task.Result.FilePath); if (!File.Exists(path) || new FileInfo(path).Length <= 0) { return FailureResult("本地媒体文件不存在或为空,无法认领。"); } var metadata = await _videoMetadataService.ExtractMetadataAsync(path, cancellationToken); if (metadata is null || string.IsNullOrWhiteSpace(metadata.VideoCodec) || metadata.DurationSeconds is not > 0) { return FailureResult("媒体仍无法读取,请先使用非破坏修复。"); } if (metadata.DurationSeconds < 5 && !confirmShortArtifact) { return FailureResult($"媒体只有 {metadata.DurationSeconds:0.###} 秒,需要确认短分片后才能认领。"); } var originalError = task.ErrorMessage; var now = DateTimeOffset.UtcNow; task.MarkCompleted(task.EndedAt ?? now, metadata.DurationSeconds); task.Result.Update( path, new FileInfo(path).Length, metadata.DurationSeconds, task.Result.DanmakuFilePath, task.Result.DanmakuMessageCount, RecordTaskStatus.Completed, null); task.Result.ResetUploadForRecoveredArtifact(); if (task.RecordSession is not null && task.RecordSession.RecordTasks.All(static item => item.Status is RecordTaskStatus.Completed or RecordTaskStatus.Stopped)) { task.RecordSession.MarkCompleted(now); } await _dbContext.SaveChangesAsync(cancellationToken); await _systemLogService.WriteAsync( SystemLogLevel.Info, "Recovery", "录制失败产物已由管理员确认有效,等待手动上传。", $"path={path}; duration={metadata.DurationSeconds:0.###}; originalError={originalError}", task.LiveRoomId, task.RecordSessionId, task.Id, cancellationToken); return new RecoveryActionResultDto { RequestedCount = 1, SuccessCount = 1, FailedCount = 0, Messages = ["分片已认领为有效产物,可前往上传任务页面手动上传。"] }; } public async Task RepairRecordingArtifactAsync( Guid recordTaskId, CancellationToken cancellationToken = default) { var started = await _ffmpegService.StartArtifactRepairAsync(recordTaskId, cancellationToken); return new RecoveryActionResultDto { RequestedCount = 1, SuccessCount = started ? 1 : 0, FailedCount = started ? 0 : 1, Messages = [ started ? "非破坏修复已排队;原文件不会被覆盖或删除。" : "无法启动修复:任务可能正在处理、文件缺失或可用空间不足。" ] }; } private static RecordingFailureItemDto MapRecordingFailure(RecordTask task) { var result = task.Result; var path = result?.FilePath; var normalizedPath = string.IsNullOrWhiteSpace(path) ? null : NormalizeAbsolutePath(path); var fileExists = normalizedPath is not null && File.Exists(normalizedPath) && new FileInfo(normalizedPath).Length > 0; var classification = ClassifyRecordingFailure(task, fileExists); var isRepairing = task.Status == RecordTaskStatus.Processing && task.ErrorMessage?.StartsWith("[artifact-repair]", StringComparison.Ordinal) == true; return new RecordingFailureItemDto { RecordTaskId = task.Id, RecordSessionId = task.RecordSessionId, LiveRoomId = task.LiveRoomId, LiveRoomTitle = task.LiveRoom?.Title ?? task.LiveRoom?.AnchorName ?? task.LiveRoom?.RoomId ?? "未知直播间", RoomId = task.LiveRoom?.RoomId ?? "-", PlatformName = task.LiveRoom?.Platform.ToString() ?? "Unknown", SegmentIndex = task.SegmentIndex, FailureKind = classification.Kind, FailureLabel = classification.Label, RecommendedAction = classification.Action, ErrorMessage = task.ErrorMessage ?? result?.ErrorMessage, FilePath = normalizedPath, FileSizeBytes = result?.FileSizeBytes, DurationSeconds = result?.DurationSeconds ?? task.DurationSeconds, FileExists = fileExists, CanAccept = !isRepairing && fileExists && classification.Kind is "ReadableFragment" or "TooShort" or "Unknown", CanRepair = !isRepairing && fileExists && classification.Kind is "UnreadableMedia" or "FinalizationFailed" or "Unknown", CanRetryRoom = !isRepairing && task.LiveRoom?.AvailabilityStatus == LiveRoomAvailabilityStatus.Live, IsRepairing = isRepairing, CreatedAt = task.CreatedAt }; } private static (string Kind, string Label, string Action) ClassifyRecordingFailure(RecordTask task, bool fileExists) { var error = task.ErrorMessage ?? task.Result?.ErrorMessage ?? string.Empty; if (task.Status == RecordTaskStatus.Processing && error.StartsWith("[artifact-repair]", StringComparison.Ordinal)) { return ("Repairing", "正在修复", "等待修复完成,应用重启后会自动续排。"); } if (!fileExists) { return ("MissingMedia", "文件缺失", "历史媒体文件不存在,无法恢复;若直播仍在线可重新开录。"); } if (error.Contains("offline", StringComparison.OrdinalIgnoreCase) || error.Contains("已离线", StringComparison.OrdinalIgnoreCase)) { return ("SourceOffline", "直播已离线", "历史时段无法补录;直播重新在线后可重新开录。"); } if (error.Contains("播放地址", StringComparison.OrdinalIgnoreCase) || error.Contains("stream url", StringComparison.OrdinalIgnoreCase)) { return ("StreamUrlUnavailable", "未取得播放地址", "等待下一次巡检刷新播放地址,在线时可手动重试开录。"); } if (error.Contains("short fragment", StringComparison.OrdinalIgnoreCase) || task.Result?.DurationSeconds is > 0 and < 5) { return ("TooShort", "短分片", "确认内容有价值后可人工认领;不足 5 秒需要二次确认。"); } if (error.Contains("ffprobe", StringComparison.OrdinalIgnoreCase) || error.Contains("does not contain a video", StringComparison.OrdinalIgnoreCase)) { return ("UnreadableMedia", "媒体不可读", "尝试生成新的恢复文件;原文件保持不变。"); } if (error.Contains("finaliz", StringComparison.OrdinalIgnoreCase) || error.Contains("mux", StringComparison.OrdinalIgnoreCase)) { return ("FinalizationFailed", "封装或收尾失败", "先重新封装,失败后再转码修复。"); } if (task.Result?.DurationSeconds is > 0) { return ("ReadableFragment", "异常退出分片", "重新校验媒体;确认有效后转为待上传。"); } return ("Unknown", "待检测", "可先尝试校验认领;无法读取时再执行非破坏修复。"); } private async Task> ListRecoverableLiveRoomsAsync(CancellationToken cancellationToken) { var activeLiveRoomIds = await _dbContext.RecordSessions .AsNoTracking() .Where(item => item.Status == RecordSessionStatus.Starting || item.Status == RecordSessionStatus.Running || item.Status == RecordSessionStatus.Stopping) .Select(item => item.LiveRoomId) .Distinct() .ToListAsync(cancellationToken); var rooms = await _dbContext.LiveRooms .AsNoTracking() .Where(item => item.IsEnabled && item.AvailabilityStatus == LiveRoomAvailabilityStatus.Live && item.LastAutoStartDecisionCode != AutoStartDecisionCodes.Started) .ToListAsync(cancellationToken); return rooms .Where(item => !activeLiveRoomIds.Contains(item.Id)) .OrderByDescending(item => item.LastAutoStartDecisionAt ?? item.LastCheckedAt ?? item.UpdatedAt) .Select(item => new RecoverableLiveRoomDto { LiveRoomId = item.Id, PlatformName = item.Platform.ToString(), RoomId = item.RoomId, Title = item.Title, AnchorName = item.AnchorName, LastAutoStartDecisionCode = item.LastAutoStartDecisionCode, LastAutoStartDecisionSummary = item.LastAutoStartDecisionSummary, LastAutoStartDecisionDetail = item.LastAutoStartDecisionDetail, LastAutoStartDecisionAt = item.LastAutoStartDecisionAt, LastCheckedAt = item.LastCheckedAt }) .ToList(); } private async Task> ListRecoverableFinalizationsAsync(CancellationToken cancellationToken) { var tasks = await _dbContext.RecordTasks .AsNoTracking() .Include(item => item.LiveRoom) .Include(item => item.RecordSession) .Include(item => item.Result) .Where(item => item.OutputFormat == RecordOutputFormat.Mp4 && item.RecordSession != null) .ToListAsync(cancellationToken); return tasks .Where(IsRecoverableFinalization) .OrderByDescending(item => item.UpdatedAt) .Select(item => new RecoverableFinalizationDto { RecordTaskId = item.Id, RecordSessionId = item.RecordSessionId, LiveRoomId = item.LiveRoomId, LiveRoomTitle = item.LiveRoom?.Title ?? item.LiveRoom?.AnchorName ?? item.LiveRoom?.RoomId ?? item.LiveRoomId.ToString(), RoomId = item.LiveRoom?.RoomId ?? "-", PlatformName = item.LiveRoom?.Platform.ToString() ?? "Unknown", SegmentIndex = item.SegmentIndex, Status = item.Status, OutputFilePath = item.OutputFilePath, Reason = BuildFinalizationReason(item), CreatedAt = item.CreatedAt, EndedAt = item.EndedAt }) .ToList(); } private static bool IsRecoverableFinalization(RecordTask recordTask) { if (recordTask.RecordSession is null || recordTask.OutputFormat != RecordOutputFormat.Mp4) { return false; } if (recordTask.RecordSession.Status is RecordSessionStatus.Starting or RecordSessionStatus.Running or RecordSessionStatus.Stopping) { return false; } if (string.IsNullOrWhiteSpace(recordTask.OutputFilePath)) { return false; } var manifestSources = FfmpegService.ReadRecorderSegmentsManifest(recordTask.OutputFilePath); var recorderOutputPath = ResolveManualFinalizeSourcePath(recordTask, recordTask.RecordSession); if (manifestSources.Count == 0 && !File.Exists(recorderOutputPath)) { return false; } if (recordTask.Status == RecordTaskStatus.Completed) { var finalOutputPath = NormalizeAbsolutePath(recordTask.OutputFilePath); return !HasUsableOutput(finalOutputPath, CalculateFileSize(finalOutputPath)); } return true; } private static string BuildFinalizationReason(RecordTask recordTask) { if (recordTask.Status == RecordTaskStatus.Processing) { return string.IsNullOrWhiteSpace(recordTask.ErrorMessage) ? "MP4 转码正在排队或已暂停,可以继续恢复。" : recordTask.ErrorMessage!; } if (recordTask.Status is RecordTaskStatus.Starting or RecordTaskStatus.Running or RecordTaskStatus.Stopping) { return "父会话已经结束,但该分片仍显示为录制中;可安全对账并恢复转码、上传。"; } if (recordTask.Status == RecordTaskStatus.Completed) { return "最终 MP4 文件缺失,但中间录制文件仍然存在。"; } return "可以使用保留的中间录制文件重新执行 MP4 转码。"; } private static string ResolveManualFinalizeSourcePath(RecordTask recordTask, RecordSession recordSession) { var resultPath = recordTask.Result?.FilePath; if (!string.IsNullOrWhiteSpace(resultPath) && resultPath.EndsWith(".ts", StringComparison.OrdinalIgnoreCase)) { var normalizedResultPath = NormalizeAbsolutePath(resultPath); if (File.Exists(normalizedResultPath)) { return normalizedResultPath; } } return NormalizeAbsolutePath( GetRecorderOutputPath( recordTask.OutputFilePath ?? resultPath ?? string.Empty, recordSession.OutputFormat, recordSession.SaveMode)); } private static string GetRecorderOutputPath( string finalOutputPath, RecordOutputFormat outputFormat, RecordSaveMode saveMode) { if (outputFormat != RecordOutputFormat.Mp4) { return finalOutputPath; } if (saveMode == RecordSaveMode.SingleFile) { return Path.Combine( Path.GetDirectoryName(finalOutputPath)!, $"{Path.GetFileNameWithoutExtension(finalOutputPath)}.recording.ts"); } return Path.ChangeExtension(finalOutputPath, ".ts"); } private static long? CalculateFileSize(string? outputPath) { if (string.IsNullOrWhiteSpace(outputPath)) { return null; } if (File.Exists(outputPath)) { return new FileInfo(outputPath).Length; } if (Directory.Exists(outputPath)) { return new DirectoryInfo(outputPath) .EnumerateFiles("*", SearchOption.TopDirectoryOnly) .Sum(static file => file.Length); } return null; } private static bool HasUsableOutput(string? outputPath, long? fileSize) { if (string.IsNullOrWhiteSpace(outputPath)) { return false; } if (File.Exists(outputPath)) { return fileSize.GetValueOrDefault() > 0; } if (Directory.Exists(outputPath)) { return Directory.EnumerateFiles(outputPath, "*", SearchOption.TopDirectoryOnly).Any(); } return false; } private static string NormalizeAbsolutePath(string path) => Path.IsPathRooted(path) ? path : Path.GetFullPath(path, AppContext.BaseDirectory); private static RecoveryActionResultDto FailureResult(string message) => new() { RequestedCount = 1, SuccessCount = 0, FailedCount = 1, Messages = [message] }; }