Files
live_recorder/src/LiveRecorder.Infrastructure/Services/RecoveryService.cs
T
nanxunandClaude Opus 4.8 noreply@anthropic.com 24e5cf2a06 feat: add tiered storage guard (Green/Yellow/Red) and dashboard queue monitor
Storage Tier System:
- Add StorageTier enum (Green >30% / Yellow 10-30% / Red <10%)
- Extend StorageGuardResult with Tier, CanStartNewRecording, ShouldPauseActive, UsagePercent
- Yellow tier: deny new recordings but allow existing to finish and upload
- Red tier: deny new recordings and pause active sessions
- Auto-recovery: when disk frees up, polling automatically resumes new recordings
- Update LiveRoomPollingBackgroundService to use tier-based checks
- Expose tier + usage percent in Recovery API

Dashboard Queue Monitor:
- Add pending transcode count, pending upload count, queued data volume to dashboard
- Add storage tier badge (Green/Yellow/Red) with usage percentage
- Add queue monitoring card to dashboard view

Co-Authored-By: Claude Opus 4.8 noreply@anthropic.com
2026-06-04 23:13:26 +08:00

440 lines
16 KiB
C#

using LiveRecorder.Application.Abstractions.Recording;
using LiveRecorder.Application.Abstractions.Settings;
using LiveRecorder.Application.Abstractions.Storage;
using LiveRecorder.Application.Common;
using LiveRecorder.Application.Models.Recovery;
using LiveRecorder.Application.Models.RecordTasks;
using LiveRecorder.Application.Services;
using LiveRecorder.Domain.Entities;
using LiveRecorder.Domain.Enums;
using LiveRecorder.Infrastructure.Persistence;
using Microsoft.EntityFrameworkCore;
namespace LiveRecorder.Infrastructure.Services;
public sealed class RecoveryService
{
private readonly LiveRecorderDbContext _dbContext;
private readonly ISystemSettingsService _systemSettingsService;
private readonly IStorageGuardService _storageGuardService;
private readonly IFfmpegService _ffmpegService;
private readonly RecordService _recordService;
public RecoveryService(
LiveRecorderDbContext dbContext,
ISystemSettingsService systemSettingsService,
IStorageGuardService storageGuardService,
IFfmpegService ffmpegService,
RecordService recordService)
{
_dbContext = dbContext;
_systemSettingsService = systemSettingsService;
_storageGuardService = storageGuardService;
_ffmpegService = ffmpegService;
_recordService = recordService;
}
public async Task<RecoveryOverviewDto> GetOverviewAsync(CancellationToken cancellationToken = default)
{
var settings = await _systemSettingsService.GetAsync(cancellationToken);
var storage = _storageGuardService.CheckCanStartOrResume(settings);
var liveRooms = await ListRecoverableLiveRoomsAsync(cancellationToken);
var finalizations = await ListRecoverableFinalizationsAsync(cancellationToken);
return new RecoveryOverviewDto
{
Storage = new StorageGuardStatusDto
{
IsEnabled = storage.IsEnabled,
HasEnoughSpace = storage.HasEnoughSpace,
CheckedPath = storage.CheckedPath,
AvailableBytes = storage.AvailableBytes,
RequiredBytes = storage.RequiredBytes,
Message = storage.Message,
Tier = storage.Tier.ToString(),
UsagePercent = storage.UsagePercent
},
LiveRooms = liveRooms,
Finalizations = finalizations
};
}
public async Task<RecoveryActionResultDto> RetryLiveRoomAsync(Guid liveRoomId, CancellationToken cancellationToken = default)
{
var room = await _dbContext.LiveRooms.FirstOrDefaultAsync(item => item.Id == liveRoomId, cancellationToken);
if (room is null)
{
return FailureResult("Live room was not found.");
}
if (!room.IsEnabled)
{
return FailureResult("Live room is disabled and cannot be retried.");
}
if (room.AvailabilityStatus != LiveRoomAvailabilityStatus.Live)
{
return FailureResult("Live room is not currently online.");
}
var hasActiveSession = await _dbContext.RecordSessions.AnyAsync(
item => item.LiveRoomId == room.Id &&
(item.Status == RecordSessionStatus.Starting ||
item.Status == RecordSessionStatus.Running ||
item.Status == RecordSessionStatus.Stopping),
cancellationToken);
if (hasActiveSession)
{
room.SetLastAutoStartDecision(
AutoStartDecisionCodes.SkippedActiveSession,
"Auto-start skipped because an active recording session already exists.",
null,
DateTimeOffset.UtcNow);
await _dbContext.SaveChangesAsync(cancellationToken);
return FailureResult("An active recording session already exists for this live room.");
}
try
{
var task = await _recordService.StartAsync(
new StartRecordTaskRequest
{
LiveRoomId = liveRoomId
},
trackAutoStartDecision: true,
cancellationToken);
var started = task.Status is RecordTaskStatus.Starting or RecordTaskStatus.Running;
return new RecoveryActionResultDto
{
RequestedCount = 1,
SuccessCount = started ? 1 : 0,
FailedCount = started ? 0 : 1,
Messages =
[
started
? $"Recording retry started for room {room.RoomId}."
: $"Recording retry did not start for room {room.RoomId}. Status={task.Status}; Error={task.ErrorMessage ?? "n/a"}"
]
};
}
catch (Exception ex)
{
return FailureResult($"Recording retry failed for room {room.RoomId}: {ex.Message}");
}
}
public async Task<RecoveryActionResultDto> RetryAllLiveRoomsAsync(CancellationToken cancellationToken = default)
{
var liveRooms = await ListRecoverableLiveRoomsAsync(cancellationToken);
if (liveRooms.Count == 0)
{
return new RecoveryActionResultDto
{
RequestedCount = 0,
SuccessCount = 0,
FailedCount = 0,
Messages = ["No live rooms currently require retry."]
};
}
var messages = new List<string>();
var successCount = 0;
foreach (var item in liveRooms)
{
var result = await RetryLiveRoomAsync(item.LiveRoomId, cancellationToken);
successCount += result.SuccessCount;
messages.AddRange(result.Messages);
}
return new RecoveryActionResultDto
{
RequestedCount = liveRooms.Count,
SuccessCount = successCount,
FailedCount = liveRooms.Count - successCount,
Messages = messages
};
}
public async Task<RecoveryActionResultDto> ResumeFinalizationAsync(Guid recordTaskId, CancellationToken cancellationToken = default)
{
var started = await _ffmpegService.StartManualFinalizeTaskAsync(recordTaskId, cancellationToken);
return new RecoveryActionResultDto
{
RequestedCount = 1,
SuccessCount = started ? 1 : 0,
FailedCount = started ? 0 : 1,
Messages =
[
started
? $"MP4 finalization resumed for task {recordTaskId}."
: $"MP4 finalization could not be resumed for task {recordTaskId}."
]
};
}
public async Task<RecoveryActionResultDto> ResumeAllFinalizationsAsync(CancellationToken cancellationToken = default)
{
var finalizations = await ListRecoverableFinalizationsAsync(cancellationToken);
if (finalizations.Count == 0)
{
return new RecoveryActionResultDto
{
RequestedCount = 0,
SuccessCount = 0,
FailedCount = 0,
Messages = ["No MP4 finalization tasks currently require recovery."]
};
}
var successCount = 0;
var messages = new List<string>();
foreach (var item in finalizations)
{
var started = await _ffmpegService.StartManualFinalizeTaskAsync(item.RecordTaskId, cancellationToken);
if (started)
{
successCount++;
}
messages.Add(
started
? $"MP4 finalization resumed for task {item.RecordTaskId}."
: $"MP4 finalization could not be resumed for task {item.RecordTaskId}.");
}
return new RecoveryActionResultDto
{
RequestedCount = finalizations.Count,
SuccessCount = successCount,
FailedCount = finalizations.Count - successCount,
Messages = messages
};
}
private async Task<IReadOnlyList<RecoverableLiveRoomDto>> ListRecoverableLiveRoomsAsync(CancellationToken cancellationToken)
{
var activeLiveRoomIds = await _dbContext.RecordSessions
.AsNoTracking()
.Where(item => item.Status == RecordSessionStatus.Starting ||
item.Status == RecordSessionStatus.Running ||
item.Status == RecordSessionStatus.Stopping)
.Select(item => item.LiveRoomId)
.Distinct()
.ToListAsync(cancellationToken);
var rooms = await _dbContext.LiveRooms
.AsNoTracking()
.Where(item => item.IsEnabled &&
item.AvailabilityStatus == LiveRoomAvailabilityStatus.Live &&
item.LastAutoStartDecisionCode != AutoStartDecisionCodes.Started)
.ToListAsync(cancellationToken);
return rooms
.Where(item => !activeLiveRoomIds.Contains(item.Id))
.OrderByDescending(item => item.LastAutoStartDecisionAt ?? item.LastCheckedAt ?? item.UpdatedAt)
.Select(item => new RecoverableLiveRoomDto
{
LiveRoomId = item.Id,
PlatformName = item.Platform.ToString(),
RoomId = item.RoomId,
Title = item.Title,
AnchorName = item.AnchorName,
LastAutoStartDecisionCode = item.LastAutoStartDecisionCode,
LastAutoStartDecisionSummary = item.LastAutoStartDecisionSummary,
LastAutoStartDecisionDetail = item.LastAutoStartDecisionDetail,
LastAutoStartDecisionAt = item.LastAutoStartDecisionAt,
LastCheckedAt = item.LastCheckedAt
})
.ToList();
}
private async Task<IReadOnlyList<RecoverableFinalizationDto>> ListRecoverableFinalizationsAsync(CancellationToken cancellationToken)
{
var tasks = await _dbContext.RecordTasks
.AsNoTracking()
.Include(item => item.LiveRoom)
.Include(item => item.RecordSession)
.Include(item => item.Result)
.Where(item => item.OutputFormat == RecordOutputFormat.Mp4 &&
item.RecordSession != null)
.ToListAsync(cancellationToken);
return tasks
.Where(IsRecoverableFinalization)
.OrderByDescending(item => item.UpdatedAt)
.Select(item => new RecoverableFinalizationDto
{
RecordTaskId = item.Id,
RecordSessionId = item.RecordSessionId,
LiveRoomId = item.LiveRoomId,
LiveRoomTitle = item.LiveRoom?.Title ?? item.LiveRoom?.AnchorName ?? item.LiveRoom?.RoomId ?? item.LiveRoomId.ToString(),
RoomId = item.LiveRoom?.RoomId ?? "-",
PlatformName = item.LiveRoom?.Platform.ToString() ?? "Unknown",
SegmentIndex = item.SegmentIndex,
Status = item.Status,
OutputFilePath = item.OutputFilePath,
Reason = BuildFinalizationReason(item),
CreatedAt = item.CreatedAt,
EndedAt = item.EndedAt
})
.ToList();
}
private static bool IsRecoverableFinalization(RecordTask recordTask)
{
if (recordTask.RecordSession is null ||
recordTask.OutputFormat != RecordOutputFormat.Mp4)
{
return false;
}
if (recordTask.Status == RecordTaskStatus.Processing)
{
return true;
}
if (recordTask.Status is RecordTaskStatus.Starting or RecordTaskStatus.Running or RecordTaskStatus.Stopping)
{
return false;
}
if (recordTask.RecordSession.Status is RecordSessionStatus.Starting or RecordSessionStatus.Running or RecordSessionStatus.Stopping)
{
return false;
}
if (string.IsNullOrWhiteSpace(recordTask.OutputFilePath))
{
return false;
}
var recorderOutputPath = ResolveManualFinalizeSourcePath(recordTask, recordTask.RecordSession);
if (!File.Exists(recorderOutputPath))
{
return false;
}
if (recordTask.Status == RecordTaskStatus.Completed)
{
var finalOutputPath = NormalizeAbsolutePath(recordTask.OutputFilePath);
return !HasUsableOutput(finalOutputPath, CalculateFileSize(finalOutputPath));
}
return true;
}
private static string BuildFinalizationReason(RecordTask recordTask)
{
if (recordTask.Status == RecordTaskStatus.Processing)
{
return string.IsNullOrWhiteSpace(recordTask.ErrorMessage)
? "MP4 finalization is queued or paused and can be resumed."
: recordTask.ErrorMessage!;
}
if (recordTask.Status == RecordTaskStatus.Completed)
{
return "The final MP4 output is missing, but the intermediate recording file is still available.";
}
return "Manual MP4 finalization can be retried from the intermediate recording file.";
}
private static string ResolveManualFinalizeSourcePath(RecordTask recordTask, RecordSession recordSession)
{
var resultPath = recordTask.Result?.FilePath;
if (!string.IsNullOrWhiteSpace(resultPath) &&
resultPath.EndsWith(".ts", StringComparison.OrdinalIgnoreCase))
{
var normalizedResultPath = NormalizeAbsolutePath(resultPath);
if (File.Exists(normalizedResultPath))
{
return normalizedResultPath;
}
}
return NormalizeAbsolutePath(
GetRecorderOutputPath(
recordTask.OutputFilePath ?? resultPath ?? string.Empty,
recordSession.OutputFormat,
recordSession.SaveMode));
}
private static string GetRecorderOutputPath(
string finalOutputPath,
RecordOutputFormat outputFormat,
RecordSaveMode saveMode)
{
if (outputFormat != RecordOutputFormat.Mp4)
{
return finalOutputPath;
}
if (saveMode == RecordSaveMode.SingleFile)
{
return Path.Combine(
Path.GetDirectoryName(finalOutputPath)!,
$"{Path.GetFileNameWithoutExtension(finalOutputPath)}.recording.ts");
}
return Path.ChangeExtension(finalOutputPath, ".ts");
}
private static long? CalculateFileSize(string? outputPath)
{
if (string.IsNullOrWhiteSpace(outputPath))
{
return null;
}
if (File.Exists(outputPath))
{
return new FileInfo(outputPath).Length;
}
if (Directory.Exists(outputPath))
{
return new DirectoryInfo(outputPath)
.EnumerateFiles("*", SearchOption.TopDirectoryOnly)
.Sum(static file => file.Length);
}
return null;
}
private static bool HasUsableOutput(string? outputPath, long? fileSize)
{
if (string.IsNullOrWhiteSpace(outputPath))
{
return false;
}
if (File.Exists(outputPath))
{
return fileSize.GetValueOrDefault() > 0;
}
if (Directory.Exists(outputPath))
{
return Directory.EnumerateFiles(outputPath, "*", SearchOption.TopDirectoryOnly).Any();
}
return false;
}
private static string NormalizeAbsolutePath(string path) =>
Path.IsPathRooted(path)
? path
: Path.GetFullPath(path, AppContext.BaseDirectory);
private static RecoveryActionResultDto FailureResult(string message) => new()
{
RequestedCount = 1,
SuccessCount = 0,
FailedCount = 1,
Messages = [message]
};
}