Storage Tier System: - Add StorageTier enum (Green >30% / Yellow 10-30% / Red <10%) - Extend StorageGuardResult with Tier, CanStartNewRecording, ShouldPauseActive, UsagePercent - Yellow tier: deny new recordings but allow existing to finish and upload - Red tier: deny new recordings and pause active sessions - Auto-recovery: when disk frees up, polling automatically resumes new recordings - Update LiveRoomPollingBackgroundService to use tier-based checks - Expose tier + usage percent in Recovery API Dashboard Queue Monitor: - Add pending transcode count, pending upload count, queued data volume to dashboard - Add storage tier badge (Green/Yellow/Red) with usage percentage - Add queue monitoring card to dashboard view Co-Authored-By: Claude Opus 4.8 noreply@anthropic.com
440 lines
16 KiB
C#
440 lines
16 KiB
C#
using LiveRecorder.Application.Abstractions.Recording;
|
|
using LiveRecorder.Application.Abstractions.Settings;
|
|
using LiveRecorder.Application.Abstractions.Storage;
|
|
using LiveRecorder.Application.Common;
|
|
using LiveRecorder.Application.Models.Recovery;
|
|
using LiveRecorder.Application.Models.RecordTasks;
|
|
using LiveRecorder.Application.Services;
|
|
using LiveRecorder.Domain.Entities;
|
|
using LiveRecorder.Domain.Enums;
|
|
using LiveRecorder.Infrastructure.Persistence;
|
|
using Microsoft.EntityFrameworkCore;
|
|
|
|
namespace LiveRecorder.Infrastructure.Services;
|
|
|
|
public sealed class RecoveryService
|
|
{
|
|
private readonly LiveRecorderDbContext _dbContext;
|
|
private readonly ISystemSettingsService _systemSettingsService;
|
|
private readonly IStorageGuardService _storageGuardService;
|
|
private readonly IFfmpegService _ffmpegService;
|
|
private readonly RecordService _recordService;
|
|
|
|
public RecoveryService(
|
|
LiveRecorderDbContext dbContext,
|
|
ISystemSettingsService systemSettingsService,
|
|
IStorageGuardService storageGuardService,
|
|
IFfmpegService ffmpegService,
|
|
RecordService recordService)
|
|
{
|
|
_dbContext = dbContext;
|
|
_systemSettingsService = systemSettingsService;
|
|
_storageGuardService = storageGuardService;
|
|
_ffmpegService = ffmpegService;
|
|
_recordService = recordService;
|
|
}
|
|
|
|
public async Task<RecoveryOverviewDto> GetOverviewAsync(CancellationToken cancellationToken = default)
|
|
{
|
|
var settings = await _systemSettingsService.GetAsync(cancellationToken);
|
|
var storage = _storageGuardService.CheckCanStartOrResume(settings);
|
|
var liveRooms = await ListRecoverableLiveRoomsAsync(cancellationToken);
|
|
var finalizations = await ListRecoverableFinalizationsAsync(cancellationToken);
|
|
|
|
return new RecoveryOverviewDto
|
|
{
|
|
Storage = new StorageGuardStatusDto
|
|
{
|
|
IsEnabled = storage.IsEnabled,
|
|
HasEnoughSpace = storage.HasEnoughSpace,
|
|
CheckedPath = storage.CheckedPath,
|
|
AvailableBytes = storage.AvailableBytes,
|
|
RequiredBytes = storage.RequiredBytes,
|
|
Message = storage.Message,
|
|
Tier = storage.Tier.ToString(),
|
|
UsagePercent = storage.UsagePercent
|
|
},
|
|
LiveRooms = liveRooms,
|
|
Finalizations = finalizations
|
|
};
|
|
}
|
|
|
|
public async Task<RecoveryActionResultDto> RetryLiveRoomAsync(Guid liveRoomId, CancellationToken cancellationToken = default)
|
|
{
|
|
var room = await _dbContext.LiveRooms.FirstOrDefaultAsync(item => item.Id == liveRoomId, cancellationToken);
|
|
if (room is null)
|
|
{
|
|
return FailureResult("Live room was not found.");
|
|
}
|
|
|
|
if (!room.IsEnabled)
|
|
{
|
|
return FailureResult("Live room is disabled and cannot be retried.");
|
|
}
|
|
|
|
if (room.AvailabilityStatus != LiveRoomAvailabilityStatus.Live)
|
|
{
|
|
return FailureResult("Live room is not currently online.");
|
|
}
|
|
|
|
var hasActiveSession = await _dbContext.RecordSessions.AnyAsync(
|
|
item => item.LiveRoomId == room.Id &&
|
|
(item.Status == RecordSessionStatus.Starting ||
|
|
item.Status == RecordSessionStatus.Running ||
|
|
item.Status == RecordSessionStatus.Stopping),
|
|
cancellationToken);
|
|
|
|
if (hasActiveSession)
|
|
{
|
|
room.SetLastAutoStartDecision(
|
|
AutoStartDecisionCodes.SkippedActiveSession,
|
|
"Auto-start skipped because an active recording session already exists.",
|
|
null,
|
|
DateTimeOffset.UtcNow);
|
|
await _dbContext.SaveChangesAsync(cancellationToken);
|
|
return FailureResult("An active recording session already exists for this live room.");
|
|
}
|
|
|
|
try
|
|
{
|
|
var task = await _recordService.StartAsync(
|
|
new StartRecordTaskRequest
|
|
{
|
|
LiveRoomId = liveRoomId
|
|
},
|
|
trackAutoStartDecision: true,
|
|
cancellationToken);
|
|
|
|
var started = task.Status is RecordTaskStatus.Starting or RecordTaskStatus.Running;
|
|
return new RecoveryActionResultDto
|
|
{
|
|
RequestedCount = 1,
|
|
SuccessCount = started ? 1 : 0,
|
|
FailedCount = started ? 0 : 1,
|
|
Messages =
|
|
[
|
|
started
|
|
? $"Recording retry started for room {room.RoomId}."
|
|
: $"Recording retry did not start for room {room.RoomId}. Status={task.Status}; Error={task.ErrorMessage ?? "n/a"}"
|
|
]
|
|
};
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
return FailureResult($"Recording retry failed for room {room.RoomId}: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
public async Task<RecoveryActionResultDto> RetryAllLiveRoomsAsync(CancellationToken cancellationToken = default)
|
|
{
|
|
var liveRooms = await ListRecoverableLiveRoomsAsync(cancellationToken);
|
|
if (liveRooms.Count == 0)
|
|
{
|
|
return new RecoveryActionResultDto
|
|
{
|
|
RequestedCount = 0,
|
|
SuccessCount = 0,
|
|
FailedCount = 0,
|
|
Messages = ["No live rooms currently require retry."]
|
|
};
|
|
}
|
|
|
|
var messages = new List<string>();
|
|
var successCount = 0;
|
|
foreach (var item in liveRooms)
|
|
{
|
|
var result = await RetryLiveRoomAsync(item.LiveRoomId, cancellationToken);
|
|
successCount += result.SuccessCount;
|
|
messages.AddRange(result.Messages);
|
|
}
|
|
|
|
return new RecoveryActionResultDto
|
|
{
|
|
RequestedCount = liveRooms.Count,
|
|
SuccessCount = successCount,
|
|
FailedCount = liveRooms.Count - successCount,
|
|
Messages = messages
|
|
};
|
|
}
|
|
|
|
public async Task<RecoveryActionResultDto> ResumeFinalizationAsync(Guid recordTaskId, CancellationToken cancellationToken = default)
|
|
{
|
|
var started = await _ffmpegService.StartManualFinalizeTaskAsync(recordTaskId, cancellationToken);
|
|
return new RecoveryActionResultDto
|
|
{
|
|
RequestedCount = 1,
|
|
SuccessCount = started ? 1 : 0,
|
|
FailedCount = started ? 0 : 1,
|
|
Messages =
|
|
[
|
|
started
|
|
? $"MP4 finalization resumed for task {recordTaskId}."
|
|
: $"MP4 finalization could not be resumed for task {recordTaskId}."
|
|
]
|
|
};
|
|
}
|
|
|
|
public async Task<RecoveryActionResultDto> ResumeAllFinalizationsAsync(CancellationToken cancellationToken = default)
|
|
{
|
|
var finalizations = await ListRecoverableFinalizationsAsync(cancellationToken);
|
|
if (finalizations.Count == 0)
|
|
{
|
|
return new RecoveryActionResultDto
|
|
{
|
|
RequestedCount = 0,
|
|
SuccessCount = 0,
|
|
FailedCount = 0,
|
|
Messages = ["No MP4 finalization tasks currently require recovery."]
|
|
};
|
|
}
|
|
|
|
var successCount = 0;
|
|
var messages = new List<string>();
|
|
foreach (var item in finalizations)
|
|
{
|
|
var started = await _ffmpegService.StartManualFinalizeTaskAsync(item.RecordTaskId, cancellationToken);
|
|
if (started)
|
|
{
|
|
successCount++;
|
|
}
|
|
|
|
messages.Add(
|
|
started
|
|
? $"MP4 finalization resumed for task {item.RecordTaskId}."
|
|
: $"MP4 finalization could not be resumed for task {item.RecordTaskId}.");
|
|
}
|
|
|
|
return new RecoveryActionResultDto
|
|
{
|
|
RequestedCount = finalizations.Count,
|
|
SuccessCount = successCount,
|
|
FailedCount = finalizations.Count - successCount,
|
|
Messages = messages
|
|
};
|
|
}
|
|
|
|
private async Task<IReadOnlyList<RecoverableLiveRoomDto>> ListRecoverableLiveRoomsAsync(CancellationToken cancellationToken)
|
|
{
|
|
var activeLiveRoomIds = await _dbContext.RecordSessions
|
|
.AsNoTracking()
|
|
.Where(item => item.Status == RecordSessionStatus.Starting ||
|
|
item.Status == RecordSessionStatus.Running ||
|
|
item.Status == RecordSessionStatus.Stopping)
|
|
.Select(item => item.LiveRoomId)
|
|
.Distinct()
|
|
.ToListAsync(cancellationToken);
|
|
|
|
var rooms = await _dbContext.LiveRooms
|
|
.AsNoTracking()
|
|
.Where(item => item.IsEnabled &&
|
|
item.AvailabilityStatus == LiveRoomAvailabilityStatus.Live &&
|
|
item.LastAutoStartDecisionCode != AutoStartDecisionCodes.Started)
|
|
.ToListAsync(cancellationToken);
|
|
|
|
return rooms
|
|
.Where(item => !activeLiveRoomIds.Contains(item.Id))
|
|
.OrderByDescending(item => item.LastAutoStartDecisionAt ?? item.LastCheckedAt ?? item.UpdatedAt)
|
|
.Select(item => new RecoverableLiveRoomDto
|
|
{
|
|
LiveRoomId = item.Id,
|
|
PlatformName = item.Platform.ToString(),
|
|
RoomId = item.RoomId,
|
|
Title = item.Title,
|
|
AnchorName = item.AnchorName,
|
|
LastAutoStartDecisionCode = item.LastAutoStartDecisionCode,
|
|
LastAutoStartDecisionSummary = item.LastAutoStartDecisionSummary,
|
|
LastAutoStartDecisionDetail = item.LastAutoStartDecisionDetail,
|
|
LastAutoStartDecisionAt = item.LastAutoStartDecisionAt,
|
|
LastCheckedAt = item.LastCheckedAt
|
|
})
|
|
.ToList();
|
|
}
|
|
|
|
private async Task<IReadOnlyList<RecoverableFinalizationDto>> ListRecoverableFinalizationsAsync(CancellationToken cancellationToken)
|
|
{
|
|
var tasks = await _dbContext.RecordTasks
|
|
.AsNoTracking()
|
|
.Include(item => item.LiveRoom)
|
|
.Include(item => item.RecordSession)
|
|
.Include(item => item.Result)
|
|
.Where(item => item.OutputFormat == RecordOutputFormat.Mp4 &&
|
|
item.RecordSession != null)
|
|
.ToListAsync(cancellationToken);
|
|
|
|
return tasks
|
|
.Where(IsRecoverableFinalization)
|
|
.OrderByDescending(item => item.UpdatedAt)
|
|
.Select(item => new RecoverableFinalizationDto
|
|
{
|
|
RecordTaskId = item.Id,
|
|
RecordSessionId = item.RecordSessionId,
|
|
LiveRoomId = item.LiveRoomId,
|
|
LiveRoomTitle = item.LiveRoom?.Title ?? item.LiveRoom?.AnchorName ?? item.LiveRoom?.RoomId ?? item.LiveRoomId.ToString(),
|
|
RoomId = item.LiveRoom?.RoomId ?? "-",
|
|
PlatformName = item.LiveRoom?.Platform.ToString() ?? "Unknown",
|
|
SegmentIndex = item.SegmentIndex,
|
|
Status = item.Status,
|
|
OutputFilePath = item.OutputFilePath,
|
|
Reason = BuildFinalizationReason(item),
|
|
CreatedAt = item.CreatedAt,
|
|
EndedAt = item.EndedAt
|
|
})
|
|
.ToList();
|
|
}
|
|
|
|
private static bool IsRecoverableFinalization(RecordTask recordTask)
|
|
{
|
|
if (recordTask.RecordSession is null ||
|
|
recordTask.OutputFormat != RecordOutputFormat.Mp4)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if (recordTask.Status == RecordTaskStatus.Processing)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
if (recordTask.Status is RecordTaskStatus.Starting or RecordTaskStatus.Running or RecordTaskStatus.Stopping)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if (recordTask.RecordSession.Status is RecordSessionStatus.Starting or RecordSessionStatus.Running or RecordSessionStatus.Stopping)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if (string.IsNullOrWhiteSpace(recordTask.OutputFilePath))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
var recorderOutputPath = ResolveManualFinalizeSourcePath(recordTask, recordTask.RecordSession);
|
|
if (!File.Exists(recorderOutputPath))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if (recordTask.Status == RecordTaskStatus.Completed)
|
|
{
|
|
var finalOutputPath = NormalizeAbsolutePath(recordTask.OutputFilePath);
|
|
return !HasUsableOutput(finalOutputPath, CalculateFileSize(finalOutputPath));
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
private static string BuildFinalizationReason(RecordTask recordTask)
|
|
{
|
|
if (recordTask.Status == RecordTaskStatus.Processing)
|
|
{
|
|
return string.IsNullOrWhiteSpace(recordTask.ErrorMessage)
|
|
? "MP4 finalization is queued or paused and can be resumed."
|
|
: recordTask.ErrorMessage!;
|
|
}
|
|
|
|
if (recordTask.Status == RecordTaskStatus.Completed)
|
|
{
|
|
return "The final MP4 output is missing, but the intermediate recording file is still available.";
|
|
}
|
|
|
|
return "Manual MP4 finalization can be retried from the intermediate recording file.";
|
|
}
|
|
|
|
private static string ResolveManualFinalizeSourcePath(RecordTask recordTask, RecordSession recordSession)
|
|
{
|
|
var resultPath = recordTask.Result?.FilePath;
|
|
if (!string.IsNullOrWhiteSpace(resultPath) &&
|
|
resultPath.EndsWith(".ts", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
var normalizedResultPath = NormalizeAbsolutePath(resultPath);
|
|
if (File.Exists(normalizedResultPath))
|
|
{
|
|
return normalizedResultPath;
|
|
}
|
|
}
|
|
|
|
return NormalizeAbsolutePath(
|
|
GetRecorderOutputPath(
|
|
recordTask.OutputFilePath ?? resultPath ?? string.Empty,
|
|
recordSession.OutputFormat,
|
|
recordSession.SaveMode));
|
|
}
|
|
|
|
private static string GetRecorderOutputPath(
|
|
string finalOutputPath,
|
|
RecordOutputFormat outputFormat,
|
|
RecordSaveMode saveMode)
|
|
{
|
|
if (outputFormat != RecordOutputFormat.Mp4)
|
|
{
|
|
return finalOutputPath;
|
|
}
|
|
|
|
if (saveMode == RecordSaveMode.SingleFile)
|
|
{
|
|
return Path.Combine(
|
|
Path.GetDirectoryName(finalOutputPath)!,
|
|
$"{Path.GetFileNameWithoutExtension(finalOutputPath)}.recording.ts");
|
|
}
|
|
|
|
return Path.ChangeExtension(finalOutputPath, ".ts");
|
|
}
|
|
|
|
private static long? CalculateFileSize(string? outputPath)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(outputPath))
|
|
{
|
|
return null;
|
|
}
|
|
|
|
if (File.Exists(outputPath))
|
|
{
|
|
return new FileInfo(outputPath).Length;
|
|
}
|
|
|
|
if (Directory.Exists(outputPath))
|
|
{
|
|
return new DirectoryInfo(outputPath)
|
|
.EnumerateFiles("*", SearchOption.TopDirectoryOnly)
|
|
.Sum(static file => file.Length);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
private static bool HasUsableOutput(string? outputPath, long? fileSize)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(outputPath))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if (File.Exists(outputPath))
|
|
{
|
|
return fileSize.GetValueOrDefault() > 0;
|
|
}
|
|
|
|
if (Directory.Exists(outputPath))
|
|
{
|
|
return Directory.EnumerateFiles(outputPath, "*", SearchOption.TopDirectoryOnly).Any();
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
private static string NormalizeAbsolutePath(string path) =>
|
|
Path.IsPathRooted(path)
|
|
? path
|
|
: Path.GetFullPath(path, AppContext.BaseDirectory);
|
|
|
|
private static RecoveryActionResultDto FailureResult(string message) => new()
|
|
{
|
|
RequestedCount = 1,
|
|
SuccessCount = 0,
|
|
FailedCount = 1,
|
|
Messages = [message]
|
|
};
|
|
}
|