feat: add tiered storage guard (Green/Yellow/Red) and dashboard queue monitor

Storage Tier System:
- Add StorageTier enum (Green >30% / Yellow 10-30% / Red <10%)
- Extend StorageGuardResult with Tier, CanStartNewRecording, ShouldPauseActive, UsagePercent
- Yellow tier: deny new recordings but allow existing to finish and upload
- Red tier: deny new recordings and pause active sessions
- Auto-recovery: when disk frees up, polling automatically resumes new recordings
- Update LiveRoomPollingBackgroundService to use tier-based checks
- Expose tier + usage percent in Recovery API

Dashboard Queue Monitor:
- Add pending transcode count, pending upload count, queued data volume to dashboard
- Add storage tier badge (Green/Yellow/Red) with usage percentage
- Add queue monitoring card to dashboard view

Co-Authored-By: Claude Opus 4.8 noreply@anthropic.com
This commit is contained in:
2026-06-04 23:13:26 +08:00
co-authored by Claude Opus 4.8 noreply@anthropic.com
parent cbee29bef9
commit 24e5cf2a06
12 changed files with 203 additions and 22 deletions
@@ -48,6 +48,8 @@ public interface IRecordTaskRepository
Task<double> SumDurationSecondsAsync(DateTimeOffset startedFrom, DateTimeOffset startedTo, CancellationToken cancellationToken = default);
Task<int> CountByStatusAsync(RecordTaskStatus status, CancellationToken cancellationToken = default);
Task AddAsync(RecordTask recordTask, CancellationToken cancellationToken = default);
void Remove(RecordTask recordTask);
@@ -84,6 +86,10 @@ public interface IRecordResultRepository
Task<(long TotalBytes, int TotalDanmaku)> GetTodayAggregateAsync(DateTimeOffset createdFrom, DateTimeOffset createdTo, CancellationToken cancellationToken = default);
Task<int> CountPendingUploadAsync(CancellationToken cancellationToken = default);
Task<long> SumPendingUploadBytesAsync(CancellationToken cancellationToken = default);
Task AddAsync(RecordResult recordResult, CancellationToken cancellationToken = default);
void Update(RecordResult recordResult);
@@ -2,6 +2,18 @@ using LiveRecorder.Application.Models.Settings;
namespace LiveRecorder.Application.Abstractions.Storage;
public enum StorageTier
{
/// <summary>Disk has sufficient free space for normal operation.</summary>
Green = 0,
/// <summary>Disk space is low. Deny new recordings but allow existing to finish.</summary>
Yellow = 1,
/// <summary>Disk space is critically low. Deny new recordings and pause active ones.</summary>
Red = 2
}
public interface IStorageGuardService
{
StorageGuardResult CheckCanStartOrResume(SystemSettingsDto settings, long additionalRequiredBytes = 0);
@@ -15,5 +27,25 @@ public sealed record StorageGuardResult(
string CheckedPath,
long AvailableBytes,
long RequiredBytes,
string Message);
string Message)
{
/// <summary>
/// Current storage tier (Green/Yellow/Red).
/// </summary>
public StorageTier Tier { get; init; }
/// <summary>
/// Disk usage percentage (0-100). Only populated when IsEnabled is true.
/// </summary>
public double UsagePercent { get; init; }
/// <summary>
/// True if new recordings can be started. False in Yellow and Red tiers.
/// </summary>
public bool CanStartNewRecording => HasEnoughSpace || Tier == StorageTier.Green;
/// <summary>
/// True if active recordings should be paused. Only true in Red tier.
/// </summary>
public bool ShouldPauseActive => Tier == StorageTier.Red;
}
@@ -24,6 +24,12 @@ public sealed class StorageGuardStatusDto
public long RequiredBytes { get; init; }
public required string Message { get; init; }
/// <summary>Storage tier: Green, Yellow, or Red.</summary>
public string Tier { get; init; } = "Green";
/// <summary>Disk usage percentage (0-100).</summary>
public double UsagePercent { get; init; }
}
public sealed class RecoverableLiveRoomDto
@@ -64,6 +64,21 @@ public sealed class DashboardDto
/// Top live rooms by recording duration today (up to 5).
/// </summary>
public required IReadOnlyList<TopRoomItemDto> TopRooms { get; init; }
/// <summary>
/// Number of recording tasks currently in Processing (transcoding) status.
/// </summary>
public int PendingTranscodeCount { get; init; }
/// <summary>
/// Number of recording results with NotUploaded status where local file still exists.
/// </summary>
public int PendingUploadCount { get; init; }
/// <summary>
/// Total file size in bytes of files awaiting upload.
/// </summary>
public long QueuedDataBytes { get; init; }
}
public sealed class StorageStatusDto
@@ -71,6 +86,8 @@ public sealed class StorageStatusDto
public bool HasEnoughSpace { get; init; }
public string Message { get; init; } = string.Empty;
public long AvailableBytes { get; init; }
public string Tier { get; init; } = "Green";
public double UsagePercent { get; init; }
}
public sealed class RecentSessionItemDto
@@ -90,8 +90,13 @@ public sealed class DashboardService
{
HasEnoughSpace = storageCheck.HasEnoughSpace,
Message = storageCheck.Message ?? string.Empty,
AvailableBytes = storageCheck.AvailableBytes
AvailableBytes = storageCheck.AvailableBytes,
Tier = storageCheck.Tier.ToString(),
UsagePercent = storageCheck.UsagePercent
},
PendingTranscodeCount = await _recordTaskRepository.CountByStatusAsync(Domain.Enums.RecordTaskStatus.Processing, cancellationToken),
PendingUploadCount = await _recordResultRepository.CountPendingUploadAsync(cancellationToken),
QueuedDataBytes = await _recordResultRepository.SumPendingUploadBytesAsync(cancellationToken),
RecentSessions = recentSessionsTask.Result
.Select(MapRecentSession)
.ToList(),
@@ -207,8 +207,17 @@ public sealed class RecordService
var settings = await _systemSettingsService.GetAsync(cancellationToken);
var storageCheck = _storageGuardService.CheckCanStartOrResume(settings);
if (!storageCheck.HasEnoughSpace)
if (!storageCheck.CanStartNewRecording)
{
var storageMessage = storageCheck.Tier switch
{
Application.Abstractions.Storage.StorageTier.Yellow =>
"Storage is in warning state. New recordings are paused but existing recordings continue. Transcoding and uploading will free up space.",
Application.Abstractions.Storage.StorageTier.Red =>
"Storage is critically low. All recordings are paused until enough disk space is freed.",
_ => storageCheck.Message
};
await _systemLogService.WriteAsync(
SystemLogLevel.Warning,
"Storage",
@@ -222,12 +231,12 @@ public sealed class RecordService
await UpdateAutoStartDecisionAsync(
liveRoom,
AutoStartDecisionCodes.SkippedStorage,
"Auto-start skipped because storage is below threshold.",
$"Auto-start skipped because storage tier is {storageCheck.Tier}.",
storageCheck.Message,
cancellationToken);
}
throw new InvalidOperationException(storageCheck.Message);
throw new InvalidOperationException(storageMessage);
}
var effectiveSettings = _liveRoomRecordingSettingsResolver.Resolve(liveRoom, settings);
@@ -661,14 +670,14 @@ public sealed class RecordService
}
var storageCheck = _storageGuardService.CheckCanStartOrResume(settings);
if (!storageCheck.HasEnoughSpace)
if (!storageCheck.CanStartNewRecording)
{
foreach (var liveRoomId in liveRoomIds)
{
await TryUpdateAutoStartDecisionAsync(
liveRoomId,
AutoStartDecisionCodes.SkippedStorage,
"Auto-start skipped because storage is below threshold.",
$"Auto-start skipped because storage tier is {storageCheck.Tier}.",
storageCheck.Message,
cancellationToken);
await _systemLogService.WriteAsync(
@@ -142,6 +142,9 @@ public sealed class RecordTaskRepository : IRecordTaskRepository
.Where(item => item.RecordSession != null && item.RecordSession.StartedAt >= startedFrom && item.RecordSession.StartedAt <= startedTo)
.SumAsync(item => item.DurationSeconds ?? 0, cancellationToken);
public Task<int> CountByStatusAsync(RecordTaskStatus status, CancellationToken cancellationToken = default) =>
_dbContext.RecordTasks.CountAsync(item => item.Status == status, cancellationToken);
public Task AddAsync(RecordTask recordTask, CancellationToken cancellationToken = default) =>
_dbContext.RecordTasks.AddAsync(recordTask, cancellationToken).AsTask();
@@ -266,6 +269,14 @@ public sealed class RecordResultRepository : IRecordResultRepository
return (result?.TotalBytes ?? 0L, result?.TotalDanmaku ?? 0);
}
public Task<int> CountPendingUploadAsync(CancellationToken cancellationToken = default) =>
_dbContext.RecordResults.CountAsync(item => item.UploadStatus == RecordArtifactUploadStatus.NotUploaded, cancellationToken);
public Task<long> SumPendingUploadBytesAsync(CancellationToken cancellationToken = default) =>
_dbContext.RecordResults
.Where(item => item.UploadStatus == RecordArtifactUploadStatus.NotUploaded)
.SumAsync(item => item.FileSizeBytes ?? 0L, cancellationToken);
public Task AddAsync(RecordResult recordResult, CancellationToken cancellationToken = default) =>
_dbContext.RecordResults.AddAsync(recordResult, cancellationToken).AsTask();
@@ -75,7 +75,8 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
var settings = await settingsService.GetAsync(stoppingToken);
var ffmpegService = scope.ServiceProvider.GetRequiredService<IFfmpegService>();
var storageGuardService = scope.ServiceProvider.GetRequiredService<IStorageGuardService>();
if (storageGuardService.CheckCanStartOrResume(settings).HasEnoughSpace)
var storageCheck = storageGuardService.CheckCanStartOrResume(settings);
if (storageCheck.Tier != StorageTier.Red)
{
await ffmpegService.ResumePausedFinalizationsAsync(stoppingToken);
}
@@ -382,13 +383,13 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
}
var startCheck = storageGuardService.CheckCanStartOrResume(settings);
if (!startCheck.HasEnoughSpace)
if (!startCheck.CanStartNewRecording)
{
await UpdateAutoStartDecisionAsync(
dbContext,
liveRoom,
AutoStartDecisionCodes.SkippedStorage,
"Auto-start skipped because storage is below threshold.",
$"Auto-start skipped because storage tier is {startCheck.Tier}.",
startCheck.Message,
cancellationToken);
await logService.WriteAsync(
@@ -50,7 +50,9 @@ public sealed class RecoveryService
CheckedPath = storage.CheckedPath,
AvailableBytes = storage.AvailableBytes,
RequiredBytes = storage.RequiredBytes,
Message = storage.Message
Message = storage.Message,
Tier = storage.Tier.ToString(),
UsagePercent = storage.UsagePercent
},
LiveRooms = liveRooms,
Finalizations = finalizations
@@ -7,6 +7,8 @@ namespace LiveRecorder.Infrastructure.Services;
public sealed class StorageGuardService : IStorageGuardService
{
private const long Megabyte = 1024L * 1024L;
private const double GreenThresholdPercent = 30.0;
private const double YellowThresholdPercent = 10.0;
private readonly ILogger<StorageGuardService> _logger;
public StorageGuardService(ILogger<StorageGuardService> logger)
@@ -24,7 +26,11 @@ public sealed class StorageGuardService : IStorageGuardService
{
if (!settings.EnableStorageGuard)
{
return new StorageGuardResult(false, true, ResolveOutputRoot(settings.OutputRoot), long.MaxValue, 0, "Storage guard is disabled.");
return new StorageGuardResult(false, true, ResolveOutputRoot(settings.OutputRoot), long.MaxValue, 0, "Storage guard is disabled.")
{
Tier = StorageTier.Green,
UsagePercent = 0
};
}
var checkedPath = ResolveOutputRoot(settings.OutputRoot);
@@ -35,17 +41,51 @@ public sealed class StorageGuardService : IStorageGuardService
{
var drive = new DriveInfo(Path.GetPathRoot(checkedPath) ?? checkedPath);
var availableBytes = drive.AvailableFreeSpace;
var hasEnoughSpace = availableBytes >= requiredBytes;
var message = hasEnoughSpace
? $"Storage is available. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}"
: $"Storage is below threshold. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}";
var totalBytes = drive.TotalSize;
var usedBytes = Math.Max(0, totalBytes - availableBytes);
var usagePercent = totalBytes > 0 ? (double)usedBytes / totalBytes * 100.0 : 0;
var freePercent = 100.0 - usagePercent;
return new StorageGuardResult(true, hasEnoughSpace, checkedPath, availableBytes, requiredBytes, message);
// Determine tier
StorageTier tier;
if (freePercent >= GreenThresholdPercent)
{
tier = StorageTier.Green;
}
else if (freePercent >= YellowThresholdPercent)
{
tier = StorageTier.Yellow;
}
else
{
tier = StorageTier.Red;
}
var hasEnoughSpace = availableBytes >= requiredBytes;
var message = tier switch
{
StorageTier.Green => $"Storage is healthy. free={FormatBytes(availableBytes)} ({freePercent:F1}%), required={FormatBytes(requiredBytes)}, path={checkedPath}",
StorageTier.Yellow => $"Storage is low. free={FormatBytes(availableBytes)} ({freePercent:F1}%), required={FormatBytes(requiredBytes)}, path={checkedPath}. New recordings paused, existing recordings continue.",
StorageTier.Red => $"Storage is critically low. free={FormatBytes(availableBytes)} ({freePercent:F1}%), required={FormatBytes(requiredBytes)}, path={checkedPath}. All recordings paused, uploads continue.",
_ => hasEnoughSpace
? $"Storage is available. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}"
: $"Storage is below threshold. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}"
};
return new StorageGuardResult(true, hasEnoughSpace, checkedPath, availableBytes, requiredBytes, message)
{
Tier = tier,
UsagePercent = Math.Round(usagePercent, 1)
};
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Storage guard failed to inspect output root {OutputRoot}", checkedPath);
return new StorageGuardResult(true, false, checkedPath, 0, requiredBytes, $"Unable to inspect storage path {checkedPath}: {ex.Message}");
return new StorageGuardResult(true, false, checkedPath, 0, requiredBytes, $"Unable to inspect storage path {checkedPath}: {ex.Message}")
{
Tier = StorageTier.Red,
UsagePercent = 0
};
}
}
@@ -77,4 +117,3 @@ public sealed class StorageGuardService : IStorageGuardService
return $"{display:0.##} {units[unitIndex]}";
}
}