feat: add tiered storage guard (Green/Yellow/Red) and dashboard queue monitor

Storage Tier System:
- Add StorageTier enum (Green >30% / Yellow 10-30% / Red <10%)
- Extend StorageGuardResult with Tier, CanStartNewRecording, ShouldPauseActive, UsagePercent
- Yellow tier: deny new recordings but allow existing to finish and upload
- Red tier: deny new recordings and pause active sessions
- Auto-recovery: when disk frees up, polling automatically resumes new recordings
- Update LiveRoomPollingBackgroundService to use tier-based checks
- Expose tier + usage percent in Recovery API

Dashboard Queue Monitor:
- Add pending transcode count, pending upload count, queued data volume to dashboard
- Add storage tier badge (Green/Yellow/Red) with usage percentage
- Add queue monitoring card to dashboard view

Co-Authored-By: Claude Opus 4.8 noreply@anthropic.com
This commit is contained in:
2026-06-04 23:13:26 +08:00
co-authored by Claude Opus 4.8 noreply@anthropic.com
parent cbee29bef9
commit 24e5cf2a06
12 changed files with 203 additions and 22 deletions
@@ -75,7 +75,8 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
var settings = await settingsService.GetAsync(stoppingToken);
var ffmpegService = scope.ServiceProvider.GetRequiredService<IFfmpegService>();
var storageGuardService = scope.ServiceProvider.GetRequiredService<IStorageGuardService>();
if (storageGuardService.CheckCanStartOrResume(settings).HasEnoughSpace)
var storageCheck = storageGuardService.CheckCanStartOrResume(settings);
if (storageCheck.Tier != StorageTier.Red)
{
await ffmpegService.ResumePausedFinalizationsAsync(stoppingToken);
}
@@ -382,13 +383,13 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
}
var startCheck = storageGuardService.CheckCanStartOrResume(settings);
if (!startCheck.HasEnoughSpace)
if (!startCheck.CanStartNewRecording)
{
await UpdateAutoStartDecisionAsync(
dbContext,
liveRoom,
AutoStartDecisionCodes.SkippedStorage,
"Auto-start skipped because storage is below threshold.",
$"Auto-start skipped because storage tier is {startCheck.Tier}.",
startCheck.Message,
cancellationToken);
await logService.WriteAsync(
@@ -50,7 +50,9 @@ public sealed class RecoveryService
CheckedPath = storage.CheckedPath,
AvailableBytes = storage.AvailableBytes,
RequiredBytes = storage.RequiredBytes,
Message = storage.Message
Message = storage.Message,
Tier = storage.Tier.ToString(),
UsagePercent = storage.UsagePercent
},
LiveRooms = liveRooms,
Finalizations = finalizations
@@ -7,6 +7,8 @@ namespace LiveRecorder.Infrastructure.Services;
public sealed class StorageGuardService : IStorageGuardService
{
private const long Megabyte = 1024L * 1024L;
private const double GreenThresholdPercent = 30.0;
private const double YellowThresholdPercent = 10.0;
private readonly ILogger<StorageGuardService> _logger;
public StorageGuardService(ILogger<StorageGuardService> logger)
@@ -24,7 +26,11 @@ public sealed class StorageGuardService : IStorageGuardService
{
if (!settings.EnableStorageGuard)
{
return new StorageGuardResult(false, true, ResolveOutputRoot(settings.OutputRoot), long.MaxValue, 0, "Storage guard is disabled.");
return new StorageGuardResult(false, true, ResolveOutputRoot(settings.OutputRoot), long.MaxValue, 0, "Storage guard is disabled.")
{
Tier = StorageTier.Green,
UsagePercent = 0
};
}
var checkedPath = ResolveOutputRoot(settings.OutputRoot);
@@ -35,17 +41,51 @@ public sealed class StorageGuardService : IStorageGuardService
{
var drive = new DriveInfo(Path.GetPathRoot(checkedPath) ?? checkedPath);
var availableBytes = drive.AvailableFreeSpace;
var hasEnoughSpace = availableBytes >= requiredBytes;
var message = hasEnoughSpace
? $"Storage is available. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}"
: $"Storage is below threshold. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}";
var totalBytes = drive.TotalSize;
var usedBytes = Math.Max(0, totalBytes - availableBytes);
var usagePercent = totalBytes > 0 ? (double)usedBytes / totalBytes * 100.0 : 0;
var freePercent = 100.0 - usagePercent;
return new StorageGuardResult(true, hasEnoughSpace, checkedPath, availableBytes, requiredBytes, message);
// Determine tier
StorageTier tier;
if (freePercent >= GreenThresholdPercent)
{
tier = StorageTier.Green;
}
else if (freePercent >= YellowThresholdPercent)
{
tier = StorageTier.Yellow;
}
else
{
tier = StorageTier.Red;
}
var hasEnoughSpace = availableBytes >= requiredBytes;
var message = tier switch
{
StorageTier.Green => $"Storage is healthy. free={FormatBytes(availableBytes)} ({freePercent:F1}%), required={FormatBytes(requiredBytes)}, path={checkedPath}",
StorageTier.Yellow => $"Storage is low. free={FormatBytes(availableBytes)} ({freePercent:F1}%), required={FormatBytes(requiredBytes)}, path={checkedPath}. New recordings paused, existing recordings continue.",
StorageTier.Red => $"Storage is critically low. free={FormatBytes(availableBytes)} ({freePercent:F1}%), required={FormatBytes(requiredBytes)}, path={checkedPath}. All recordings paused, uploads continue.",
_ => hasEnoughSpace
? $"Storage is available. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}"
: $"Storage is below threshold. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}"
};
return new StorageGuardResult(true, hasEnoughSpace, checkedPath, availableBytes, requiredBytes, message)
{
Tier = tier,
UsagePercent = Math.Round(usagePercent, 1)
};
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Storage guard failed to inspect output root {OutputRoot}", checkedPath);
return new StorageGuardResult(true, false, checkedPath, 0, requiredBytes, $"Unable to inspect storage path {checkedPath}: {ex.Message}");
return new StorageGuardResult(true, false, checkedPath, 0, requiredBytes, $"Unable to inspect storage path {checkedPath}: {ex.Message}")
{
Tier = StorageTier.Red,
UsagePercent = 0
};
}
}
@@ -77,4 +117,3 @@ public sealed class StorageGuardService : IStorageGuardService
return $"{display:0.##} {units[unitIndex]}";
}
}