feat: add tiered storage guard (Green/Yellow/Red) and dashboard queue monitor
Storage Tier System: - Add StorageTier enum (Green >30% / Yellow 10-30% / Red <10%) - Extend StorageGuardResult with Tier, CanStartNewRecording, ShouldPauseActive, UsagePercent - Yellow tier: deny new recordings but allow existing to finish and upload - Red tier: deny new recordings and pause active sessions - Auto-recovery: when disk frees up, polling automatically resumes new recordings - Update LiveRoomPollingBackgroundService to use tier-based checks - Expose tier + usage percent in Recovery API Dashboard Queue Monitor: - Add pending transcode count, pending upload count, queued data volume to dashboard - Add storage tier badge (Green/Yellow/Red) with usage percentage - Add queue monitoring card to dashboard view Co-Authored-By: Claude Opus 4.8 noreply@anthropic.com
This commit is contained in:
co-authored by
Claude Opus 4.8 noreply@anthropic.com
parent
cbee29bef9
commit
24e5cf2a06
@@ -75,7 +75,8 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
|
||||
var settings = await settingsService.GetAsync(stoppingToken);
|
||||
var ffmpegService = scope.ServiceProvider.GetRequiredService<IFfmpegService>();
|
||||
var storageGuardService = scope.ServiceProvider.GetRequiredService<IStorageGuardService>();
|
||||
if (storageGuardService.CheckCanStartOrResume(settings).HasEnoughSpace)
|
||||
var storageCheck = storageGuardService.CheckCanStartOrResume(settings);
|
||||
if (storageCheck.Tier != StorageTier.Red)
|
||||
{
|
||||
await ffmpegService.ResumePausedFinalizationsAsync(stoppingToken);
|
||||
}
|
||||
@@ -382,13 +383,13 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
|
||||
}
|
||||
|
||||
var startCheck = storageGuardService.CheckCanStartOrResume(settings);
|
||||
if (!startCheck.HasEnoughSpace)
|
||||
if (!startCheck.CanStartNewRecording)
|
||||
{
|
||||
await UpdateAutoStartDecisionAsync(
|
||||
dbContext,
|
||||
liveRoom,
|
||||
AutoStartDecisionCodes.SkippedStorage,
|
||||
"Auto-start skipped because storage is below threshold.",
|
||||
$"Auto-start skipped because storage tier is {startCheck.Tier}.",
|
||||
startCheck.Message,
|
||||
cancellationToken);
|
||||
await logService.WriteAsync(
|
||||
|
||||
@@ -50,7 +50,9 @@ public sealed class RecoveryService
|
||||
CheckedPath = storage.CheckedPath,
|
||||
AvailableBytes = storage.AvailableBytes,
|
||||
RequiredBytes = storage.RequiredBytes,
|
||||
Message = storage.Message
|
||||
Message = storage.Message,
|
||||
Tier = storage.Tier.ToString(),
|
||||
UsagePercent = storage.UsagePercent
|
||||
},
|
||||
LiveRooms = liveRooms,
|
||||
Finalizations = finalizations
|
||||
|
||||
@@ -7,6 +7,8 @@ namespace LiveRecorder.Infrastructure.Services;
|
||||
public sealed class StorageGuardService : IStorageGuardService
|
||||
{
|
||||
private const long Megabyte = 1024L * 1024L;
|
||||
private const double GreenThresholdPercent = 30.0;
|
||||
private const double YellowThresholdPercent = 10.0;
|
||||
private readonly ILogger<StorageGuardService> _logger;
|
||||
|
||||
public StorageGuardService(ILogger<StorageGuardService> logger)
|
||||
@@ -24,7 +26,11 @@ public sealed class StorageGuardService : IStorageGuardService
|
||||
{
|
||||
if (!settings.EnableStorageGuard)
|
||||
{
|
||||
return new StorageGuardResult(false, true, ResolveOutputRoot(settings.OutputRoot), long.MaxValue, 0, "Storage guard is disabled.");
|
||||
return new StorageGuardResult(false, true, ResolveOutputRoot(settings.OutputRoot), long.MaxValue, 0, "Storage guard is disabled.")
|
||||
{
|
||||
Tier = StorageTier.Green,
|
||||
UsagePercent = 0
|
||||
};
|
||||
}
|
||||
|
||||
var checkedPath = ResolveOutputRoot(settings.OutputRoot);
|
||||
@@ -35,17 +41,51 @@ public sealed class StorageGuardService : IStorageGuardService
|
||||
{
|
||||
var drive = new DriveInfo(Path.GetPathRoot(checkedPath) ?? checkedPath);
|
||||
var availableBytes = drive.AvailableFreeSpace;
|
||||
var hasEnoughSpace = availableBytes >= requiredBytes;
|
||||
var message = hasEnoughSpace
|
||||
? $"Storage is available. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}"
|
||||
: $"Storage is below threshold. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}";
|
||||
var totalBytes = drive.TotalSize;
|
||||
var usedBytes = Math.Max(0, totalBytes - availableBytes);
|
||||
var usagePercent = totalBytes > 0 ? (double)usedBytes / totalBytes * 100.0 : 0;
|
||||
var freePercent = 100.0 - usagePercent;
|
||||
|
||||
return new StorageGuardResult(true, hasEnoughSpace, checkedPath, availableBytes, requiredBytes, message);
|
||||
// Determine tier
|
||||
StorageTier tier;
|
||||
if (freePercent >= GreenThresholdPercent)
|
||||
{
|
||||
tier = StorageTier.Green;
|
||||
}
|
||||
else if (freePercent >= YellowThresholdPercent)
|
||||
{
|
||||
tier = StorageTier.Yellow;
|
||||
}
|
||||
else
|
||||
{
|
||||
tier = StorageTier.Red;
|
||||
}
|
||||
|
||||
var hasEnoughSpace = availableBytes >= requiredBytes;
|
||||
var message = tier switch
|
||||
{
|
||||
StorageTier.Green => $"Storage is healthy. free={FormatBytes(availableBytes)} ({freePercent:F1}%), required={FormatBytes(requiredBytes)}, path={checkedPath}",
|
||||
StorageTier.Yellow => $"Storage is low. free={FormatBytes(availableBytes)} ({freePercent:F1}%), required={FormatBytes(requiredBytes)}, path={checkedPath}. New recordings paused, existing recordings continue.",
|
||||
StorageTier.Red => $"Storage is critically low. free={FormatBytes(availableBytes)} ({freePercent:F1}%), required={FormatBytes(requiredBytes)}, path={checkedPath}. All recordings paused, uploads continue.",
|
||||
_ => hasEnoughSpace
|
||||
? $"Storage is available. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}"
|
||||
: $"Storage is below threshold. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}"
|
||||
};
|
||||
|
||||
return new StorageGuardResult(true, hasEnoughSpace, checkedPath, availableBytes, requiredBytes, message)
|
||||
{
|
||||
Tier = tier,
|
||||
UsagePercent = Math.Round(usagePercent, 1)
|
||||
};
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Storage guard failed to inspect output root {OutputRoot}", checkedPath);
|
||||
return new StorageGuardResult(true, false, checkedPath, 0, requiredBytes, $"Unable to inspect storage path {checkedPath}: {ex.Message}");
|
||||
return new StorageGuardResult(true, false, checkedPath, 0, requiredBytes, $"Unable to inspect storage path {checkedPath}: {ex.Message}")
|
||||
{
|
||||
Tier = StorageTier.Red,
|
||||
UsagePercent = 0
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -77,4 +117,3 @@ public sealed class StorageGuardService : IStorageGuardService
|
||||
return $"{display:0.##} {units[unitIndex]}";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user