feat: add tiered storage guard (Green/Yellow/Red) and dashboard queue monitor
Storage Tier System: - Add StorageTier enum (Green >30% / Yellow 10-30% / Red <10%) - Extend StorageGuardResult with Tier, CanStartNewRecording, ShouldPauseActive, UsagePercent - Yellow tier: deny new recordings but allow existing to finish and upload - Red tier: deny new recordings and pause active sessions - Auto-recovery: when disk frees up, polling automatically resumes new recordings - Update LiveRoomPollingBackgroundService to use tier-based checks - Expose tier + usage percent in Recovery API Dashboard Queue Monitor: - Add pending transcode count, pending upload count, queued data volume to dashboard - Add storage tier badge (Green/Yellow/Red) with usage percentage - Add queue monitoring card to dashboard view Co-Authored-By: Claude Opus 4.8 noreply@anthropic.com
This commit is contained in:
co-authored by
Claude Opus 4.8 noreply@anthropic.com
parent
cbee29bef9
commit
24e5cf2a06
@@ -48,6 +48,8 @@ public interface IRecordTaskRepository
|
||||
|
||||
Task<double> SumDurationSecondsAsync(DateTimeOffset startedFrom, DateTimeOffset startedTo, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<int> CountByStatusAsync(RecordTaskStatus status, CancellationToken cancellationToken = default);
|
||||
|
||||
Task AddAsync(RecordTask recordTask, CancellationToken cancellationToken = default);
|
||||
|
||||
void Remove(RecordTask recordTask);
|
||||
@@ -84,6 +86,10 @@ public interface IRecordResultRepository
|
||||
|
||||
Task<(long TotalBytes, int TotalDanmaku)> GetTodayAggregateAsync(DateTimeOffset createdFrom, DateTimeOffset createdTo, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<int> CountPendingUploadAsync(CancellationToken cancellationToken = default);
|
||||
|
||||
Task<long> SumPendingUploadBytesAsync(CancellationToken cancellationToken = default);
|
||||
|
||||
Task AddAsync(RecordResult recordResult, CancellationToken cancellationToken = default);
|
||||
|
||||
void Update(RecordResult recordResult);
|
||||
|
||||
@@ -2,6 +2,18 @@ using LiveRecorder.Application.Models.Settings;
|
||||
|
||||
namespace LiveRecorder.Application.Abstractions.Storage;
|
||||
|
||||
public enum StorageTier
|
||||
{
|
||||
/// <summary>Disk has sufficient free space for normal operation.</summary>
|
||||
Green = 0,
|
||||
|
||||
/// <summary>Disk space is low. Deny new recordings but allow existing to finish.</summary>
|
||||
Yellow = 1,
|
||||
|
||||
/// <summary>Disk space is critically low. Deny new recordings and pause active ones.</summary>
|
||||
Red = 2
|
||||
}
|
||||
|
||||
public interface IStorageGuardService
|
||||
{
|
||||
StorageGuardResult CheckCanStartOrResume(SystemSettingsDto settings, long additionalRequiredBytes = 0);
|
||||
@@ -15,5 +27,25 @@ public sealed record StorageGuardResult(
|
||||
string CheckedPath,
|
||||
long AvailableBytes,
|
||||
long RequiredBytes,
|
||||
string Message);
|
||||
string Message)
|
||||
{
|
||||
/// <summary>
|
||||
/// Current storage tier (Green/Yellow/Red).
|
||||
/// </summary>
|
||||
public StorageTier Tier { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Disk usage percentage (0-100). Only populated when IsEnabled is true.
|
||||
/// </summary>
|
||||
public double UsagePercent { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// True if new recordings can be started. False in Yellow and Red tiers.
|
||||
/// </summary>
|
||||
public bool CanStartNewRecording => HasEnoughSpace || Tier == StorageTier.Green;
|
||||
|
||||
/// <summary>
|
||||
/// True if active recordings should be paused. Only true in Red tier.
|
||||
/// </summary>
|
||||
public bool ShouldPauseActive => Tier == StorageTier.Red;
|
||||
}
|
||||
|
||||
@@ -24,6 +24,12 @@ public sealed class StorageGuardStatusDto
|
||||
public long RequiredBytes { get; init; }
|
||||
|
||||
public required string Message { get; init; }
|
||||
|
||||
/// <summary>Storage tier: Green, Yellow, or Red.</summary>
|
||||
public string Tier { get; init; } = "Green";
|
||||
|
||||
/// <summary>Disk usage percentage (0-100).</summary>
|
||||
public double UsagePercent { get; init; }
|
||||
}
|
||||
|
||||
public sealed class RecoverableLiveRoomDto
|
||||
|
||||
@@ -64,6 +64,21 @@ public sealed class DashboardDto
|
||||
/// Top live rooms by recording duration today (up to 5).
|
||||
/// </summary>
|
||||
public required IReadOnlyList<TopRoomItemDto> TopRooms { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Number of recording tasks currently in Processing (transcoding) status.
|
||||
/// </summary>
|
||||
public int PendingTranscodeCount { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Number of recording results with NotUploaded status where local file still exists.
|
||||
/// </summary>
|
||||
public int PendingUploadCount { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Total file size in bytes of files awaiting upload.
|
||||
/// </summary>
|
||||
public long QueuedDataBytes { get; init; }
|
||||
}
|
||||
|
||||
public sealed class StorageStatusDto
|
||||
@@ -71,6 +86,8 @@ public sealed class StorageStatusDto
|
||||
public bool HasEnoughSpace { get; init; }
|
||||
public string Message { get; init; } = string.Empty;
|
||||
public long AvailableBytes { get; init; }
|
||||
public string Tier { get; init; } = "Green";
|
||||
public double UsagePercent { get; init; }
|
||||
}
|
||||
|
||||
public sealed class RecentSessionItemDto
|
||||
|
||||
@@ -90,8 +90,13 @@ public sealed class DashboardService
|
||||
{
|
||||
HasEnoughSpace = storageCheck.HasEnoughSpace,
|
||||
Message = storageCheck.Message ?? string.Empty,
|
||||
AvailableBytes = storageCheck.AvailableBytes
|
||||
AvailableBytes = storageCheck.AvailableBytes,
|
||||
Tier = storageCheck.Tier.ToString(),
|
||||
UsagePercent = storageCheck.UsagePercent
|
||||
},
|
||||
PendingTranscodeCount = await _recordTaskRepository.CountByStatusAsync(Domain.Enums.RecordTaskStatus.Processing, cancellationToken),
|
||||
PendingUploadCount = await _recordResultRepository.CountPendingUploadAsync(cancellationToken),
|
||||
QueuedDataBytes = await _recordResultRepository.SumPendingUploadBytesAsync(cancellationToken),
|
||||
RecentSessions = recentSessionsTask.Result
|
||||
.Select(MapRecentSession)
|
||||
.ToList(),
|
||||
|
||||
@@ -207,8 +207,17 @@ public sealed class RecordService
|
||||
|
||||
var settings = await _systemSettingsService.GetAsync(cancellationToken);
|
||||
var storageCheck = _storageGuardService.CheckCanStartOrResume(settings);
|
||||
if (!storageCheck.HasEnoughSpace)
|
||||
if (!storageCheck.CanStartNewRecording)
|
||||
{
|
||||
var storageMessage = storageCheck.Tier switch
|
||||
{
|
||||
Application.Abstractions.Storage.StorageTier.Yellow =>
|
||||
"Storage is in warning state. New recordings are paused but existing recordings continue. Transcoding and uploading will free up space.",
|
||||
Application.Abstractions.Storage.StorageTier.Red =>
|
||||
"Storage is critically low. All recordings are paused until enough disk space is freed.",
|
||||
_ => storageCheck.Message
|
||||
};
|
||||
|
||||
await _systemLogService.WriteAsync(
|
||||
SystemLogLevel.Warning,
|
||||
"Storage",
|
||||
@@ -222,12 +231,12 @@ public sealed class RecordService
|
||||
await UpdateAutoStartDecisionAsync(
|
||||
liveRoom,
|
||||
AutoStartDecisionCodes.SkippedStorage,
|
||||
"Auto-start skipped because storage is below threshold.",
|
||||
$"Auto-start skipped because storage tier is {storageCheck.Tier}.",
|
||||
storageCheck.Message,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
throw new InvalidOperationException(storageCheck.Message);
|
||||
throw new InvalidOperationException(storageMessage);
|
||||
}
|
||||
|
||||
var effectiveSettings = _liveRoomRecordingSettingsResolver.Resolve(liveRoom, settings);
|
||||
@@ -661,14 +670,14 @@ public sealed class RecordService
|
||||
}
|
||||
|
||||
var storageCheck = _storageGuardService.CheckCanStartOrResume(settings);
|
||||
if (!storageCheck.HasEnoughSpace)
|
||||
if (!storageCheck.CanStartNewRecording)
|
||||
{
|
||||
foreach (var liveRoomId in liveRoomIds)
|
||||
{
|
||||
await TryUpdateAutoStartDecisionAsync(
|
||||
liveRoomId,
|
||||
AutoStartDecisionCodes.SkippedStorage,
|
||||
"Auto-start skipped because storage is below threshold.",
|
||||
$"Auto-start skipped because storage tier is {storageCheck.Tier}.",
|
||||
storageCheck.Message,
|
||||
cancellationToken);
|
||||
await _systemLogService.WriteAsync(
|
||||
|
||||
Reference in New Issue
Block a user