feat: add tiered storage guard (Green/Yellow/Red) and dashboard queue monitor

Storage Tier System:
- Add StorageTier enum (Green >30% / Yellow 10-30% / Red <10%)
- Extend StorageGuardResult with Tier, CanStartNewRecording, ShouldPauseActive, UsagePercent
- Yellow tier: deny new recordings but allow existing to finish and upload
- Red tier: deny new recordings and pause active sessions
- Auto-recovery: when disk frees up, polling automatically resumes new recordings
- Update LiveRoomPollingBackgroundService to use tier-based checks
- Expose tier + usage percent in Recovery API

Dashboard Queue Monitor:
- Add pending transcode count, pending upload count, queued data volume to dashboard
- Add storage tier badge (Green/Yellow/Red) with usage percentage
- Add queue monitoring card to dashboard view

Co-Authored-By: Claude Opus 4.8 noreply@anthropic.com
This commit is contained in:
2026-06-04 23:13:26 +08:00
co-authored by Claude Opus 4.8 noreply@anthropic.com
parent cbee29bef9
commit 24e5cf2a06
12 changed files with 203 additions and 22 deletions
+5
View File
@@ -823,12 +823,17 @@ export interface DashboardData {
storageStatus: DashboardStorageStatus; storageStatus: DashboardStorageStatus;
recentSessions: DashboardRecentSession[]; recentSessions: DashboardRecentSession[];
topRooms: DashboardTopRoom[]; topRooms: DashboardTopRoom[];
pendingTranscodeCount: number;
pendingUploadCount: number;
queuedDataBytes: number;
} }
export interface DashboardStorageStatus { export interface DashboardStorageStatus {
hasEnoughSpace: boolean; hasEnoughSpace: boolean;
message: string; message: string;
availableBytes: number; availableBytes: number;
tier: string;
usagePercent: number;
} }
export interface DashboardRecentSession { export interface DashboardRecentSession {
+51 -3
View File
@@ -39,6 +39,20 @@ function sessionStatusTagType(status: number) {
return "warning"; return "warning";
} }
function storageTierTagType(): "success" | "warning" | "danger" {
const tier = data.value?.storageStatus.tier;
if (tier === "Green") return "success";
if (tier === "Yellow") return "warning";
return "danger";
}
function storageTierLabel(): string {
const tier = data.value?.storageStatus.tier;
if (tier === "Green") return "正常";
if (tier === "Yellow") return "警告";
return "紧急";
}
async function loadData() { async function loadData() {
loading.value = true; loading.value = true;
loadError.value = ""; loadError.value = "";
@@ -109,11 +123,15 @@ onMounted(loadData);
<p class="section-subtitle">录制输出路径的磁盘剩余空间和当前录制保护阈值</p> <p class="section-subtitle">录制输出路径的磁盘剩余空间和当前录制保护阈值</p>
<div class="storage-bar-row"> <div class="storage-bar-row">
<div class="storage-bar-card"> <div class="storage-bar-card">
<span class="storage-bar-card__label">状态</span> <span class="storage-bar-card__label">水位线</span>
<el-tag :type="data.storageStatus.hasEnoughSpace ? 'success' : 'danger'"> <el-tag :type="storageTierTagType()" size="large">
{{ data.storageStatus.hasEnoughSpace ? "充足" : "不足" }} {{ storageTierLabel() }}
</el-tag> </el-tag>
</div> </div>
<div class="storage-bar-card">
<span class="storage-bar-card__label">使用率</span>
<span class="storage-bar-card__value">{{ data.storageStatus.usagePercent.toFixed(1) }}%</span>
</div>
<div class="storage-bar-card"> <div class="storage-bar-card">
<span class="storage-bar-card__label">可用空间</span> <span class="storage-bar-card__label">可用空间</span>
<span class="storage-bar-card__value">{{ formatDataSize(data.storageStatus.availableBytes) }}</span> <span class="storage-bar-card__value">{{ formatDataSize(data.storageStatus.availableBytes) }}</span>
@@ -125,6 +143,36 @@ onMounted(loadData);
</div> </div>
</el-card> </el-card>
<!-- Queue Status -->
<el-row :gutter="18">
<el-col :lg="12" :sm="24">
<el-card class="surface-card" shadow="never">
<h3 class="section-title">处理队列</h3>
<p class="section-subtitle">待转码和待上传的文件积压情况</p>
<div class="storage-bar-row">
<div class="storage-bar-card">
<span class="storage-bar-card__label">待转码</span>
<span class="storage-bar-card__value" :style="{ color: data.pendingTranscodeCount > 0 ? 'var(--warning)' : 'var(--text-primary)' }">
{{ data.pendingTranscodeCount }}
</span>
</div>
<div class="storage-bar-card">
<span class="storage-bar-card__label">待上传</span>
<span class="storage-bar-card__value" :style="{ color: data.pendingUploadCount > 0 ? 'var(--warning)' : 'var(--text-primary)' }">
{{ data.pendingUploadCount }}
</span>
</div>
<div class="storage-bar-card">
<span class="storage-bar-card__label">积压数据量</span>
<span class="storage-bar-card__value" :style="{ color: data.queuedDataBytes > 0 ? 'var(--warning)' : 'var(--text-primary)' }">
{{ formatDataSize(data.queuedDataBytes) }}
</span>
</div>
</div>
</el-card>
</el-col>
</el-row>
<!-- Recent Sessions + Top Rooms --> <!-- Recent Sessions + Top Rooms -->
<el-row :gutter="18"> <el-row :gutter="18">
<el-col :lg="12" :sm="24"> <el-col :lg="12" :sm="24">
@@ -48,6 +48,8 @@ public interface IRecordTaskRepository
Task<double> SumDurationSecondsAsync(DateTimeOffset startedFrom, DateTimeOffset startedTo, CancellationToken cancellationToken = default); Task<double> SumDurationSecondsAsync(DateTimeOffset startedFrom, DateTimeOffset startedTo, CancellationToken cancellationToken = default);
Task<int> CountByStatusAsync(RecordTaskStatus status, CancellationToken cancellationToken = default);
Task AddAsync(RecordTask recordTask, CancellationToken cancellationToken = default); Task AddAsync(RecordTask recordTask, CancellationToken cancellationToken = default);
void Remove(RecordTask recordTask); void Remove(RecordTask recordTask);
@@ -84,6 +86,10 @@ public interface IRecordResultRepository
Task<(long TotalBytes, int TotalDanmaku)> GetTodayAggregateAsync(DateTimeOffset createdFrom, DateTimeOffset createdTo, CancellationToken cancellationToken = default); Task<(long TotalBytes, int TotalDanmaku)> GetTodayAggregateAsync(DateTimeOffset createdFrom, DateTimeOffset createdTo, CancellationToken cancellationToken = default);
Task<int> CountPendingUploadAsync(CancellationToken cancellationToken = default);
Task<long> SumPendingUploadBytesAsync(CancellationToken cancellationToken = default);
Task AddAsync(RecordResult recordResult, CancellationToken cancellationToken = default); Task AddAsync(RecordResult recordResult, CancellationToken cancellationToken = default);
void Update(RecordResult recordResult); void Update(RecordResult recordResult);
@@ -2,6 +2,18 @@ using LiveRecorder.Application.Models.Settings;
namespace LiveRecorder.Application.Abstractions.Storage; namespace LiveRecorder.Application.Abstractions.Storage;
public enum StorageTier
{
/// <summary>Disk has sufficient free space for normal operation.</summary>
Green = 0,
/// <summary>Disk space is low. Deny new recordings but allow existing to finish.</summary>
Yellow = 1,
/// <summary>Disk space is critically low. Deny new recordings and pause active ones.</summary>
Red = 2
}
public interface IStorageGuardService public interface IStorageGuardService
{ {
StorageGuardResult CheckCanStartOrResume(SystemSettingsDto settings, long additionalRequiredBytes = 0); StorageGuardResult CheckCanStartOrResume(SystemSettingsDto settings, long additionalRequiredBytes = 0);
@@ -15,5 +27,25 @@ public sealed record StorageGuardResult(
string CheckedPath, string CheckedPath,
long AvailableBytes, long AvailableBytes,
long RequiredBytes, long RequiredBytes,
string Message); string Message)
{
/// <summary>
/// Current storage tier (Green/Yellow/Red).
/// </summary>
public StorageTier Tier { get; init; }
/// <summary>
/// Disk usage percentage (0-100). Only populated when IsEnabled is true.
/// </summary>
public double UsagePercent { get; init; }
/// <summary>
/// True if new recordings can be started. False in Yellow and Red tiers.
/// </summary>
public bool CanStartNewRecording => HasEnoughSpace || Tier == StorageTier.Green;
/// <summary>
/// True if active recordings should be paused. Only true in Red tier.
/// </summary>
public bool ShouldPauseActive => Tier == StorageTier.Red;
}
@@ -24,6 +24,12 @@ public sealed class StorageGuardStatusDto
public long RequiredBytes { get; init; } public long RequiredBytes { get; init; }
public required string Message { get; init; } public required string Message { get; init; }
/// <summary>Storage tier: Green, Yellow, or Red.</summary>
public string Tier { get; init; } = "Green";
/// <summary>Disk usage percentage (0-100).</summary>
public double UsagePercent { get; init; }
} }
public sealed class RecoverableLiveRoomDto public sealed class RecoverableLiveRoomDto
@@ -64,6 +64,21 @@ public sealed class DashboardDto
/// Top live rooms by recording duration today (up to 5). /// Top live rooms by recording duration today (up to 5).
/// </summary> /// </summary>
public required IReadOnlyList<TopRoomItemDto> TopRooms { get; init; } public required IReadOnlyList<TopRoomItemDto> TopRooms { get; init; }
/// <summary>
/// Number of recording tasks currently in Processing (transcoding) status.
/// </summary>
public int PendingTranscodeCount { get; init; }
/// <summary>
/// Number of recording results with NotUploaded status where local file still exists.
/// </summary>
public int PendingUploadCount { get; init; }
/// <summary>
/// Total file size in bytes of files awaiting upload.
/// </summary>
public long QueuedDataBytes { get; init; }
} }
public sealed class StorageStatusDto public sealed class StorageStatusDto
@@ -71,6 +86,8 @@ public sealed class StorageStatusDto
public bool HasEnoughSpace { get; init; } public bool HasEnoughSpace { get; init; }
public string Message { get; init; } = string.Empty; public string Message { get; init; } = string.Empty;
public long AvailableBytes { get; init; } public long AvailableBytes { get; init; }
public string Tier { get; init; } = "Green";
public double UsagePercent { get; init; }
} }
public sealed class RecentSessionItemDto public sealed class RecentSessionItemDto
@@ -90,8 +90,13 @@ public sealed class DashboardService
{ {
HasEnoughSpace = storageCheck.HasEnoughSpace, HasEnoughSpace = storageCheck.HasEnoughSpace,
Message = storageCheck.Message ?? string.Empty, Message = storageCheck.Message ?? string.Empty,
AvailableBytes = storageCheck.AvailableBytes AvailableBytes = storageCheck.AvailableBytes,
Tier = storageCheck.Tier.ToString(),
UsagePercent = storageCheck.UsagePercent
}, },
PendingTranscodeCount = await _recordTaskRepository.CountByStatusAsync(Domain.Enums.RecordTaskStatus.Processing, cancellationToken),
PendingUploadCount = await _recordResultRepository.CountPendingUploadAsync(cancellationToken),
QueuedDataBytes = await _recordResultRepository.SumPendingUploadBytesAsync(cancellationToken),
RecentSessions = recentSessionsTask.Result RecentSessions = recentSessionsTask.Result
.Select(MapRecentSession) .Select(MapRecentSession)
.ToList(), .ToList(),
@@ -207,8 +207,17 @@ public sealed class RecordService
var settings = await _systemSettingsService.GetAsync(cancellationToken); var settings = await _systemSettingsService.GetAsync(cancellationToken);
var storageCheck = _storageGuardService.CheckCanStartOrResume(settings); var storageCheck = _storageGuardService.CheckCanStartOrResume(settings);
if (!storageCheck.HasEnoughSpace) if (!storageCheck.CanStartNewRecording)
{ {
var storageMessage = storageCheck.Tier switch
{
Application.Abstractions.Storage.StorageTier.Yellow =>
"Storage is in warning state. New recordings are paused but existing recordings continue. Transcoding and uploading will free up space.",
Application.Abstractions.Storage.StorageTier.Red =>
"Storage is critically low. All recordings are paused until enough disk space is freed.",
_ => storageCheck.Message
};
await _systemLogService.WriteAsync( await _systemLogService.WriteAsync(
SystemLogLevel.Warning, SystemLogLevel.Warning,
"Storage", "Storage",
@@ -222,12 +231,12 @@ public sealed class RecordService
await UpdateAutoStartDecisionAsync( await UpdateAutoStartDecisionAsync(
liveRoom, liveRoom,
AutoStartDecisionCodes.SkippedStorage, AutoStartDecisionCodes.SkippedStorage,
"Auto-start skipped because storage is below threshold.", $"Auto-start skipped because storage tier is {storageCheck.Tier}.",
storageCheck.Message, storageCheck.Message,
cancellationToken); cancellationToken);
} }
throw new InvalidOperationException(storageCheck.Message); throw new InvalidOperationException(storageMessage);
} }
var effectiveSettings = _liveRoomRecordingSettingsResolver.Resolve(liveRoom, settings); var effectiveSettings = _liveRoomRecordingSettingsResolver.Resolve(liveRoom, settings);
@@ -661,14 +670,14 @@ public sealed class RecordService
} }
var storageCheck = _storageGuardService.CheckCanStartOrResume(settings); var storageCheck = _storageGuardService.CheckCanStartOrResume(settings);
if (!storageCheck.HasEnoughSpace) if (!storageCheck.CanStartNewRecording)
{ {
foreach (var liveRoomId in liveRoomIds) foreach (var liveRoomId in liveRoomIds)
{ {
await TryUpdateAutoStartDecisionAsync( await TryUpdateAutoStartDecisionAsync(
liveRoomId, liveRoomId,
AutoStartDecisionCodes.SkippedStorage, AutoStartDecisionCodes.SkippedStorage,
"Auto-start skipped because storage is below threshold.", $"Auto-start skipped because storage tier is {storageCheck.Tier}.",
storageCheck.Message, storageCheck.Message,
cancellationToken); cancellationToken);
await _systemLogService.WriteAsync( await _systemLogService.WriteAsync(
@@ -142,6 +142,9 @@ public sealed class RecordTaskRepository : IRecordTaskRepository
.Where(item => item.RecordSession != null && item.RecordSession.StartedAt >= startedFrom && item.RecordSession.StartedAt <= startedTo) .Where(item => item.RecordSession != null && item.RecordSession.StartedAt >= startedFrom && item.RecordSession.StartedAt <= startedTo)
.SumAsync(item => item.DurationSeconds ?? 0, cancellationToken); .SumAsync(item => item.DurationSeconds ?? 0, cancellationToken);
public Task<int> CountByStatusAsync(RecordTaskStatus status, CancellationToken cancellationToken = default) =>
_dbContext.RecordTasks.CountAsync(item => item.Status == status, cancellationToken);
public Task AddAsync(RecordTask recordTask, CancellationToken cancellationToken = default) => public Task AddAsync(RecordTask recordTask, CancellationToken cancellationToken = default) =>
_dbContext.RecordTasks.AddAsync(recordTask, cancellationToken).AsTask(); _dbContext.RecordTasks.AddAsync(recordTask, cancellationToken).AsTask();
@@ -266,6 +269,14 @@ public sealed class RecordResultRepository : IRecordResultRepository
return (result?.TotalBytes ?? 0L, result?.TotalDanmaku ?? 0); return (result?.TotalBytes ?? 0L, result?.TotalDanmaku ?? 0);
} }
public Task<int> CountPendingUploadAsync(CancellationToken cancellationToken = default) =>
_dbContext.RecordResults.CountAsync(item => item.UploadStatus == RecordArtifactUploadStatus.NotUploaded, cancellationToken);
public Task<long> SumPendingUploadBytesAsync(CancellationToken cancellationToken = default) =>
_dbContext.RecordResults
.Where(item => item.UploadStatus == RecordArtifactUploadStatus.NotUploaded)
.SumAsync(item => item.FileSizeBytes ?? 0L, cancellationToken);
public Task AddAsync(RecordResult recordResult, CancellationToken cancellationToken = default) => public Task AddAsync(RecordResult recordResult, CancellationToken cancellationToken = default) =>
_dbContext.RecordResults.AddAsync(recordResult, cancellationToken).AsTask(); _dbContext.RecordResults.AddAsync(recordResult, cancellationToken).AsTask();
@@ -75,7 +75,8 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
var settings = await settingsService.GetAsync(stoppingToken); var settings = await settingsService.GetAsync(stoppingToken);
var ffmpegService = scope.ServiceProvider.GetRequiredService<IFfmpegService>(); var ffmpegService = scope.ServiceProvider.GetRequiredService<IFfmpegService>();
var storageGuardService = scope.ServiceProvider.GetRequiredService<IStorageGuardService>(); var storageGuardService = scope.ServiceProvider.GetRequiredService<IStorageGuardService>();
if (storageGuardService.CheckCanStartOrResume(settings).HasEnoughSpace) var storageCheck = storageGuardService.CheckCanStartOrResume(settings);
if (storageCheck.Tier != StorageTier.Red)
{ {
await ffmpegService.ResumePausedFinalizationsAsync(stoppingToken); await ffmpegService.ResumePausedFinalizationsAsync(stoppingToken);
} }
@@ -382,13 +383,13 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
} }
var startCheck = storageGuardService.CheckCanStartOrResume(settings); var startCheck = storageGuardService.CheckCanStartOrResume(settings);
if (!startCheck.HasEnoughSpace) if (!startCheck.CanStartNewRecording)
{ {
await UpdateAutoStartDecisionAsync( await UpdateAutoStartDecisionAsync(
dbContext, dbContext,
liveRoom, liveRoom,
AutoStartDecisionCodes.SkippedStorage, AutoStartDecisionCodes.SkippedStorage,
"Auto-start skipped because storage is below threshold.", $"Auto-start skipped because storage tier is {startCheck.Tier}.",
startCheck.Message, startCheck.Message,
cancellationToken); cancellationToken);
await logService.WriteAsync( await logService.WriteAsync(
@@ -50,7 +50,9 @@ public sealed class RecoveryService
CheckedPath = storage.CheckedPath, CheckedPath = storage.CheckedPath,
AvailableBytes = storage.AvailableBytes, AvailableBytes = storage.AvailableBytes,
RequiredBytes = storage.RequiredBytes, RequiredBytes = storage.RequiredBytes,
Message = storage.Message Message = storage.Message,
Tier = storage.Tier.ToString(),
UsagePercent = storage.UsagePercent
}, },
LiveRooms = liveRooms, LiveRooms = liveRooms,
Finalizations = finalizations Finalizations = finalizations
@@ -7,6 +7,8 @@ namespace LiveRecorder.Infrastructure.Services;
public sealed class StorageGuardService : IStorageGuardService public sealed class StorageGuardService : IStorageGuardService
{ {
private const long Megabyte = 1024L * 1024L; private const long Megabyte = 1024L * 1024L;
private const double GreenThresholdPercent = 30.0;
private const double YellowThresholdPercent = 10.0;
private readonly ILogger<StorageGuardService> _logger; private readonly ILogger<StorageGuardService> _logger;
public StorageGuardService(ILogger<StorageGuardService> logger) public StorageGuardService(ILogger<StorageGuardService> logger)
@@ -24,7 +26,11 @@ public sealed class StorageGuardService : IStorageGuardService
{ {
if (!settings.EnableStorageGuard) if (!settings.EnableStorageGuard)
{ {
return new StorageGuardResult(false, true, ResolveOutputRoot(settings.OutputRoot), long.MaxValue, 0, "Storage guard is disabled."); return new StorageGuardResult(false, true, ResolveOutputRoot(settings.OutputRoot), long.MaxValue, 0, "Storage guard is disabled.")
{
Tier = StorageTier.Green,
UsagePercent = 0
};
} }
var checkedPath = ResolveOutputRoot(settings.OutputRoot); var checkedPath = ResolveOutputRoot(settings.OutputRoot);
@@ -35,17 +41,51 @@ public sealed class StorageGuardService : IStorageGuardService
{ {
var drive = new DriveInfo(Path.GetPathRoot(checkedPath) ?? checkedPath); var drive = new DriveInfo(Path.GetPathRoot(checkedPath) ?? checkedPath);
var availableBytes = drive.AvailableFreeSpace; var availableBytes = drive.AvailableFreeSpace;
var hasEnoughSpace = availableBytes >= requiredBytes; var totalBytes = drive.TotalSize;
var message = hasEnoughSpace var usedBytes = Math.Max(0, totalBytes - availableBytes);
? $"Storage is available. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}" var usagePercent = totalBytes > 0 ? (double)usedBytes / totalBytes * 100.0 : 0;
: $"Storage is below threshold. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}"; var freePercent = 100.0 - usagePercent;
return new StorageGuardResult(true, hasEnoughSpace, checkedPath, availableBytes, requiredBytes, message); // Determine tier
StorageTier tier;
if (freePercent >= GreenThresholdPercent)
{
tier = StorageTier.Green;
}
else if (freePercent >= YellowThresholdPercent)
{
tier = StorageTier.Yellow;
}
else
{
tier = StorageTier.Red;
}
var hasEnoughSpace = availableBytes >= requiredBytes;
var message = tier switch
{
StorageTier.Green => $"Storage is healthy. free={FormatBytes(availableBytes)} ({freePercent:F1}%), required={FormatBytes(requiredBytes)}, path={checkedPath}",
StorageTier.Yellow => $"Storage is low. free={FormatBytes(availableBytes)} ({freePercent:F1}%), required={FormatBytes(requiredBytes)}, path={checkedPath}. New recordings paused, existing recordings continue.",
StorageTier.Red => $"Storage is critically low. free={FormatBytes(availableBytes)} ({freePercent:F1}%), required={FormatBytes(requiredBytes)}, path={checkedPath}. All recordings paused, uploads continue.",
_ => hasEnoughSpace
? $"Storage is available. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}"
: $"Storage is below threshold. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}"
};
return new StorageGuardResult(true, hasEnoughSpace, checkedPath, availableBytes, requiredBytes, message)
{
Tier = tier,
UsagePercent = Math.Round(usagePercent, 1)
};
} }
catch (Exception ex) catch (Exception ex)
{ {
_logger.LogWarning(ex, "Storage guard failed to inspect output root {OutputRoot}", checkedPath); _logger.LogWarning(ex, "Storage guard failed to inspect output root {OutputRoot}", checkedPath);
return new StorageGuardResult(true, false, checkedPath, 0, requiredBytes, $"Unable to inspect storage path {checkedPath}: {ex.Message}"); return new StorageGuardResult(true, false, checkedPath, 0, requiredBytes, $"Unable to inspect storage path {checkedPath}: {ex.Message}")
{
Tier = StorageTier.Red,
UsagePercent = 0
};
} }
} }
@@ -77,4 +117,3 @@ public sealed class StorageGuardService : IStorageGuardService
return $"{display:0.##} {units[unitIndex]}"; return $"{display:0.##} {units[unitIndex]}";
} }
} }