nanxun/live_recorder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add tiered storage guard (Green/Yellow/Red) and dashboard queue monitor

Browse Source 
Storage Tier System:
- Add StorageTier enum (Green >30% / Yellow 10-30% / Red <10%)
- Extend StorageGuardResult with Tier, CanStartNewRecording, ShouldPauseActive, UsagePercent
- Yellow tier: deny new recordings but allow existing to finish and upload
- Red tier: deny new recordings and pause active sessions
- Auto-recovery: when disk frees up, polling automatically resumes new recordings
- Update LiveRoomPollingBackgroundService to use tier-based checks
- Expose tier + usage percent in Recovery API

Dashboard Queue Monitor:
- Add pending transcode count, pending upload count, queued data volume to dashboard
- Add storage tier badge (Green/Yellow/Red) with usage percentage
- Add queue monitoring card to dashboard view

Co-Authored-By: Claude Opus 4.8 noreply@anthropic.com
This commit is contained in:
nanxun
2026-06-04 23:13:26 +08:00
co-authored by Claude Opus 4.8 noreply@anthropic.com
parent cbee29bef9
commit 24e5cf2a06
12 changed files with 203 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src/types.ts
+5
View File
@@ -823,12 +823,17 @@ export interface DashboardData {
storageStatus: DashboardStorageStatus;
recentSessions: DashboardRecentSession[];
topRooms: DashboardTopRoom[];
pendingTranscodeCount: number;
pendingUploadCount: number;
queuedDataBytes: number;
}
export interface DashboardStorageStatus {
hasEnoughSpace: boolean;
message: string;
availableBytes: number;
tier: string;
usagePercent: number;
}
export interface DashboardRecentSession {
frontend/src/views/DashboardView.vue
+51 -3
View File
@@ -39,6 +39,20 @@ function sessionStatusTagType(status: number) {
return "warning";
}
function storageTierTagType(): "success" | "warning" | "danger" {
const tier = data.value?.storageStatus.tier;
if (tier === "Green") return "success";
if (tier === "Yellow") return "warning";
return "danger";
}
function storageTierLabel(): string {
const tier = data.value?.storageStatus.tier;
if (tier === "Green") return "正常";
if (tier === "Yellow") return "警告";
return "紧急";
}
async function loadData() {
loading.value = true;
loadError.value = "";
@@ -109,11 +123,15 @@ onMounted(loadData);
<p class="section-subtitle">录制输出路径的磁盘剩余空间和当前录制保护阈值。</p>
<div class="storage-bar-row">
<div class="storage-bar-card">
<span class="storage-bar-card__label">状态</span>
<el-tag :type="data.storageStatus.hasEnoughSpace ? 'success' : 'danger'">
{{ data.storageStatus.hasEnoughSpace ? "充足" : "不足" }}
<span class="storage-bar-card__label">水位线</span>
<el-tag :type="storageTierTagType()" size="large">
{{ storageTierLabel() }}
</el-tag>
</div>
<div class="storage-bar-card">
<span class="storage-bar-card__label">使用率</span>
<span class="storage-bar-card__value">{{ data.storageStatus.usagePercent.toFixed(1) }}%</span>
</div>
<div class="storage-bar-card">
<span class="storage-bar-card__label">可用空间</span>
<span class="storage-bar-card__value">{{ formatDataSize(data.storageStatus.availableBytes) }}</span>
@@ -125,6 +143,36 @@ onMounted(loadData);
</div>
</el-card>
<!-- Queue Status -->
<el-row :gutter="18">
<el-col :lg="12" :sm="24">
<el-card class="surface-card" shadow="never">
<h3 class="section-title">处理队列</h3>
<p class="section-subtitle">待转码和待上传的文件积压情况。</p>
<div class="storage-bar-row">
<div class="storage-bar-card">
<span class="storage-bar-card__label">待转码</span>
<span class="storage-bar-card__value" :style="{ color: data.pendingTranscodeCount > 0 ? 'var(--warning)' : 'var(--text-primary)' }">
{{ data.pendingTranscodeCount }}
</span>
</div>
<div class="storage-bar-card">
<span class="storage-bar-card__label">待上传</span>
<span class="storage-bar-card__value" :style="{ color: data.pendingUploadCount > 0 ? 'var(--warning)' : 'var(--text-primary)' }">
{{ data.pendingUploadCount }}
</span>
</div>
<div class="storage-bar-card">
<span class="storage-bar-card__label">积压数据量</span>
<span class="storage-bar-card__value" :style="{ color: data.queuedDataBytes > 0 ? 'var(--warning)' : 'var(--text-primary)' }">
{{ formatDataSize(data.queuedDataBytes) }}
</span>
</div>
</div>
</el-card>
</el-col>
</el-row>
<!-- Recent Sessions + Top Rooms -->
<el-row :gutter="18">
<el-col :lg="12" :sm="24">
src/LiveRecorder.Application/Abstractions/Persistence/PersistenceContracts.cs
+6
View File
@@ -48,6 +48,8 @@ public interface IRecordTaskRepository
Task<double> SumDurationSecondsAsync(DateTimeOffset startedFrom, DateTimeOffset startedTo, CancellationToken cancellationToken = default);
Task<int> CountByStatusAsync(RecordTaskStatus status, CancellationToken cancellationToken = default);
Task AddAsync(RecordTask recordTask, CancellationToken cancellationToken = default);
void Remove(RecordTask recordTask);
@@ -84,6 +86,10 @@ public interface IRecordResultRepository
Task<(long TotalBytes, int TotalDanmaku)> GetTodayAggregateAsync(DateTimeOffset createdFrom, DateTimeOffset createdTo, CancellationToken cancellationToken = default);
Task<int> CountPendingUploadAsync(CancellationToken cancellationToken = default);
Task<long> SumPendingUploadBytesAsync(CancellationToken cancellationToken = default);
Task AddAsync(RecordResult recordResult, CancellationToken cancellationToken = default);
void Update(RecordResult recordResult);
src/LiveRecorder.Application/Abstractions/Storage/IStorageGuardService.cs
+33 -1
View File
@@ -2,6 +2,18 @@ using LiveRecorder.Application.Models.Settings;
namespace LiveRecorder.Application.Abstractions.Storage;
public enum StorageTier
{
/// <summary>Disk has sufficient free space for normal operation.</summary>
Green = 0,
/// <summary>Disk space is low. Deny new recordings but allow existing to finish.</summary>
Yellow = 1,
/// <summary>Disk space is critically low. Deny new recordings and pause active ones.</summary>
Red = 2
}
public interface IStorageGuardService
{
StorageGuardResult CheckCanStartOrResume(SystemSettingsDto settings, long additionalRequiredBytes = 0);
@@ -15,5 +27,25 @@ public sealed record StorageGuardResult(
string CheckedPath,
long AvailableBytes,
long RequiredBytes,
string Message);
string Message)
{
/// <summary>
/// Current storage tier (Green/Yellow/Red).
/// </summary>
public StorageTier Tier { get; init; }
/// <summary>
/// Disk usage percentage (0-100). Only populated when IsEnabled is true.
/// </summary>
public double UsagePercent { get; init; }
/// <summary>
/// True if new recordings can be started. False in Yellow and Red tiers.
/// </summary>
public bool CanStartNewRecording => HasEnoughSpace || Tier == StorageTier.Green;
/// <summary>
/// True if active recordings should be paused. Only true in Red tier.
/// </summary>
public bool ShouldPauseActive => Tier == StorageTier.Red;
}
src/LiveRecorder.Application/Models/Recovery/RecoveryModels.cs
+6
View File
@@ -24,6 +24,12 @@ public sealed class StorageGuardStatusDto
public long RequiredBytes { get; init; }
public required string Message { get; init; }
/// <summary>Storage tier: Green, Yellow, or Red.</summary>
public string Tier { get; init; } = "Green";
/// <summary>Disk usage percentage (0-100).</summary>
public double UsagePercent { get; init; }
}
public sealed class RecoverableLiveRoomDto
src/LiveRecorder.Application/Models/Reports/DashboardModels.cs
+17
View File
@@ -64,6 +64,21 @@ public sealed class DashboardDto
/// Top live rooms by recording duration today (up to 5).
/// </summary>
public required IReadOnlyList<TopRoomItemDto> TopRooms { get; init; }
/// <summary>
/// Number of recording tasks currently in Processing (transcoding) status.
/// </summary>
public int PendingTranscodeCount { get; init; }
/// <summary>
/// Number of recording results with NotUploaded status where local file still exists.
/// </summary>
public int PendingUploadCount { get; init; }
/// <summary>
/// Total file size in bytes of files awaiting upload.
/// </summary>
public long QueuedDataBytes { get; init; }
}
public sealed class StorageStatusDto
@@ -71,6 +86,8 @@ public sealed class StorageStatusDto
public bool HasEnoughSpace { get; init; }
public string Message { get; init; } = string.Empty;
public long AvailableBytes { get; init; }
public string Tier { get; init; } = "Green";
public double UsagePercent { get; init; }
}
public sealed class RecentSessionItemDto
src/LiveRecorder.Application/Services/DashboardService.cs
+6 -1
View File
@@ -90,8 +90,13 @@ public sealed class DashboardService
{
HasEnoughSpace = storageCheck.HasEnoughSpace,
Message = storageCheck.Message ?? string.Empty,
AvailableBytes = storageCheck.AvailableBytes
AvailableBytes = storageCheck.AvailableBytes,
Tier = storageCheck.Tier.ToString(),
UsagePercent = storageCheck.UsagePercent
},
PendingTranscodeCount = await _recordTaskRepository.CountByStatusAsync(Domain.Enums.RecordTaskStatus.Processing, cancellationToken),
PendingUploadCount = await _recordResultRepository.CountPendingUploadAsync(cancellationToken),
QueuedDataBytes = await _recordResultRepository.SumPendingUploadBytesAsync(cancellationToken),
RecentSessions = recentSessionsTask.Result
.Select(MapRecentSession)
.ToList(),
src/LiveRecorder.Application/Services/RecordService.cs
+14 -5
View File
@@ -207,8 +207,17 @@ public sealed class RecordService
var settings = await _systemSettingsService.GetAsync(cancellationToken);
var storageCheck = _storageGuardService.CheckCanStartOrResume(settings);
if (!storageCheck.HasEnoughSpace)
if (!storageCheck.CanStartNewRecording)
{
var storageMessage = storageCheck.Tier switch
{
Application.Abstractions.Storage.StorageTier.Yellow =>
"Storage is in warning state. New recordings are paused but existing recordings continue. Transcoding and uploading will free up space.",
Application.Abstractions.Storage.StorageTier.Red =>
"Storage is critically low. All recordings are paused until enough disk space is freed.",
_ => storageCheck.Message
};
await _systemLogService.WriteAsync(
SystemLogLevel.Warning,
"Storage",
@@ -222,12 +231,12 @@ public sealed class RecordService
await UpdateAutoStartDecisionAsync(
liveRoom,
AutoStartDecisionCodes.SkippedStorage,
"Auto-start skipped because storage is below threshold.",
$"Auto-start skipped because storage tier is {storageCheck.Tier}.",
storageCheck.Message,
cancellationToken);
}
throw new InvalidOperationException(storageCheck.Message);
throw new InvalidOperationException(storageMessage);
}
var effectiveSettings = _liveRoomRecordingSettingsResolver.Resolve(liveRoom, settings);
@@ -661,14 +670,14 @@ public sealed class RecordService
}
var storageCheck = _storageGuardService.CheckCanStartOrResume(settings);
if (!storageCheck.HasEnoughSpace)
if (!storageCheck.CanStartNewRecording)
{
foreach (var liveRoomId in liveRoomIds)
{
await TryUpdateAutoStartDecisionAsync(
liveRoomId,
AutoStartDecisionCodes.SkippedStorage,
"Auto-start skipped because storage is below threshold.",
$"Auto-start skipped because storage tier is {storageCheck.Tier}.",
storageCheck.Message,
cancellationToken);
await _systemLogService.WriteAsync(
src/LiveRecorder.Infrastructure/Persistence/Repositories/Repositories.cs
+11
View File
@@ -142,6 +142,9 @@ public sealed class RecordTaskRepository : IRecordTaskRepository
.Where(item => item.RecordSession != null && item.RecordSession.StartedAt >= startedFrom && item.RecordSession.StartedAt <= startedTo)
.SumAsync(item => item.DurationSeconds ?? 0, cancellationToken);
public Task<int> CountByStatusAsync(RecordTaskStatus status, CancellationToken cancellationToken = default) =>
_dbContext.RecordTasks.CountAsync(item => item.Status == status, cancellationToken);
public Task AddAsync(RecordTask recordTask, CancellationToken cancellationToken = default) =>
_dbContext.RecordTasks.AddAsync(recordTask, cancellationToken).AsTask();
@@ -266,6 +269,14 @@ public sealed class RecordResultRepository : IRecordResultRepository
return (result?.TotalBytes ?? 0L, result?.TotalDanmaku ?? 0);
}
public Task<int> CountPendingUploadAsync(CancellationToken cancellationToken = default) =>
_dbContext.RecordResults.CountAsync(item => item.UploadStatus == RecordArtifactUploadStatus.NotUploaded, cancellationToken);
public Task<long> SumPendingUploadBytesAsync(CancellationToken cancellationToken = default) =>
_dbContext.RecordResults
.Where(item => item.UploadStatus == RecordArtifactUploadStatus.NotUploaded)
.SumAsync(item => item.FileSizeBytes ?? 0L, cancellationToken);
public Task AddAsync(RecordResult recordResult, CancellationToken cancellationToken = default) =>
_dbContext.RecordResults.AddAsync(recordResult, cancellationToken).AsTask();
src/LiveRecorder.Infrastructure/Services/LiveRoomPollingBackgroundService.cs
+4 -3
View File
@@ -75,7 +75,8 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
var settings = await settingsService.GetAsync(stoppingToken);
var ffmpegService = scope.ServiceProvider.GetRequiredService<IFfmpegService>();
var storageGuardService = scope.ServiceProvider.GetRequiredService<IStorageGuardService>();
if (storageGuardService.CheckCanStartOrResume(settings).HasEnoughSpace)
var storageCheck = storageGuardService.CheckCanStartOrResume(settings);
if (storageCheck.Tier != StorageTier.Red)
{
await ffmpegService.ResumePausedFinalizationsAsync(stoppingToken);
}
@@ -382,13 +383,13 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
}
var startCheck = storageGuardService.CheckCanStartOrResume(settings);
if (!startCheck.HasEnoughSpace)
if (!startCheck.CanStartNewRecording)
{
await UpdateAutoStartDecisionAsync(
dbContext,
liveRoom,
AutoStartDecisionCodes.SkippedStorage,
"Auto-start skipped because storage is below threshold.",
$"Auto-start skipped because storage tier is {startCheck.Tier}.",
startCheck.Message,
cancellationToken);
await logService.WriteAsync(
src/LiveRecorder.Infrastructure/Services/RecoveryService.cs
+3 -1
View File
@@ -50,7 +50,9 @@ public sealed class RecoveryService
CheckedPath = storage.CheckedPath,
AvailableBytes = storage.AvailableBytes,
RequiredBytes = storage.RequiredBytes,
Message = storage.Message
Message = storage.Message,
Tier = storage.Tier.ToString(),
UsagePercent = storage.UsagePercent
},
LiveRooms = liveRooms,
Finalizations = finalizations
src/LiveRecorder.Infrastructure/Services/StorageGuardService.cs
+47 -8
View File
@@ -7,6 +7,8 @@ namespace LiveRecorder.Infrastructure.Services;
public sealed class StorageGuardService : IStorageGuardService
{
private const long Megabyte = 1024L * 1024L;
private const double GreenThresholdPercent = 30.0;
private const double YellowThresholdPercent = 10.0;
private readonly ILogger<StorageGuardService> _logger;
public StorageGuardService(ILogger<StorageGuardService> logger)
@@ -24,7 +26,11 @@ public sealed class StorageGuardService : IStorageGuardService
{
if (!settings.EnableStorageGuard)
{
return new StorageGuardResult(false, true, ResolveOutputRoot(settings.OutputRoot), long.MaxValue, 0, "Storage guard is disabled.");
return new StorageGuardResult(false, true, ResolveOutputRoot(settings.OutputRoot), long.MaxValue, 0, "Storage guard is disabled.")
{
Tier = StorageTier.Green,
UsagePercent = 0
};
}
var checkedPath = ResolveOutputRoot(settings.OutputRoot);
@@ -35,17 +41,51 @@ public sealed class StorageGuardService : IStorageGuardService
{
var drive = new DriveInfo(Path.GetPathRoot(checkedPath) ?? checkedPath);
var availableBytes = drive.AvailableFreeSpace;
var hasEnoughSpace = availableBytes >= requiredBytes;
var message = hasEnoughSpace
? $"Storage is available. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}"
: $"Storage is below threshold. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}";
var totalBytes = drive.TotalSize;
var usedBytes = Math.Max(0, totalBytes - availableBytes);
var usagePercent = totalBytes > 0 ? (double)usedBytes / totalBytes * 100.0 : 0;
var freePercent = 100.0 - usagePercent;
return new StorageGuardResult(true, hasEnoughSpace, checkedPath, availableBytes, requiredBytes, message);
// Determine tier
StorageTier tier;
if (freePercent >= GreenThresholdPercent)
{
tier = StorageTier.Green;
}
else if (freePercent >= YellowThresholdPercent)
{
tier = StorageTier.Yellow;
}
else
{
tier = StorageTier.Red;
}
var hasEnoughSpace = availableBytes >= requiredBytes;
var message = tier switch
{
StorageTier.Green => $"Storage is healthy. free={FormatBytes(availableBytes)} ({freePercent:F1}%), required={FormatBytes(requiredBytes)}, path={checkedPath}",
StorageTier.Yellow => $"Storage is low. free={FormatBytes(availableBytes)} ({freePercent:F1}%), required={FormatBytes(requiredBytes)}, path={checkedPath}. New recordings paused, existing recordings continue.",
StorageTier.Red => $"Storage is critically low. free={FormatBytes(availableBytes)} ({freePercent:F1}%), required={FormatBytes(requiredBytes)}, path={checkedPath}. All recordings paused, uploads continue.",
_ => hasEnoughSpace
? $"Storage is available. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}"
: $"Storage is below threshold. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}"
};
return new StorageGuardResult(true, hasEnoughSpace, checkedPath, availableBytes, requiredBytes, message)
{
Tier = tier,
UsagePercent = Math.Round(usagePercent, 1)
};
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Storage guard failed to inspect output root {OutputRoot}", checkedPath);
return new StorageGuardResult(true, false, checkedPath, 0, requiredBytes, $"Unable to inspect storage path {checkedPath}: {ex.Message}");
return new StorageGuardResult(true, false, checkedPath, 0, requiredBytes, $"Unable to inspect storage path {checkedPath}: {ex.Message}")
{
Tier = StorageTier.Red,
UsagePercent = 0
};
}
}
@@ -77,4 +117,3 @@ public sealed class StorageGuardService : IStorageGuardService
return $"{display:0.##} {units[unitIndex]}";
}
}