fix: harden recording recovery and dashboard metrics
This commit is contained in:
@@ -5,6 +5,7 @@ interface StorageCapacityStatus {
|
|||||||
isEnabled: boolean;
|
isEnabled: boolean;
|
||||||
isAvailable: boolean;
|
isAvailable: boolean;
|
||||||
checkedPath: string;
|
checkedPath: string;
|
||||||
|
volumeRoot: string;
|
||||||
totalBytes: number;
|
totalBytes: number;
|
||||||
usedBytes: number;
|
usedBytes: number;
|
||||||
availableBytes: number;
|
availableBytes: number;
|
||||||
@@ -83,6 +84,9 @@ function formatBytes(bytes: number) {
|
|||||||
<el-tag v-else :type="tagType" effect="light">{{ statusLabel }}</el-tag>
|
<el-tag v-else :type="tagType" effect="light">{{ statusLabel }}</el-tag>
|
||||||
</div>
|
</div>
|
||||||
<p class="storage-capacity__path" :title="status.checkedPath">{{ status.checkedPath || "未配置输出路径" }}</p>
|
<p class="storage-capacity__path" :title="status.checkedPath">{{ status.checkedPath || "未配置输出路径" }}</p>
|
||||||
|
<p class="storage-capacity__volume" :title="status.volumeRoot">
|
||||||
|
检测卷:{{ status.volumeRoot || "--" }}
|
||||||
|
</p>
|
||||||
|
|
||||||
<dl class="storage-capacity__metrics">
|
<dl class="storage-capacity__metrics">
|
||||||
<div><dt>已使用</dt><dd>{{ status.isAvailable ? formatBytes(status.usedBytes) : "--" }}</dd></div>
|
<div><dt>已使用</dt><dd>{{ status.isAvailable ? formatBytes(status.usedBytes) : "--" }}</dd></div>
|
||||||
@@ -103,6 +107,7 @@ function formatBytes(bytes: number) {
|
|||||||
.storage-capacity__summary { display: flex; align-items: center; justify-content: space-between; gap: 12px; }
|
.storage-capacity__summary { display: flex; align-items: center; justify-content: space-between; gap: 12px; }
|
||||||
.storage-capacity__summary > strong { min-width: 0; color: var(--text-primary); font-size: 13px; font-variant-numeric: tabular-nums; }
|
.storage-capacity__summary > strong { min-width: 0; color: var(--text-primary); font-size: 13px; font-variant-numeric: tabular-nums; }
|
||||||
.storage-capacity__path { margin: 7px 0 0; overflow: hidden; color: var(--text-secondary); font-family: var(--font-mono); font-size: 10px; text-overflow: ellipsis; white-space: nowrap; }
|
.storage-capacity__path { margin: 7px 0 0; overflow: hidden; color: var(--text-secondary); font-family: var(--font-mono); font-size: 10px; text-overflow: ellipsis; white-space: nowrap; }
|
||||||
|
.storage-capacity__volume { margin: 3px 0 0; overflow: hidden; color: var(--text-muted); font-family: var(--font-mono); font-size: 9px; text-overflow: ellipsis; white-space: nowrap; }
|
||||||
.storage-capacity__metrics { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 8px; margin: 13px 0 0; }
|
.storage-capacity__metrics { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 8px; margin: 13px 0 0; }
|
||||||
.storage-capacity__metrics > div { min-width: 0; padding: 9px; border-radius: var(--radius-sm); background: var(--surface-muted); }
|
.storage-capacity__metrics > div { min-width: 0; padding: 9px; border-radius: var(--radius-sm); background: var(--surface-muted); }
|
||||||
.storage-capacity__metrics dt { color: var(--text-muted); font-size: 9px; }
|
.storage-capacity__metrics dt { color: var(--text-muted); font-size: 9px; }
|
||||||
|
|||||||
@@ -703,6 +703,7 @@ export interface StorageGuardStatus {
|
|||||||
isAvailable: boolean;
|
isAvailable: boolean;
|
||||||
hasEnoughSpace: boolean;
|
hasEnoughSpace: boolean;
|
||||||
checkedPath: string;
|
checkedPath: string;
|
||||||
|
volumeRoot: string;
|
||||||
totalBytes: number;
|
totalBytes: number;
|
||||||
usedBytes: number;
|
usedBytes: number;
|
||||||
availableBytes: number;
|
availableBytes: number;
|
||||||
@@ -1011,6 +1012,7 @@ export interface DashboardStorageStatus {
|
|||||||
hasEnoughSpace: boolean;
|
hasEnoughSpace: boolean;
|
||||||
message: string;
|
message: string;
|
||||||
checkedPath: string;
|
checkedPath: string;
|
||||||
|
volumeRoot: string;
|
||||||
totalBytes: number;
|
totalBytes: number;
|
||||||
usedBytes: number;
|
usedBytes: number;
|
||||||
availableBytes: number;
|
availableBytes: number;
|
||||||
|
|||||||
@@ -116,6 +116,8 @@ public interface IRecordResultRepository
|
|||||||
|
|
||||||
Task<long> SumPendingUploadBytesAsync(CancellationToken cancellationToken = default);
|
Task<long> SumPendingUploadBytesAsync(CancellationToken cancellationToken = default);
|
||||||
|
|
||||||
|
Task<PendingUploadMetrics> GetPendingUploadMetricsAsync(CancellationToken cancellationToken = default);
|
||||||
|
|
||||||
Task<int> CountFailedArtifactAsync(CancellationToken cancellationToken = default);
|
Task<int> CountFailedArtifactAsync(CancellationToken cancellationToken = default);
|
||||||
|
|
||||||
Task<List<(RecordResult Result, RecordTask Task)>> ListUploadStatusAsync(
|
Task<List<(RecordResult Result, RecordTask Task)>> ListUploadStatusAsync(
|
||||||
@@ -202,6 +204,8 @@ public interface IUserSessionRepository
|
|||||||
void Update(UserSession session);
|
void Update(UserSession session);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public sealed record PendingUploadMetrics(int Count, long TotalBytes);
|
||||||
|
|
||||||
public interface IUnitOfWork
|
public interface IUnitOfWork
|
||||||
{
|
{
|
||||||
Task<int> SaveChangesAsync(CancellationToken cancellationToken = default);
|
Task<int> SaveChangesAsync(CancellationToken cancellationToken = default);
|
||||||
|
|||||||
@@ -37,6 +37,8 @@ public interface IFfmpegService
|
|||||||
|
|
||||||
Task<int> RecoverOrphanedTerminalSessionTasksAsync(CancellationToken cancellationToken = default);
|
Task<int> RecoverOrphanedTerminalSessionTasksAsync(CancellationToken cancellationToken = default);
|
||||||
|
|
||||||
|
Task<int> RecoverStalePendingSessionsAsync(CancellationToken cancellationToken = default);
|
||||||
|
|
||||||
Task<bool> TryRecoverOrphanedTerminalTaskAsync(Guid recordTaskId, CancellationToken cancellationToken = default);
|
Task<bool> TryRecoverOrphanedTerminalTaskAsync(Guid recordTaskId, CancellationToken cancellationToken = default);
|
||||||
|
|
||||||
Task<bool> StartManualFinalizeTaskAsync(Guid recordTaskId, CancellationToken cancellationToken = default);
|
Task<bool> StartManualFinalizeTaskAsync(Guid recordTaskId, CancellationToken cancellationToken = default);
|
||||||
|
|||||||
@@ -23,6 +23,16 @@ public interface IStorageGuardService
|
|||||||
StorageGuardResult CheckCanFinalize(SystemSettingsDto settings, long estimatedTemporaryBytes);
|
StorageGuardResult CheckCanFinalize(SystemSettingsDto settings, long estimatedTemporaryBytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public interface IStorageCapacityProvider
|
||||||
|
{
|
||||||
|
StorageVolumeCapacity GetCapacity(string outputPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed record StorageVolumeCapacity(
|
||||||
|
string VolumeRoot,
|
||||||
|
long TotalBytes,
|
||||||
|
long AvailableBytes);
|
||||||
|
|
||||||
public sealed record StorageGuardResult(
|
public sealed record StorageGuardResult(
|
||||||
bool IsEnabled,
|
bool IsEnabled,
|
||||||
bool HasEnoughSpace,
|
bool HasEnoughSpace,
|
||||||
@@ -33,6 +43,8 @@ public sealed record StorageGuardResult(
|
|||||||
{
|
{
|
||||||
public bool IsAvailable { get; init; }
|
public bool IsAvailable { get; init; }
|
||||||
|
|
||||||
|
public string VolumeRoot { get; init; } = string.Empty;
|
||||||
|
|
||||||
public long TotalBytes { get; init; }
|
public long TotalBytes { get; init; }
|
||||||
|
|
||||||
public long UsedBytes { get; init; }
|
public long UsedBytes { get; init; }
|
||||||
|
|||||||
@@ -36,6 +36,8 @@ public sealed class StorageGuardStatusDto
|
|||||||
|
|
||||||
public required string CheckedPath { get; init; }
|
public required string CheckedPath { get; init; }
|
||||||
|
|
||||||
|
public string VolumeRoot { get; init; } = string.Empty;
|
||||||
|
|
||||||
public long AvailableBytes { get; init; }
|
public long AvailableBytes { get; init; }
|
||||||
|
|
||||||
public long TotalBytes { get; init; }
|
public long TotalBytes { get; init; }
|
||||||
|
|||||||
@@ -101,6 +101,7 @@ public sealed class StorageStatusDto
|
|||||||
public bool HasEnoughSpace { get; init; }
|
public bool HasEnoughSpace { get; init; }
|
||||||
public string Message { get; init; } = string.Empty;
|
public string Message { get; init; } = string.Empty;
|
||||||
public string CheckedPath { get; init; } = string.Empty;
|
public string CheckedPath { get; init; } = string.Empty;
|
||||||
|
public string VolumeRoot { get; init; } = string.Empty;
|
||||||
public long TotalBytes { get; init; }
|
public long TotalBytes { get; init; }
|
||||||
public long UsedBytes { get; init; }
|
public long UsedBytes { get; init; }
|
||||||
public long AvailableBytes { get; init; }
|
public long AvailableBytes { get; init; }
|
||||||
|
|||||||
@@ -65,8 +65,7 @@ public sealed class DashboardService
|
|||||||
var settings = await _systemSettingsService.GetAsync(cancellationToken);
|
var settings = await _systemSettingsService.GetAsync(cancellationToken);
|
||||||
var storageCheck = _storageGuardService.CheckCanStartOrResume(settings);
|
var storageCheck = _storageGuardService.CheckCanStartOrResume(settings);
|
||||||
var pendingTranscodeCount = await _recordTaskRepository.CountByStatusAsync(RecordTaskStatus.Processing, cancellationToken);
|
var pendingTranscodeCount = await _recordTaskRepository.CountByStatusAsync(RecordTaskStatus.Processing, cancellationToken);
|
||||||
var pendingUploadCount = await _recordResultRepository.CountPendingUploadAsync(cancellationToken);
|
var pendingUploadMetrics = await _recordResultRepository.GetPendingUploadMetricsAsync(cancellationToken);
|
||||||
var queuedDataBytes = await _recordResultRepository.SumPendingUploadBytesAsync(cancellationToken);
|
|
||||||
var operationsMetrics = await _operationsMetricsRepository.GetAsync(cancellationToken);
|
var operationsMetrics = await _operationsMetricsRepository.GetAsync(cancellationToken);
|
||||||
|
|
||||||
return new DashboardDto
|
return new DashboardDto
|
||||||
@@ -88,6 +87,7 @@ public sealed class DashboardService
|
|||||||
HasEnoughSpace = storageCheck.HasEnoughSpace,
|
HasEnoughSpace = storageCheck.HasEnoughSpace,
|
||||||
Message = storageCheck.Message ?? string.Empty,
|
Message = storageCheck.Message ?? string.Empty,
|
||||||
CheckedPath = storageCheck.CheckedPath,
|
CheckedPath = storageCheck.CheckedPath,
|
||||||
|
VolumeRoot = storageCheck.VolumeRoot,
|
||||||
TotalBytes = storageCheck.TotalBytes,
|
TotalBytes = storageCheck.TotalBytes,
|
||||||
UsedBytes = storageCheck.UsedBytes,
|
UsedBytes = storageCheck.UsedBytes,
|
||||||
AvailableBytes = storageCheck.AvailableBytes,
|
AvailableBytes = storageCheck.AvailableBytes,
|
||||||
@@ -99,8 +99,8 @@ public sealed class DashboardService
|
|||||||
RedThresholdPercent = storageCheck.RedThresholdPercent
|
RedThresholdPercent = storageCheck.RedThresholdPercent
|
||||||
},
|
},
|
||||||
PendingTranscodeCount = pendingTranscodeCount,
|
PendingTranscodeCount = pendingTranscodeCount,
|
||||||
PendingUploadCount = pendingUploadCount,
|
PendingUploadCount = pendingUploadMetrics.Count,
|
||||||
QueuedDataBytes = queuedDataBytes,
|
QueuedDataBytes = pendingUploadMetrics.TotalBytes,
|
||||||
OldestTranscodeUpdatedAt = operationsMetrics.OldestTranscodeUpdatedAt,
|
OldestTranscodeUpdatedAt = operationsMetrics.OldestTranscodeUpdatedAt,
|
||||||
OldestUploadProgressAt = operationsMetrics.OldestUploadProgressAt,
|
OldestUploadProgressAt = operationsMetrics.OldestUploadProgressAt,
|
||||||
StalledUploadCount = operationsMetrics.StalledUploadCount,
|
StalledUploadCount = operationsMetrics.StalledUploadCount,
|
||||||
|
|||||||
@@ -352,8 +352,9 @@ public sealed class RecordService
|
|||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
initialTask.MarkFailed(ex.Message, DateTimeOffset.UtcNow);
|
var failedAt = DateTimeOffset.UtcNow;
|
||||||
recordSession.MarkFailed(ex.Message, DateTimeOffset.UtcNow);
|
initialTask.MarkFailed(ex.Message, failedAt);
|
||||||
|
recordSession.MarkFailed(ex.Message, failedAt);
|
||||||
if (trackAutoStartDecision)
|
if (trackAutoStartDecision)
|
||||||
{
|
{
|
||||||
ApplyAutoStartDecision(
|
ApplyAutoStartDecision(
|
||||||
@@ -363,7 +364,8 @@ public sealed class RecordService
|
|||||||
ex.Message);
|
ex.Message);
|
||||||
}
|
}
|
||||||
|
|
||||||
await _unitOfWork.SaveChangesAsync(cancellationToken);
|
using var terminalStateCts = new CancellationTokenSource(TimeSpan.FromSeconds(10));
|
||||||
|
await _unitOfWork.SaveChangesAsync(terminalStateCts.Token);
|
||||||
|
|
||||||
await _systemLogService.WriteAsync(
|
await _systemLogService.WriteAsync(
|
||||||
SystemLogLevel.Error,
|
SystemLogLevel.Error,
|
||||||
@@ -373,7 +375,12 @@ public sealed class RecordService
|
|||||||
liveRoom.Id,
|
liveRoom.Id,
|
||||||
recordSession.Id,
|
recordSession.Id,
|
||||||
initialTask.Id,
|
initialTask.Id,
|
||||||
cancellationToken);
|
terminalStateCts.Token);
|
||||||
|
|
||||||
|
if (ex is OperationCanceledException && cancellationToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
|
||||||
await _emailNotificationService.SendExceptionAsync(
|
await _emailNotificationService.SendExceptionAsync(
|
||||||
"RecordSession",
|
"RecordSession",
|
||||||
|
|||||||
@@ -452,9 +452,9 @@ public sealed class RecordSessionRepository : IRecordSessionRepository
|
|||||||
|
|
||||||
public Task<int> CountActiveAsync(CancellationToken cancellationToken = default) =>
|
public Task<int> CountActiveAsync(CancellationToken cancellationToken = default) =>
|
||||||
_dbContext.RecordSessions.CountAsync(item =>
|
_dbContext.RecordSessions.CountAsync(item =>
|
||||||
item.Status == RecordSessionStatus.Pending ||
|
|
||||||
item.Status == RecordSessionStatus.Starting ||
|
item.Status == RecordSessionStatus.Starting ||
|
||||||
item.Status == RecordSessionStatus.Running, cancellationToken);
|
item.Status == RecordSessionStatus.Running ||
|
||||||
|
item.Status == RecordSessionStatus.Stopping, cancellationToken);
|
||||||
|
|
||||||
public async Task<IReadOnlyList<RecordSession>> ListRecentAsync(int take, CancellationToken cancellationToken = default) =>
|
public async Task<IReadOnlyList<RecordSession>> ListRecentAsync(int take, CancellationToken cancellationToken = default) =>
|
||||||
await _dbContext.RecordSessions
|
await _dbContext.RecordSessions
|
||||||
@@ -508,21 +508,49 @@ public sealed class RecordResultRepository : IRecordResultRepository
|
|||||||
return (result?.TotalBytes ?? 0L, result?.TotalDanmaku ?? 0);
|
return (result?.TotalBytes ?? 0L, result?.TotalDanmaku ?? 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
public Task<int> CountPendingUploadAsync(CancellationToken cancellationToken = default) =>
|
public async Task<int> CountPendingUploadAsync(CancellationToken cancellationToken = default) =>
|
||||||
_dbContext.RecordResults.CountAsync(
|
(await GetPendingUploadMetricsAsync(cancellationToken)).Count;
|
||||||
item => item.UploadStatus == RecordArtifactUploadStatus.NotUploaded &&
|
|
||||||
item.RecordTask != null &&
|
|
||||||
(item.RecordTask.Status == RecordTaskStatus.Completed ||
|
|
||||||
item.RecordTask.Status == RecordTaskStatus.Stopped),
|
|
||||||
cancellationToken);
|
|
||||||
|
|
||||||
public Task<long> SumPendingUploadBytesAsync(CancellationToken cancellationToken = default) =>
|
public async Task<long> SumPendingUploadBytesAsync(CancellationToken cancellationToken = default) =>
|
||||||
_dbContext.RecordResults
|
(await GetPendingUploadMetricsAsync(cancellationToken)).TotalBytes;
|
||||||
|
|
||||||
|
public async Task<PendingUploadMetrics> GetPendingUploadMetricsAsync(CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
var candidatePaths = await _dbContext.RecordResults
|
||||||
|
.AsNoTracking()
|
||||||
.Where(item => item.UploadStatus == RecordArtifactUploadStatus.NotUploaded &&
|
.Where(item => item.UploadStatus == RecordArtifactUploadStatus.NotUploaded &&
|
||||||
item.RecordTask != null &&
|
item.RecordTask != null &&
|
||||||
|
!item.RecordTask.IsHiddenArtifactSource &&
|
||||||
|
item.RecordTask.MergedIntoRecordTaskId == null &&
|
||||||
(item.RecordTask.Status == RecordTaskStatus.Completed ||
|
(item.RecordTask.Status == RecordTaskStatus.Completed ||
|
||||||
item.RecordTask.Status == RecordTaskStatus.Stopped))
|
item.RecordTask.Status == RecordTaskStatus.Stopped) &&
|
||||||
.SumAsync(item => item.FileSizeBytes ?? 0L, cancellationToken);
|
item.FilePath != null &&
|
||||||
|
item.FilePath != string.Empty)
|
||||||
|
.Select(item => item.FilePath)
|
||||||
|
.ToListAsync(cancellationToken);
|
||||||
|
|
||||||
|
var count = 0;
|
||||||
|
long totalBytes = 0;
|
||||||
|
foreach (var path in candidatePaths)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var file = new FileInfo(path);
|
||||||
|
if (!file.Exists || file.Length <= 0)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
count++;
|
||||||
|
totalBytes += file.Length;
|
||||||
|
}
|
||||||
|
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return new PendingUploadMetrics(count, totalBytes);
|
||||||
|
}
|
||||||
|
|
||||||
public Task<int> CountFailedArtifactAsync(CancellationToken cancellationToken = default) =>
|
public Task<int> CountFailedArtifactAsync(CancellationToken cancellationToken = default) =>
|
||||||
_dbContext.RecordResults.CountAsync(
|
_dbContext.RecordResults.CountAsync(
|
||||||
|
|||||||
@@ -0,0 +1,89 @@
|
|||||||
|
using LiveRecorder.Application.Abstractions.Storage;
|
||||||
|
|
||||||
|
namespace LiveRecorder.Infrastructure.Services;
|
||||||
|
|
||||||
|
public sealed class DriveInfoStorageCapacityProvider : IStorageCapacityProvider
|
||||||
|
{
|
||||||
|
public StorageVolumeCapacity GetCapacity(string outputPath)
|
||||||
|
{
|
||||||
|
var volumes = new List<StorageVolumeCapacity>();
|
||||||
|
foreach (var drive in DriveInfo.GetDrives())
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (drive.IsReady)
|
||||||
|
{
|
||||||
|
volumes.Add(new StorageVolumeCapacity(
|
||||||
|
drive.RootDirectory.FullName,
|
||||||
|
drive.TotalSize,
|
||||||
|
drive.AvailableFreeSpace));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var selected = SelectBestVolume(outputPath, volumes);
|
||||||
|
if (selected is not null)
|
||||||
|
{
|
||||||
|
return selected;
|
||||||
|
}
|
||||||
|
|
||||||
|
var fallbackRoot = Path.GetPathRoot(outputPath);
|
||||||
|
if (string.IsNullOrWhiteSpace(fallbackRoot))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"No mounted volume contains output path '{outputPath}'.");
|
||||||
|
}
|
||||||
|
|
||||||
|
var fallback = new DriveInfo(fallbackRoot);
|
||||||
|
if (!fallback.IsReady)
|
||||||
|
{
|
||||||
|
throw new IOException($"Storage volume '{fallbackRoot}' is not ready.");
|
||||||
|
}
|
||||||
|
|
||||||
|
return new StorageVolumeCapacity(
|
||||||
|
fallback.RootDirectory.FullName,
|
||||||
|
fallback.TotalSize,
|
||||||
|
fallback.AvailableFreeSpace);
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static StorageVolumeCapacity? SelectBestVolume(
|
||||||
|
string outputPath,
|
||||||
|
IReadOnlyCollection<StorageVolumeCapacity> volumes)
|
||||||
|
{
|
||||||
|
var normalizedPath = NormalizeForMountComparison(outputPath);
|
||||||
|
return volumes
|
||||||
|
.Select(volume => new
|
||||||
|
{
|
||||||
|
Volume = volume,
|
||||||
|
NormalizedRoot = NormalizeForMountComparison(volume.VolumeRoot)
|
||||||
|
})
|
||||||
|
.Where(item => IsPathWithinMount(normalizedPath, item.NormalizedRoot))
|
||||||
|
.OrderByDescending(item => item.NormalizedRoot.Length)
|
||||||
|
.Select(item => item.Volume)
|
||||||
|
.FirstOrDefault();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsPathWithinMount(string path, string mountRoot)
|
||||||
|
{
|
||||||
|
if (mountRoot == "/")
|
||||||
|
{
|
||||||
|
return path.StartsWith("/", StringComparison.Ordinal);
|
||||||
|
}
|
||||||
|
|
||||||
|
return string.Equals(path, mountRoot, StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
path.StartsWith(mountRoot + "/", StringComparison.OrdinalIgnoreCase);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string NormalizeForMountComparison(string path)
|
||||||
|
{
|
||||||
|
var normalized = path.Trim().Replace((char)92, '/');
|
||||||
|
if (normalized.Length > 1)
|
||||||
|
{
|
||||||
|
normalized = normalized.TrimEnd('/');
|
||||||
|
}
|
||||||
|
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -49,6 +49,21 @@ public sealed partial class FfmpegService
|
|||||||
!hasTriedCurlFallback &&
|
!hasTriedCurlFallback &&
|
||||||
string.Equals(selectedProtocol, "flv", StringComparison.OrdinalIgnoreCase);
|
string.Equals(selectedProtocol, "flv", StringComparison.OrdinalIgnoreCase);
|
||||||
|
|
||||||
|
internal static FfmpegRecoveryContext PrepareRefreshedStreamRecoveryContext(
|
||||||
|
FfmpegRecoveryContext current,
|
||||||
|
string selectedProtocol,
|
||||||
|
bool hasOverlongHeadersFailure)
|
||||||
|
{
|
||||||
|
var useCurlFallback = hasOverlongHeadersFailure &&
|
||||||
|
string.Equals(selectedProtocol, "flv", StringComparison.OrdinalIgnoreCase);
|
||||||
|
|
||||||
|
return current with
|
||||||
|
{
|
||||||
|
UseCurlFallback = useCurlFallback,
|
||||||
|
HasTriedCurlFallback = current.HasTriedCurlFallback || useCurlFallback
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
internal static bool ShouldApplyRuntimeFailureBackoff(
|
internal static bool ShouldApplyRuntimeFailureBackoff(
|
||||||
RecordSessionStatus status,
|
RecordSessionStatus status,
|
||||||
TimeSpan processRuntime) =>
|
TimeSpan processRuntime) =>
|
||||||
|
|||||||
@@ -37,7 +37,10 @@ public sealed partial class FfmpegService
|
|||||||
TimeSpan.FromSeconds(15),
|
TimeSpan.FromSeconds(15),
|
||||||
TimeSpan.FromSeconds(30),
|
TimeSpan.FromSeconds(30),
|
||||||
TimeSpan.FromSeconds(60),
|
TimeSpan.FromSeconds(60),
|
||||||
TimeSpan.FromSeconds(120)
|
TimeSpan.FromSeconds(120),
|
||||||
|
TimeSpan.FromSeconds(300),
|
||||||
|
TimeSpan.FromSeconds(600),
|
||||||
|
TimeSpan.FromSeconds(900)
|
||||||
];
|
];
|
||||||
|
|
||||||
private async Task HandleProcessOutputAsync(SessionProcessRuntime runtime, string? line, bool isError)
|
private async Task HandleProcessOutputAsync(SessionProcessRuntime runtime, string? line, bool isError)
|
||||||
@@ -85,7 +88,10 @@ public sealed partial class FfmpegService
|
|||||||
runtime.MarkStartupFailure(StartupFailureKind.StreamHandshake, line);
|
runtime.MarkStartupFailure(StartupFailureKind.StreamHandshake, line);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (TryClassifyPersistedFfmpegLine(line, isError, out var level))
|
if (ShouldPersistFfmpegLine(
|
||||||
|
runtime.HasOpenedFirstSegment,
|
||||||
|
runtime.StartupFailureKind == StartupFailureKind.StreamHandshake) &&
|
||||||
|
TryClassifyPersistedFfmpegLine(line, isError, out var level))
|
||||||
{
|
{
|
||||||
await PersistFfmpegLineAsync(runtime, line, level);
|
await PersistFfmpegLineAsync(runtime, line, level);
|
||||||
}
|
}
|
||||||
@@ -728,11 +734,13 @@ public sealed partial class FfmpegService
|
|||||||
|
|
||||||
var stream = await adapter.GetStreamUrlAsync(session.LiveRoom.RoomId, session.PreferredQuality, transition.Token);
|
var stream = await adapter.GetStreamUrlAsync(session.LiveRoom.RoomId, session.PreferredQuality, transition.Token);
|
||||||
var selected = SelectRetryStreamForCurrentSession(stream, runtime);
|
var selected = SelectRetryStreamForCurrentSession(stream, runtime);
|
||||||
var context = runtime.RecoveryContext with
|
var context = PrepareRefreshedStreamRecoveryContext(runtime.RecoveryContext with
|
||||||
{
|
{
|
||||||
AttemptCount = attempt,
|
AttemptCount = attempt,
|
||||||
HasRetriedWithRefreshedStream = true
|
HasRetriedWithRefreshedStream = true
|
||||||
};
|
},
|
||||||
|
selected.SelectedProtocol,
|
||||||
|
runtime.HasHlsOverlongHeadersFailure || runtime.RecoveryContext.HasTriedCurlFallback);
|
||||||
session.MarkStarting(selected.SelectedUrl, session.OutputPathPattern ?? runtime.OutputPathPattern, DateTimeOffset.UtcNow);
|
session.MarkStarting(selected.SelectedUrl, session.OutputPathPattern ?? runtime.OutputPathPattern, DateTimeOffset.UtcNow);
|
||||||
currentTask.MarkStarting(selected.SelectedUrl, currentTask.OutputFilePath ?? runtime.CurrentOutputFilePath, DateTimeOffset.UtcNow);
|
currentTask.MarkStarting(selected.SelectedUrl, currentTask.OutputFilePath ?? runtime.CurrentOutputFilePath, DateTimeOffset.UtcNow);
|
||||||
await dbContext.SaveChangesAsync(transition.Token);
|
await dbContext.SaveChangesAsync(transition.Token);
|
||||||
@@ -1036,12 +1044,15 @@ public sealed partial class FfmpegService
|
|||||||
currentTask,
|
currentTask,
|
||||||
streamForRetry,
|
streamForRetry,
|
||||||
runtime.RecordingSettings,
|
runtime.RecordingSettings,
|
||||||
AdvanceRecoveryContext(
|
PrepareRefreshedStreamRecoveryContext(
|
||||||
runtime.RecoveryContext,
|
AdvanceRecoveryContext(
|
||||||
runtime.InputOptionProfile,
|
runtime.RecoveryContext,
|
||||||
runtime.SelectedProtocol,
|
runtime.InputOptionProfile,
|
||||||
|
runtime.SelectedProtocol,
|
||||||
|
streamForRetry.SelectedProtocol,
|
||||||
|
refreshedStream: true),
|
||||||
streamForRetry.SelectedProtocol,
|
streamForRetry.SelectedProtocol,
|
||||||
refreshedStream: true));
|
runtime.HasHlsOverlongHeadersFailure || runtime.RecoveryContext.HasTriedCurlFallback));
|
||||||
|
|
||||||
var refreshedRetryStartedAt = DateTimeOffset.UtcNow;
|
var refreshedRetryStartedAt = DateTimeOffset.UtcNow;
|
||||||
session.MarkRunning(refreshedRetryStartedAt);
|
session.MarkRunning(refreshedRetryStartedAt);
|
||||||
@@ -1362,6 +1373,11 @@ public sealed partial class FfmpegService
|
|||||||
internal static TimeSpan GetRuntimeRecoveryDelay(int attempt) =>
|
internal static TimeSpan GetRuntimeRecoveryDelay(int attempt) =>
|
||||||
RuntimeRecoveryBackoff[Math.Clamp(attempt - 1, 0, RuntimeRecoveryBackoff.Length - 1)];
|
RuntimeRecoveryBackoff[Math.Clamp(attempt - 1, 0, RuntimeRecoveryBackoff.Length - 1)];
|
||||||
|
|
||||||
|
internal static bool ShouldPersistFfmpegLine(
|
||||||
|
bool hasOpenedFirstSegment,
|
||||||
|
bool isStreamHandshakeFailure) =>
|
||||||
|
hasOpenedFirstSegment || !isStreamHandshakeFailure;
|
||||||
|
|
||||||
private async Task<bool> TryRetryWithAlternateProtocolAsync(
|
private async Task<bool> TryRetryWithAlternateProtocolAsync(
|
||||||
SessionProcessRuntime runtime,
|
SessionProcessRuntime runtime,
|
||||||
RecordSession session,
|
RecordSession session,
|
||||||
|
|||||||
@@ -22,6 +22,9 @@ namespace LiveRecorder.Infrastructure.Services;
|
|||||||
public sealed partial class FfmpegService : IFfmpegService
|
public sealed partial class FfmpegService : IFfmpegService
|
||||||
{
|
{
|
||||||
private const string ArtifactRepairMarker = "[artifact-repair]";
|
private const string ArtifactRepairMarker = "[artifact-repair]";
|
||||||
|
private const string StalePendingStartupFailure =
|
||||||
|
"Recording startup was interrupted before stream initialization.";
|
||||||
|
private static readonly TimeSpan StalePendingThreshold = TimeSpan.FromMinutes(10);
|
||||||
private static readonly Regex SegmentOpeningRegex = new(
|
private static readonly Regex SegmentOpeningRegex = new(
|
||||||
"""Opening '([^']+)' for writing""",
|
"""Opening '([^']+)' for writing""",
|
||||||
RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||||
@@ -644,6 +647,120 @@ public sealed partial class FfmpegService : IFfmpegService
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async Task<int> RecoverStalePendingSessionsAsync(CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
if (!await _orphanRecoveryGate.WaitAsync(0, cancellationToken))
|
||||||
|
{
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var scope = _serviceScopeFactory.CreateScope();
|
||||||
|
var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
|
||||||
|
var cutoff = DateTimeOffset.UtcNow - StalePendingThreshold;
|
||||||
|
var candidateIds = await dbContext.RecordSessions
|
||||||
|
.AsNoTracking()
|
||||||
|
.Where(session =>
|
||||||
|
session.Status == RecordSessionStatus.Pending &&
|
||||||
|
session.UpdatedAt <= cutoff)
|
||||||
|
.OrderBy(session => session.UpdatedAt)
|
||||||
|
.Select(session => session.Id)
|
||||||
|
.Take(50)
|
||||||
|
.ToListAsync(cancellationToken);
|
||||||
|
|
||||||
|
var recovered = 0;
|
||||||
|
foreach (var candidateId in candidateIds)
|
||||||
|
{
|
||||||
|
if (IsRunning(candidateId))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
using var candidateScope = _serviceScopeFactory.CreateScope();
|
||||||
|
var candidateDbContext = candidateScope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
|
||||||
|
var session = await candidateDbContext.RecordSessions
|
||||||
|
.Include(item => item.RecordTasks)
|
||||||
|
.ThenInclude(item => item.Result)
|
||||||
|
.FirstOrDefaultAsync(item => item.Id == candidateId, cancellationToken);
|
||||||
|
if (session is null ||
|
||||||
|
session.Status != RecordSessionStatus.Pending ||
|
||||||
|
session.UpdatedAt > cutoff ||
|
||||||
|
IsRunning(candidateId))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (HasRecoverablePendingMedia(session))
|
||||||
|
{
|
||||||
|
if (await ReconcileInactiveSessionAsync(candidateId, allowTerminalSession: false, cancellationToken))
|
||||||
|
{
|
||||||
|
recovered++;
|
||||||
|
}
|
||||||
|
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var failedAt = DateTimeOffset.UtcNow;
|
||||||
|
foreach (var task in session.RecordTasks.Where(task => task.Status == RecordTaskStatus.Pending))
|
||||||
|
{
|
||||||
|
task.MarkFailed(StalePendingStartupFailure, failedAt);
|
||||||
|
}
|
||||||
|
|
||||||
|
session.MarkFailed(StalePendingStartupFailure, failedAt);
|
||||||
|
await candidateDbContext.SaveChangesAsync(cancellationToken);
|
||||||
|
|
||||||
|
var logService = candidateScope.ServiceProvider.GetRequiredService<ISystemLogService>();
|
||||||
|
await logService.WriteAsync(
|
||||||
|
SystemLogLevel.Warning,
|
||||||
|
"RecordSession",
|
||||||
|
"A stale pending recording session was marked failed.",
|
||||||
|
StalePendingStartupFailure,
|
||||||
|
session.LiveRoomId,
|
||||||
|
session.Id,
|
||||||
|
session.RecordTasks.OrderBy(task => task.SegmentIndex).FirstOrDefault()?.Id,
|
||||||
|
cancellationToken);
|
||||||
|
recovered++;
|
||||||
|
}
|
||||||
|
|
||||||
|
return recovered;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
_orphanRecoveryGate.Release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool HasRecoverablePendingMedia(RecordSession session)
|
||||||
|
{
|
||||||
|
if (DiscoverRecoverableSegments(session.OutputPathPattern, session.OutputFormat, session.SaveMode).Count > 0)
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var task in session.RecordTasks)
|
||||||
|
{
|
||||||
|
var candidates = new[]
|
||||||
|
{
|
||||||
|
task.Result?.FilePath,
|
||||||
|
task.OutputFilePath,
|
||||||
|
string.IsNullOrWhiteSpace(task.OutputFilePath)
|
||||||
|
? null
|
||||||
|
: GetRecorderOutputPath(task.OutputFilePath, session.OutputFormat, session.SaveMode)
|
||||||
|
};
|
||||||
|
|
||||||
|
if (candidates.Any(path =>
|
||||||
|
!string.IsNullOrWhiteSpace(path) &&
|
||||||
|
File.Exists(path) &&
|
||||||
|
new FileInfo(path).Length > 0))
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
public async Task<bool> TryRecoverOrphanedTerminalTaskAsync(
|
public async Task<bool> TryRecoverOrphanedTerminalTaskAsync(
|
||||||
Guid recordTaskId,
|
Guid recordTaskId,
|
||||||
CancellationToken cancellationToken = default)
|
CancellationToken cancellationToken = default)
|
||||||
|
|||||||
@@ -76,6 +76,13 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
|
|||||||
var settings = await settingsService.GetAsync(stoppingToken);
|
var settings = await settingsService.GetAsync(stoppingToken);
|
||||||
var ffmpegService = scope.ServiceProvider.GetRequiredService<IFfmpegService>();
|
var ffmpegService = scope.ServiceProvider.GetRequiredService<IFfmpegService>();
|
||||||
await ffmpegService.ResumeRecoveringSessionsAsync(stoppingToken);
|
await ffmpegService.ResumeRecoveringSessionsAsync(stoppingToken);
|
||||||
|
var recoveredPendingSessions = await ffmpegService.RecoverStalePendingSessionsAsync(stoppingToken);
|
||||||
|
if (recoveredPendingSessions > 0)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(
|
||||||
|
"Recovered {SessionCount} stale pending recording sessions.",
|
||||||
|
recoveredPendingSessions);
|
||||||
|
}
|
||||||
var recoveredOrphanedSessions = await ffmpegService.RecoverOrphanedTerminalSessionTasksAsync(stoppingToken);
|
var recoveredOrphanedSessions = await ffmpegService.RecoverOrphanedTerminalSessionTasksAsync(stoppingToken);
|
||||||
if (recoveredOrphanedSessions > 0)
|
if (recoveredOrphanedSessions > 0)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -57,6 +57,7 @@ public sealed class RecoveryService
|
|||||||
IsAvailable = storage.IsAvailable,
|
IsAvailable = storage.IsAvailable,
|
||||||
HasEnoughSpace = storage.HasEnoughSpace,
|
HasEnoughSpace = storage.HasEnoughSpace,
|
||||||
CheckedPath = storage.CheckedPath,
|
CheckedPath = storage.CheckedPath,
|
||||||
|
VolumeRoot = storage.VolumeRoot,
|
||||||
TotalBytes = storage.TotalBytes,
|
TotalBytes = storage.TotalBytes,
|
||||||
UsedBytes = storage.UsedBytes,
|
UsedBytes = storage.UsedBytes,
|
||||||
AvailableBytes = storage.AvailableBytes,
|
AvailableBytes = storage.AvailableBytes,
|
||||||
|
|||||||
@@ -8,10 +8,14 @@ public sealed class StorageGuardService : IStorageGuardService
|
|||||||
{
|
{
|
||||||
private const long Megabyte = 1024L * 1024L;
|
private const long Megabyte = 1024L * 1024L;
|
||||||
private readonly ILogger<StorageGuardService> _logger;
|
private readonly ILogger<StorageGuardService> _logger;
|
||||||
|
private readonly IStorageCapacityProvider _capacityProvider;
|
||||||
|
|
||||||
public StorageGuardService(ILogger<StorageGuardService> logger)
|
public StorageGuardService(
|
||||||
|
ILogger<StorageGuardService> logger,
|
||||||
|
IStorageCapacityProvider capacityProvider)
|
||||||
{
|
{
|
||||||
_logger = logger;
|
_logger = logger;
|
||||||
|
_capacityProvider = capacityProvider;
|
||||||
}
|
}
|
||||||
|
|
||||||
public StorageGuardResult CheckCanStartOrResume(SystemSettingsDto settings, long additionalRequiredBytes = 0) =>
|
public StorageGuardResult CheckCanStartOrResume(SystemSettingsDto settings, long additionalRequiredBytes = 0) =>
|
||||||
@@ -44,9 +48,9 @@ public sealed class StorageGuardService : IStorageGuardService
|
|||||||
try
|
try
|
||||||
{
|
{
|
||||||
checkedPath = ResolveOutputRoot(settings.OutputRoot ?? string.Empty);
|
checkedPath = ResolveOutputRoot(settings.OutputRoot ?? string.Empty);
|
||||||
var drive = new DriveInfo(Path.GetPathRoot(checkedPath) ?? checkedPath);
|
var capacity = _capacityProvider.GetCapacity(checkedPath);
|
||||||
var availableBytes = drive.AvailableFreeSpace;
|
var availableBytes = capacity.AvailableBytes;
|
||||||
var totalBytes = drive.TotalSize;
|
var totalBytes = capacity.TotalBytes;
|
||||||
var usedBytes = Math.Max(0, totalBytes - availableBytes);
|
var usedBytes = Math.Max(0, totalBytes - availableBytes);
|
||||||
var usagePercent = totalBytes > 0 ? (double)usedBytes / totalBytes * 100.0 : 0;
|
var usagePercent = totalBytes > 0 ? (double)usedBytes / totalBytes * 100.0 : 0;
|
||||||
var freePercent = totalBytes > 0 ? (double)availableBytes / totalBytes * 100.0 : 0;
|
var freePercent = totalBytes > 0 ? (double)availableBytes / totalBytes * 100.0 : 0;
|
||||||
@@ -81,6 +85,7 @@ public sealed class StorageGuardService : IStorageGuardService
|
|||||||
return new StorageGuardResult(isEnabled, hasEnoughSpace, checkedPath, availableBytes, requiredBytes, message)
|
return new StorageGuardResult(isEnabled, hasEnoughSpace, checkedPath, availableBytes, requiredBytes, message)
|
||||||
{
|
{
|
||||||
IsAvailable = totalBytes > 0,
|
IsAvailable = totalBytes > 0,
|
||||||
|
VolumeRoot = capacity.VolumeRoot,
|
||||||
Tier = tier,
|
Tier = tier,
|
||||||
TotalBytes = totalBytes,
|
TotalBytes = totalBytes,
|
||||||
UsedBytes = usedBytes,
|
UsedBytes = usedBytes,
|
||||||
@@ -108,9 +113,10 @@ public sealed class StorageGuardService : IStorageGuardService
|
|||||||
private static string ResolveOutputRoot(string outputRoot)
|
private static string ResolveOutputRoot(string outputRoot)
|
||||||
{
|
{
|
||||||
var root = string.IsNullOrWhiteSpace(outputRoot) ? "records" : outputRoot.Trim();
|
var root = string.IsNullOrWhiteSpace(outputRoot) ? "records" : outputRoot.Trim();
|
||||||
return Path.IsPathRooted(root)
|
var resolved = Path.IsPathRooted(root)
|
||||||
? root
|
? root
|
||||||
: Path.GetFullPath(root, AppContext.BaseDirectory);
|
: Path.GetFullPath(root, AppContext.BaseDirectory);
|
||||||
|
return Path.GetFullPath(resolved);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static string FormatBytes(long bytes)
|
private static string FormatBytes(long bytes)
|
||||||
|
|||||||
@@ -57,7 +57,8 @@ public sealed class RecordTasksController : ControllerBase
|
|||||||
|
|
||||||
var items = await _recordResultRepository.ListUploadStatusAsync(filter, query, skip, take, cancellationToken);
|
var items = await _recordResultRepository.ListUploadStatusAsync(filter, query, skip, take, cancellationToken);
|
||||||
var totalCount = await _recordResultRepository.CountUploadStatusAsync(filter, query, cancellationToken);
|
var totalCount = await _recordResultRepository.CountUploadStatusAsync(filter, query, cancellationToken);
|
||||||
var notUploadedCount = await _recordResultRepository.CountPendingUploadAsync(cancellationToken);
|
var pendingUploadMetrics = await _recordResultRepository.GetPendingUploadMetricsAsync(cancellationToken);
|
||||||
|
var notUploadedCount = pendingUploadMetrics.Count;
|
||||||
var failedArtifactCount = await _recordResultRepository.CountFailedArtifactAsync(cancellationToken);
|
var failedArtifactCount = await _recordResultRepository.CountFailedArtifactAsync(cancellationToken);
|
||||||
var succeededCount = await _recordResultRepository.CountUploadStatusAsync(RecordArtifactUploadStatus.Succeeded, null, cancellationToken);
|
var succeededCount = await _recordResultRepository.CountUploadStatusAsync(RecordArtifactUploadStatus.Succeeded, null, cancellationToken);
|
||||||
var failedCount = await _recordResultRepository.CountUploadStatusAsync(RecordArtifactUploadStatus.Failed, null, cancellationToken);
|
var failedCount = await _recordResultRepository.CountUploadStatusAsync(RecordArtifactUploadStatus.Failed, null, cancellationToken);
|
||||||
|
|||||||
@@ -243,6 +243,7 @@ builder.Services.AddScoped<ILiveDanmakuAdapter, BilibiliDanmakuAdapter>();
|
|||||||
builder.Services.AddScoped<ILiveDanmakuAdapterFactory, LiveDanmakuAdapterFactory>();
|
builder.Services.AddScoped<ILiveDanmakuAdapterFactory, LiveDanmakuAdapterFactory>();
|
||||||
|
|
||||||
builder.Services.AddSingleton<IFfmpegService, FfmpegService>();
|
builder.Services.AddSingleton<IFfmpegService, FfmpegService>();
|
||||||
|
builder.Services.AddSingleton<IStorageCapacityProvider, DriveInfoStorageCapacityProvider>();
|
||||||
builder.Services.AddSingleton<IStorageGuardService, StorageGuardService>();
|
builder.Services.AddSingleton<IStorageGuardService, StorageGuardService>();
|
||||||
builder.Services.AddScoped<IRecordMediaService, RecordMediaService>();
|
builder.Services.AddScoped<IRecordMediaService, RecordMediaService>();
|
||||||
builder.Services.AddSingleton<LiveRoomPollingBackgroundService>();
|
builder.Services.AddSingleton<LiveRoomPollingBackgroundService>();
|
||||||
|
|||||||
@@ -272,7 +272,7 @@ public sealed class FfmpegFailureClassificationTests
|
|||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public void RecoveryContext_LeavesCurlAfterExitWithoutResettingOneShotBudget()
|
public void RefreshedFlvRecovery_ReEnablesCurlAfterOverlongHeaders()
|
||||||
{
|
{
|
||||||
var curlContext = FfmpegService.InitialRecoveryContext with
|
var curlContext = FfmpegService.InitialRecoveryContext with
|
||||||
{
|
{
|
||||||
@@ -280,18 +280,65 @@ public sealed class FfmpegFailureClassificationTests
|
|||||||
HasTriedCurlFallback = true
|
HasTriedCurlFallback = true
|
||||||
};
|
};
|
||||||
|
|
||||||
var nextContext = FfmpegService.AdvanceRecoveryContext(
|
var nextContext = FfmpegService.PrepareRefreshedStreamRecoveryContext(
|
||||||
curlContext,
|
FfmpegService.AdvanceRecoveryContext(
|
||||||
curlContext.InputOptionProfile,
|
curlContext,
|
||||||
|
curlContext.InputOptionProfile,
|
||||||
|
"flv",
|
||||||
|
"flv",
|
||||||
|
refreshedStream: true),
|
||||||
"flv",
|
"flv",
|
||||||
"flv",
|
hasOverlongHeadersFailure: true);
|
||||||
refreshedStream: true);
|
|
||||||
|
|
||||||
Assert.False(nextContext.UseCurlFallback);
|
Assert.True(nextContext.UseCurlFallback);
|
||||||
Assert.True(nextContext.HasTriedCurlFallback);
|
Assert.True(nextContext.HasTriedCurlFallback);
|
||||||
Assert.True(nextContext.HasRetriedWithRefreshedStream);
|
Assert.True(nextContext.HasRetriedWithRefreshedStream);
|
||||||
Assert.False(FfmpegService.ShouldFallbackToCurl("flv", true, nextContext.HasTriedCurlFallback));
|
Assert.False(FfmpegService.ShouldFallbackToCurl("flv", true, nextContext.HasTriedCurlFallback));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void RefreshedNonFlvRecovery_DisablesCurl()
|
||||||
|
{
|
||||||
|
var nextContext = FfmpegService.PrepareRefreshedStreamRecoveryContext(
|
||||||
|
FfmpegService.InitialRecoveryContext with
|
||||||
|
{
|
||||||
|
UseCurlFallback = true,
|
||||||
|
HasTriedCurlFallback = true
|
||||||
|
},
|
||||||
|
"hls",
|
||||||
|
hasOverlongHeadersFailure: true);
|
||||||
|
|
||||||
|
Assert.False(nextContext.UseCurlFallback);
|
||||||
|
Assert.True(nextContext.HasTriedCurlFallback);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void RefreshedFlvRecovery_PreservesCurlRequirementAcrossLaterProcesses()
|
||||||
|
{
|
||||||
|
var priorContext = FfmpegService.InitialRecoveryContext with
|
||||||
|
{
|
||||||
|
UseCurlFallback = false,
|
||||||
|
HasTriedCurlFallback = true
|
||||||
|
};
|
||||||
|
|
||||||
|
var nextContext = FfmpegService.PrepareRefreshedStreamRecoveryContext(
|
||||||
|
priorContext,
|
||||||
|
"flv",
|
||||||
|
hasOverlongHeadersFailure: priorContext.HasTriedCurlFallback);
|
||||||
|
|
||||||
|
Assert.True(nextContext.UseCurlFallback);
|
||||||
|
Assert.True(nextContext.HasTriedCurlFallback);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Theory]
|
||||||
|
[InlineData(false, true, false)]
|
||||||
|
[InlineData(false, false, true)]
|
||||||
|
[InlineData(true, true, true)]
|
||||||
|
public void FfmpegLinePersistence_SuppressesRecoverableStartupHandshakeNoise(
|
||||||
|
bool hasOpenedFirstSegment,
|
||||||
|
bool isStreamHandshakeFailure,
|
||||||
|
bool expected) =>
|
||||||
|
Assert.Equal(expected, FfmpegService.ShouldPersistFfmpegLine(hasOpenedFirstSegment, isStreamHandshakeFailure));
|
||||||
[Fact]
|
[Fact]
|
||||||
public void RecoveryContext_PreservesBudgetAcrossHlsFallbackAndReachesTimestampTranscode()
|
public void RecoveryContext_PreservesBudgetAcrossHlsFallbackAndReachesTimestampTranscode()
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -615,6 +615,62 @@ public sealed class OpenListUploadTests
|
|||||||
Assert.Equal(1, await repository.CountFailedArtifactAsync());
|
Assert.Equal(1, await repository.CountFailedArtifactAsync());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task PendingUploadMetrics_IncludeValidVisibleArtifactWithActualFileSize()
|
||||||
|
{
|
||||||
|
await using var fixture = await QueueFixture.CreateAsync();
|
||||||
|
var result = await fixture.Context.RecordResults.AsNoTracking().SingleAsync();
|
||||||
|
var repository = new RecordResultRepository(fixture.Context);
|
||||||
|
|
||||||
|
var metrics = await repository.GetPendingUploadMetricsAsync();
|
||||||
|
|
||||||
|
Assert.Equal(1, metrics.Count);
|
||||||
|
Assert.Equal(new FileInfo(result.FilePath).Length, metrics.TotalBytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task PendingUploadMetrics_ExcludeHiddenMergedSource()
|
||||||
|
{
|
||||||
|
await using var fixture = await QueueFixture.CreateAsync();
|
||||||
|
var task = await fixture.Context.RecordTasks.SingleAsync();
|
||||||
|
task.MarkMergedSource(Guid.NewGuid(), DateTimeOffset.UtcNow);
|
||||||
|
await fixture.Context.SaveChangesAsync();
|
||||||
|
var repository = new RecordResultRepository(fixture.Context);
|
||||||
|
|
||||||
|
var metrics = await repository.GetPendingUploadMetricsAsync();
|
||||||
|
|
||||||
|
Assert.Equal(0, metrics.Count);
|
||||||
|
Assert.Equal(0, metrics.TotalBytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task PendingUploadMetrics_ExcludeMissingLocalFile()
|
||||||
|
{
|
||||||
|
await using var fixture = await QueueFixture.CreateAsync();
|
||||||
|
var result = await fixture.Context.RecordResults.AsNoTracking().SingleAsync();
|
||||||
|
File.Delete(result.FilePath);
|
||||||
|
var repository = new RecordResultRepository(fixture.Context);
|
||||||
|
|
||||||
|
var metrics = await repository.GetPendingUploadMetricsAsync();
|
||||||
|
|
||||||
|
Assert.Equal(0, metrics.Count);
|
||||||
|
Assert.Equal(0, metrics.TotalBytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task PendingUploadMetrics_ExcludeEmptyLocalFile()
|
||||||
|
{
|
||||||
|
await using var fixture = await QueueFixture.CreateAsync();
|
||||||
|
var result = await fixture.Context.RecordResults.AsNoTracking().SingleAsync();
|
||||||
|
File.WriteAllBytes(result.FilePath, []);
|
||||||
|
var repository = new RecordResultRepository(fixture.Context);
|
||||||
|
|
||||||
|
var metrics = await repository.GetPendingUploadMetricsAsync();
|
||||||
|
|
||||||
|
Assert.Equal(0, metrics.Count);
|
||||||
|
Assert.Equal(0, metrics.TotalBytes);
|
||||||
|
}
|
||||||
|
|
||||||
private static OpenListClient CreateClient(HttpMessageHandler handler) =>
|
private static OpenListClient CreateClient(HttpMessageHandler handler) =>
|
||||||
new(new StubHttpClientFactory(handler));
|
new(new StubHttpClientFactory(handler));
|
||||||
|
|
||||||
|
|||||||
@@ -20,8 +20,11 @@ public sealed class RecordingContinuityTests
|
|||||||
[InlineData(3, 30)]
|
[InlineData(3, 30)]
|
||||||
[InlineData(4, 60)]
|
[InlineData(4, 60)]
|
||||||
[InlineData(5, 120)]
|
[InlineData(5, 120)]
|
||||||
[InlineData(50, 120)]
|
[InlineData(6, 300)]
|
||||||
public void RuntimeRecoveryBackoff_IsBoundedAtTwoMinutes(int attempt, int seconds) =>
|
[InlineData(7, 600)]
|
||||||
|
[InlineData(8, 900)]
|
||||||
|
[InlineData(50, 900)]
|
||||||
|
public void RuntimeRecoveryBackoff_IsBoundedAtFifteenMinutes(int attempt, int seconds) =>
|
||||||
Assert.Equal(TimeSpan.FromSeconds(seconds), FfmpegService.GetRuntimeRecoveryDelay(attempt));
|
Assert.Equal(TimeSpan.FromSeconds(seconds), FfmpegService.GetRuntimeRecoveryDelay(attempt));
|
||||||
|
|
||||||
[Theory]
|
[Theory]
|
||||||
|
|||||||
@@ -0,0 +1,120 @@
|
|||||||
|
using LiveRecorder.Application.Abstractions.Logging;
|
||||||
|
using LiveRecorder.Application.Models.Logs;
|
||||||
|
using LiveRecorder.Domain.Entities;
|
||||||
|
using LiveRecorder.Domain.Enums;
|
||||||
|
using LiveRecorder.Infrastructure.Persistence;
|
||||||
|
using LiveRecorder.Infrastructure.Services;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.EntityFrameworkCore.Storage;
|
||||||
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
|
||||||
|
namespace LiveRecorder.Tests;
|
||||||
|
|
||||||
|
public sealed class StalePendingRecoveryTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public async Task StalePendingSessionWithoutMedia_IsFailedAndRetainedForAudit()
|
||||||
|
{
|
||||||
|
var services = new ServiceCollection();
|
||||||
|
var databaseRoot = new InMemoryDatabaseRoot();
|
||||||
|
var databaseName = $"stale-pending-{Guid.NewGuid():N}";
|
||||||
|
services.AddDbContext<LiveRecorderDbContext>(options =>
|
||||||
|
options.UseInMemoryDatabase(databaseName, databaseRoot));
|
||||||
|
var logService = new CapturingSystemLogService();
|
||||||
|
services.AddSingleton<ISystemLogService>(logService);
|
||||||
|
await using var provider = services.BuildServiceProvider();
|
||||||
|
|
||||||
|
var createdAt = DateTimeOffset.UtcNow.AddMinutes(-11);
|
||||||
|
Guid sessionId;
|
||||||
|
Guid taskId;
|
||||||
|
await using (var scope = provider.CreateAsyncScope())
|
||||||
|
{
|
||||||
|
var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
|
||||||
|
var liveRoom = new LiveRoom(
|
||||||
|
LivePlatformType.Douyin,
|
||||||
|
"https://live.example/room",
|
||||||
|
"room-1",
|
||||||
|
"https://live.example/room",
|
||||||
|
createdAt);
|
||||||
|
var session = new RecordSession(
|
||||||
|
liveRoom.Id,
|
||||||
|
"origin",
|
||||||
|
RecordOutputFormat.Mp4,
|
||||||
|
RecordSaveMode.Segmented,
|
||||||
|
createdAt);
|
||||||
|
var task = new RecordTask(
|
||||||
|
liveRoom.Id,
|
||||||
|
session.Id,
|
||||||
|
1,
|
||||||
|
"origin",
|
||||||
|
RecordOutputFormat.Mp4,
|
||||||
|
createdAt);
|
||||||
|
sessionId = session.Id;
|
||||||
|
taskId = task.Id;
|
||||||
|
dbContext.AddRange(liveRoom, session, task);
|
||||||
|
await dbContext.SaveChangesAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
var service = new FfmpegService(
|
||||||
|
provider.GetRequiredService<IServiceScopeFactory>(),
|
||||||
|
null!,
|
||||||
|
null!,
|
||||||
|
NullLogger<FfmpegService>.Instance);
|
||||||
|
|
||||||
|
await using (var preconditionScope = provider.CreateAsyncScope())
|
||||||
|
{
|
||||||
|
var preconditionContext = preconditionScope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
|
||||||
|
var persisted = await preconditionContext.RecordSessions.SingleAsync();
|
||||||
|
Assert.Equal(RecordSessionStatus.Pending, persisted.Status);
|
||||||
|
Assert.True(
|
||||||
|
persisted.UpdatedAt <= DateTimeOffset.UtcNow.AddMinutes(-10),
|
||||||
|
$"updatedAt={persisted.UpdatedAt:O}");
|
||||||
|
}
|
||||||
|
|
||||||
|
var recovered = await service.RecoverStalePendingSessionsAsync();
|
||||||
|
|
||||||
|
Assert.Equal(1, recovered);
|
||||||
|
await using var verificationScope = provider.CreateAsyncScope();
|
||||||
|
var verificationContext = verificationScope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
|
||||||
|
var persistedSession = await verificationContext.RecordSessions.SingleAsync(item => item.Id == sessionId);
|
||||||
|
var persistedTask = await verificationContext.RecordTasks
|
||||||
|
.Include(item => item.Result)
|
||||||
|
.SingleAsync(item => item.Id == taskId);
|
||||||
|
Assert.Equal(RecordSessionStatus.Failed, persistedSession.Status);
|
||||||
|
Assert.Equal(RecordTaskStatus.Failed, persistedTask.Status);
|
||||||
|
Assert.Contains("interrupted before stream initialization", persistedSession.ErrorMessage);
|
||||||
|
Assert.Null(persistedTask.Result);
|
||||||
|
Assert.Single(logService.Entries);
|
||||||
|
Assert.Equal(SystemLogLevel.Warning, logService.Entries[0].Level);
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class CapturingSystemLogService : ISystemLogService
|
||||||
|
{
|
||||||
|
public List<(SystemLogLevel Level, string Message)> Entries { get; } = [];
|
||||||
|
|
||||||
|
public Task WriteAsync(
|
||||||
|
SystemLogLevel level,
|
||||||
|
string category,
|
||||||
|
string message,
|
||||||
|
string? detail = null,
|
||||||
|
Guid? liveRoomId = null,
|
||||||
|
Guid? recordSessionId = null,
|
||||||
|
Guid? recordTaskId = null,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
Entries.Add((level, message));
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task<IReadOnlyList<SystemLogDto>> ListAsync(
|
||||||
|
Guid? liveRoomId = null,
|
||||||
|
Guid? recordSessionId = null,
|
||||||
|
Guid? recordTaskId = null,
|
||||||
|
SystemLogLevel? level = null,
|
||||||
|
string? content = null,
|
||||||
|
int take = 200,
|
||||||
|
CancellationToken cancellationToken = default) =>
|
||||||
|
Task.FromResult<IReadOnlyList<SystemLogDto>>([]);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,7 +7,26 @@ namespace LiveRecorder.Tests;
|
|||||||
|
|
||||||
public sealed class StorageGuardServiceTests
|
public sealed class StorageGuardServiceTests
|
||||||
{
|
{
|
||||||
private readonly StorageGuardService _service = new(NullLogger<StorageGuardService>.Instance);
|
private readonly StorageGuardService _service = new(
|
||||||
|
NullLogger<StorageGuardService>.Instance,
|
||||||
|
new DriveInfoStorageCapacityProvider());
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void CapacityProvider_SelectsLongestContainingMountRoot()
|
||||||
|
{
|
||||||
|
var selected = DriveInfoStorageCapacityProvider.SelectBestVolume(
|
||||||
|
"/vol00/live-recorder/Record",
|
||||||
|
[
|
||||||
|
new StorageVolumeCapacity("/", 100, 10),
|
||||||
|
new StorageVolumeCapacity("/vol00", 200, 20),
|
||||||
|
new StorageVolumeCapacity("/vol00/live-recorder", 300, 30),
|
||||||
|
new StorageVolumeCapacity("/vol0", 400, 40)
|
||||||
|
]);
|
||||||
|
|
||||||
|
Assert.NotNull(selected);
|
||||||
|
Assert.Equal("/vol00/live-recorder", selected.VolumeRoot);
|
||||||
|
Assert.Equal(300, selected.TotalBytes);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public void DisabledProtection_StillReportsActualCapacityWithoutBlockingRecording()
|
public void DisabledProtection_StillReportsActualCapacityWithoutBlockingRecording()
|
||||||
|
|||||||
Reference in New Issue
Block a user