feat: harden recording lifecycle and refresh fnOS UI
This commit is contained in:
@@ -119,6 +119,7 @@ public sealed class CleanupOperationCoordinator
|
||||
new RetentionCleanupOperationFilters
|
||||
{
|
||||
RetentionDays = settings.RetentionDays,
|
||||
RequireUploadSuccess = settings.RetentionRequireUploadSuccess,
|
||||
VideoFileCondition = settings.RetentionVideoFileCondition,
|
||||
TaskStatuses = settings.RetentionTaskStatuses
|
||||
},
|
||||
|
||||
@@ -85,6 +85,8 @@ internal class ConditionalCleanupOperationFilters
|
||||
internal sealed class RetentionCleanupOperationFilters : ConditionalCleanupOperationFilters
|
||||
{
|
||||
public int RetentionDays { get; init; } = 30;
|
||||
|
||||
public bool RequireUploadSuccess { get; init; }
|
||||
}
|
||||
|
||||
internal sealed class EmptyCleanupOperationFilters
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
using LiveRecorder.Application.Abstractions.Scripting;
|
||||
using LiveRecorder.Domain.Enums;
|
||||
using LiveRecorder.Infrastructure.Persistence;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace LiveRecorder.Infrastructure.Services;
|
||||
|
||||
public sealed class CompletionDispatchService
|
||||
{
|
||||
private static readonly TimeSpan RetryDelay = TimeSpan.FromMinutes(1);
|
||||
private readonly LiveRecorderDbContext _dbContext;
|
||||
private readonly IEventScriptService _eventScriptService;
|
||||
private readonly RecordUploadService _recordUploadService;
|
||||
|
||||
public CompletionDispatchService(
|
||||
LiveRecorderDbContext dbContext,
|
||||
IEventScriptService eventScriptService,
|
||||
RecordUploadService recordUploadService)
|
||||
{
|
||||
_dbContext = dbContext;
|
||||
_eventScriptService = eventScriptService;
|
||||
_recordUploadService = recordUploadService;
|
||||
}
|
||||
|
||||
public async Task<bool> TryDispatchNextAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var dispatch = await _dbContext.RecordCompletionDispatches
|
||||
.Include(item => item.RecordTask)!.ThenInclude(task => task!.LiveRoom)
|
||||
.Include(item => item.RecordTask)!.ThenInclude(task => task!.RecordSession)
|
||||
.Include(item => item.RecordTask)!.ThenInclude(task => task!.Result)
|
||||
.Where(item => item.CompletedAt == null && (!item.NextAttemptAt.HasValue || item.NextAttemptAt <= now))
|
||||
.OrderBy(static item => item.CreatedAt)
|
||||
.FirstOrDefaultAsync(cancellationToken);
|
||||
if (dispatch is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
await DispatchAsync(dispatch, cancellationToken);
|
||||
return true;
|
||||
}
|
||||
|
||||
public async Task TryDispatchTaskAsync(Guid recordTaskId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var dispatch = await _dbContext.RecordCompletionDispatches
|
||||
.Include(item => item.RecordTask)!.ThenInclude(task => task!.LiveRoom)
|
||||
.Include(item => item.RecordTask)!.ThenInclude(task => task!.RecordSession)
|
||||
.Include(item => item.RecordTask)!.ThenInclude(task => task!.Result)
|
||||
.FirstOrDefaultAsync(item => item.RecordTaskId == recordTaskId, cancellationToken);
|
||||
if (dispatch is not null && dispatch.CompletedAt is null)
|
||||
{
|
||||
await DispatchAsync(dispatch, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task DispatchAsync(Domain.Entities.RecordCompletionDispatch dispatch, CancellationToken cancellationToken)
|
||||
{
|
||||
var task = dispatch.RecordTask;
|
||||
if (task?.RecordSession is null || task.Result is null)
|
||||
{
|
||||
dispatch.ScheduleRetry("录制任务上下文尚未准备完成。", DateTimeOffset.UtcNow.Add(RetryDelay), DateTimeOffset.UtcNow);
|
||||
await _dbContext.SaveChangesAsync(cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (!dispatch.ScriptDispatched)
|
||||
{
|
||||
var scriptResult = await _eventScriptService.RunSegmentCompletedAsync(
|
||||
task.LiveRoom,
|
||||
task.RecordSession,
|
||||
task,
|
||||
task.Result,
|
||||
task.Result.FilePath,
|
||||
task.EndedAt ?? DateTimeOffset.UtcNow,
|
||||
eventId: dispatch.Id,
|
||||
cancellationToken: cancellationToken);
|
||||
if (scriptResult is null || scriptResult.Success)
|
||||
{
|
||||
dispatch.MarkScriptDispatched(DateTimeOffset.UtcNow);
|
||||
await _dbContext.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new InvalidOperationException(scriptResult.Message);
|
||||
}
|
||||
}
|
||||
|
||||
if (!dispatch.UploadDispatched)
|
||||
{
|
||||
_ = await _recordUploadService.TryAutoUploadTaskAsync(task.Id, cancellationToken);
|
||||
dispatch.MarkUploadDispatched(DateTimeOffset.UtcNow);
|
||||
await _dbContext.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
dispatch.ScheduleRetry(ex.Message, DateTimeOffset.UtcNow.Add(RetryDelay), DateTimeOffset.UtcNow);
|
||||
await _dbContext.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class CompletionDispatchBackgroundService : BackgroundService
|
||||
{
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
private readonly ILogger<CompletionDispatchBackgroundService> _logger;
|
||||
|
||||
public CompletionDispatchBackgroundService(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
ILogger<CompletionDispatchBackgroundService> logger)
|
||||
{
|
||||
_scopeFactory = scopeFactory;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var service = scope.ServiceProvider.GetRequiredService<CompletionDispatchService>();
|
||||
var processed = await service.TryDispatchNextAsync(stoppingToken);
|
||||
await Task.Delay(processed ? TimeSpan.FromMilliseconds(200) : TimeSpan.FromSeconds(2), stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Completion dispatch worker iteration failed.");
|
||||
await Task.Delay(TimeSpan.FromSeconds(2), stoppingToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -100,11 +100,13 @@ public sealed class EventScriptService : IEventScriptService
|
||||
string segmentFilePath,
|
||||
DateTimeOffset occurredAt,
|
||||
bool forceRun = false,
|
||||
Guid? eventId = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var settings = await _settingsService.GetAsync(cancellationToken);
|
||||
var environment = BuildLiveRoomEnvironment(liveRoom, occurredAt);
|
||||
environment["LIVE_RECORDER_EVENT"] = "segment_completed";
|
||||
environment["LIVE_RECORDER_EVENT_ID"] = (eventId ?? recordTask.Id).ToString();
|
||||
environment["LIVE_RECORDER_RECORD_SESSION_ID"] = recordSession.Id.ToString();
|
||||
environment["LIVE_RECORDER_RECORD_TASK_ID"] = recordTask.Id.ToString();
|
||||
environment["LIVE_RECORDER_SEGMENT_INDEX"] = recordTask.SegmentIndex.ToString();
|
||||
|
||||
@@ -0,0 +1,297 @@
|
||||
using System.Diagnostics;
|
||||
using System.Text.RegularExpressions;
|
||||
using LiveRecorder.Domain.Enums;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace LiveRecorder.Infrastructure.Services;
|
||||
|
||||
public sealed partial class FfmpegService
|
||||
{
|
||||
internal static readonly FfmpegRecoveryContext InitialRecoveryContext = new(
|
||||
FfmpegInputOptionProfile.Baseline,
|
||||
AttemptCount: 0,
|
||||
HasRetriedWithCompatibilityProfile: false,
|
||||
HasRetriedWithRefreshedStream: false,
|
||||
HasRetriedWithAlternateProtocol: false,
|
||||
ForceSoftwareEncoder: false,
|
||||
HasRetriedWithSoftwareEncoder: false);
|
||||
|
||||
internal static FfmpegRecoveryContext AdvanceRecoveryContext(
|
||||
FfmpegRecoveryContext current,
|
||||
FfmpegInputOptionProfile inputOptionProfile,
|
||||
string currentProtocol,
|
||||
string nextProtocol,
|
||||
bool refreshedStream = false) =>
|
||||
current with
|
||||
{
|
||||
InputOptionProfile = inputOptionProfile,
|
||||
AttemptCount = current.AttemptCount + 1,
|
||||
HasRetriedWithRefreshedStream = current.HasRetriedWithRefreshedStream || refreshedStream,
|
||||
HasRetriedWithAlternateProtocol = current.HasRetriedWithAlternateProtocol ||
|
||||
!string.Equals(currentProtocol, nextProtocol, StringComparison.OrdinalIgnoreCase)
|
||||
};
|
||||
|
||||
internal static bool ShouldImmediatelyFallbackFromHls(
|
||||
string selectedProtocol,
|
||||
bool hasHlsOverlongHeadersFailure) =>
|
||||
hasHlsOverlongHeadersFailure &&
|
||||
string.Equals(selectedProtocol, "hls", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
internal static bool ShouldApplyRuntimeFailureBackoff(
|
||||
RecordSessionStatus status,
|
||||
TimeSpan processRuntime) =>
|
||||
status == RecordSessionStatus.Failed && processRuntime < StableRuntimeResetThreshold;
|
||||
|
||||
private async Task<RecoveryVideoEncoderSelection> ResolveRecoveryVideoEncoderAsync(
|
||||
string ffmpegPath,
|
||||
bool forceSoftware,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (forceSoftware)
|
||||
{
|
||||
return RecoveryVideoEncoderSelection.Software;
|
||||
}
|
||||
|
||||
if (_hasProbedRecoveryVideoEncoder)
|
||||
{
|
||||
return _cachedRecoveryVideoEncoder;
|
||||
}
|
||||
|
||||
await _recoveryEncoderProbeGate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
if (_hasProbedRecoveryVideoEncoder)
|
||||
{
|
||||
return _cachedRecoveryVideoEncoder;
|
||||
}
|
||||
|
||||
var candidates = new List<RecoveryVideoEncoderSelection>
|
||||
{
|
||||
new(RecoveryVideoEncoderKind.Nvenc, null)
|
||||
};
|
||||
|
||||
IReadOnlyList<string> renderDevices = [];
|
||||
try
|
||||
{
|
||||
if (Directory.Exists("/dev/dri"))
|
||||
{
|
||||
renderDevices = Directory.EnumerateFileSystemEntries("/dev/dri", "renderD*")
|
||||
.OrderBy(static path => path)
|
||||
.ToList();
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
_logger.LogDebug(ex, "Unable to enumerate /dev/dri render devices for recovery encoding.");
|
||||
}
|
||||
|
||||
foreach (var devicePath in renderDevices)
|
||||
{
|
||||
candidates.Add(new RecoveryVideoEncoderSelection(RecoveryVideoEncoderKind.Qsv, devicePath));
|
||||
}
|
||||
|
||||
foreach (var devicePath in renderDevices)
|
||||
{
|
||||
candidates.Add(new RecoveryVideoEncoderSelection(RecoveryVideoEncoderKind.Vaapi, devicePath));
|
||||
}
|
||||
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
if (await ProbeRecoveryVideoEncoderAsync(ffmpegPath, candidate, cancellationToken))
|
||||
{
|
||||
_cachedRecoveryVideoEncoder = candidate;
|
||||
_hasProbedRecoveryVideoEncoder = true;
|
||||
_logger.LogInformation(
|
||||
"Recovery video encoder probe selected {EncoderKind} using device {DevicePath}",
|
||||
candidate.Kind,
|
||||
candidate.DevicePath ?? "default");
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
_cachedRecoveryVideoEncoder = RecoveryVideoEncoderSelection.Software;
|
||||
_hasProbedRecoveryVideoEncoder = true;
|
||||
_logger.LogInformation("No usable hardware recovery encoder was detected; libx264 will be used.");
|
||||
return _cachedRecoveryVideoEncoder;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_recoveryEncoderProbeGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<bool> ProbeRecoveryVideoEncoderAsync(
|
||||
string ffmpegPath,
|
||||
RecoveryVideoEncoderSelection encoder,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
using var process = new Process
|
||||
{
|
||||
StartInfo = new ProcessStartInfo
|
||||
{
|
||||
FileName = ffmpegPath,
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
CreateNoWindow = true
|
||||
}
|
||||
};
|
||||
|
||||
foreach (var argument in BuildRecoveryEncoderProbeArgumentList(encoder))
|
||||
{
|
||||
process.StartInfo.ArgumentList.Add(argument);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (!process.Start())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var outputTask = process.StandardOutput.ReadToEndAsync(cancellationToken);
|
||||
var errorTask = process.StandardError.ReadToEndAsync(cancellationToken);
|
||||
using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
timeoutCts.CancelAfter(TimeSpan.FromSeconds(8));
|
||||
await process.WaitForExitAsync(timeoutCts.Token);
|
||||
await Task.WhenAll(outputTask, errorTask);
|
||||
return process.ExitCode == 0;
|
||||
}
|
||||
catch (Exception ex) when (ex is InvalidOperationException or System.ComponentModel.Win32Exception or OperationCanceledException)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!process.HasExited)
|
||||
{
|
||||
process.Kill(true);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
}
|
||||
|
||||
_logger.LogDebug(
|
||||
ex,
|
||||
"Recovery video encoder probe failed for {EncoderKind} using device {DevicePath}",
|
||||
encoder.Kind,
|
||||
encoder.DevicePath ?? "default");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private void DisableRecoveryVideoEncoder(RecoveryVideoEncoderSelection encoder)
|
||||
{
|
||||
if (encoder.Kind == RecoveryVideoEncoderKind.Software ||
|
||||
_cachedRecoveryVideoEncoder != encoder)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_cachedRecoveryVideoEncoder = RecoveryVideoEncoderSelection.Software;
|
||||
_hasProbedRecoveryVideoEncoder = true;
|
||||
_logger.LogWarning(
|
||||
"Recovery video encoder {EncoderKind} failed with a live input and was disabled until the service restarts.",
|
||||
encoder.Kind);
|
||||
}
|
||||
|
||||
internal static IReadOnlyList<string> BuildRecoveryEncoderProbeArgumentList(
|
||||
RecoveryVideoEncoderSelection encoder)
|
||||
{
|
||||
var arguments = new List<string> { "-hide_banner", "-loglevel", "error" };
|
||||
AddRecoveryEncoderDeviceArguments(arguments, encoder);
|
||||
arguments.AddRange(["-f", "lavfi", "-i", "color=c=black:s=128x128:r=1", "-frames:v", "1"]);
|
||||
arguments.AddRange(BuildRecoveryVideoCodecArguments(encoder));
|
||||
arguments.AddRange(["-an", "-f", "null", "-"]);
|
||||
return arguments;
|
||||
}
|
||||
|
||||
internal static IReadOnlyList<RecoverableRecorderSegment> DiscoverRecoverableSegments(
|
||||
string? outputPathPattern,
|
||||
RecordOutputFormat outputFormat,
|
||||
RecordSaveMode saveMode)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(outputPathPattern) ||
|
||||
outputFormat != RecordOutputFormat.Mp4 ||
|
||||
saveMode != RecordSaveMode.Segmented ||
|
||||
!outputPathPattern.Contains("%05d", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var absoluteOutputPattern = NormalizeAbsolutePath(outputPathPattern);
|
||||
var recorderPattern = GetRecorderOutputPath(absoluteOutputPattern, outputFormat, saveMode);
|
||||
var directory = Path.GetDirectoryName(recorderPattern);
|
||||
var filePattern = Path.GetFileName(recorderPattern);
|
||||
if (string.IsNullOrWhiteSpace(directory) ||
|
||||
string.IsNullOrWhiteSpace(filePattern) ||
|
||||
!Directory.Exists(directory))
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var tokenIndex = filePattern.IndexOf("%05d", StringComparison.OrdinalIgnoreCase);
|
||||
if (tokenIndex < 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var prefix = filePattern[..tokenIndex];
|
||||
var suffix = filePattern[(tokenIndex + 4)..];
|
||||
var matcher = new Regex(
|
||||
$"^{Regex.Escape(prefix)}(?<segment>[0-9]{{5}}){Regex.Escape(suffix)}$",
|
||||
RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
|
||||
try
|
||||
{
|
||||
return Directory.EnumerateFiles(directory, $"{prefix}*{suffix}", SearchOption.TopDirectoryOnly)
|
||||
.Select(path => (Path: path, Match: matcher.Match(Path.GetFileName(path))))
|
||||
.Where(item => item.Match.Success && new FileInfo(item.Path).Length > 0)
|
||||
.Select(item => new
|
||||
{
|
||||
RecorderPath = NormalizeAbsolutePath(item.Path),
|
||||
SegmentIndex = int.Parse(item.Match.Groups["segment"].Value)
|
||||
})
|
||||
.Where(item => item.SegmentIndex > 0)
|
||||
.GroupBy(item => item.SegmentIndex)
|
||||
.Select(group => group.First())
|
||||
.OrderBy(item => item.SegmentIndex)
|
||||
.Select(item => new RecoverableRecorderSegment(
|
||||
item.SegmentIndex,
|
||||
item.RecorderPath,
|
||||
NormalizeAbsolutePath(ResolveSegmentOutputPath(absoluteOutputPattern, saveMode, item.SegmentIndex))))
|
||||
.ToList();
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
catch (UnauthorizedAccessException)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record FfmpegRecoveryContext(
|
||||
FfmpegService.FfmpegInputOptionProfile InputOptionProfile,
|
||||
int AttemptCount,
|
||||
bool HasRetriedWithCompatibilityProfile,
|
||||
bool HasRetriedWithRefreshedStream,
|
||||
bool HasRetriedWithAlternateProtocol,
|
||||
bool ForceSoftwareEncoder,
|
||||
bool HasRetriedWithSoftwareEncoder);
|
||||
|
||||
internal enum RecoveryVideoEncoderKind
|
||||
{
|
||||
Software = 0,
|
||||
Nvenc = 1,
|
||||
Qsv = 2,
|
||||
Vaapi = 3
|
||||
}
|
||||
|
||||
internal sealed record RecoveryVideoEncoderSelection(RecoveryVideoEncoderKind Kind, string? DevicePath)
|
||||
{
|
||||
public static RecoveryVideoEncoderSelection Software { get; } = new(RecoveryVideoEncoderKind.Software, null);
|
||||
}
|
||||
|
||||
internal sealed record RecoverableRecorderSegment(int SegmentIndex, string RecorderPath, string OutputPath);
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,7 +1,9 @@
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using LiveRecorder.Application.Abstractions.Platforms;
|
||||
using LiveRecorder.Application.Abstractions.Recording;
|
||||
using LiveRecorder.Domain.Entities;
|
||||
using LiveRecorder.Domain.Enums;
|
||||
using LiveRecorder.Infrastructure.Persistence;
|
||||
@@ -13,7 +15,9 @@ namespace LiveRecorder.Infrastructure.Services;
|
||||
|
||||
public sealed partial class FfmpegService
|
||||
{
|
||||
private const string LowStoragePauseErrorPrefix = "MP4 finalization paused because storage tier is Red (critically low).";
|
||||
private const string LowStoragePauseErrorPrefix = "MP4 finalization paused because there is not enough temporary disk space.";
|
||||
private const string ShutdownFinalizationPauseErrorPrefix = "MP4 finalization paused because the application is shutting down.";
|
||||
private const string RecorderSegmentsManifestSuffix = ".segments.json";
|
||||
private static readonly TimeSpan Mp4FinalizeInactivityTimeout = TimeSpan.FromMinutes(10);
|
||||
private static readonly TimeSpan Mp4FinalizePollInterval = TimeSpan.FromSeconds(1);
|
||||
|
||||
@@ -23,14 +27,24 @@ public sealed partial class FfmpegService
|
||||
int mp4FinalizeTimeoutMinutes,
|
||||
Guid recordSessionId,
|
||||
Guid recordTaskId,
|
||||
string sourcePath,
|
||||
IReadOnlyList<string> sourcePaths,
|
||||
string targetPath,
|
||||
double? expectedDurationSeconds,
|
||||
string? segmentsManifestPath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!File.Exists(sourcePath))
|
||||
using var shutdownLinkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _shutdownCts.Token);
|
||||
cancellationToken = shutdownLinkedCts.Token;
|
||||
var normalizedSourcePaths = sourcePaths
|
||||
.Where(static path => !string.IsNullOrWhiteSpace(path))
|
||||
.Select(NormalizeAbsolutePath)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
if (normalizedSourcePaths.Length == 0 || normalizedSourcePaths.Any(path => !File.Exists(path)))
|
||||
{
|
||||
return (File.Exists(targetPath) ? targetPath : sourcePath, "The intermediate recording file was not found for MP4 finalization.");
|
||||
return (
|
||||
File.Exists(targetPath) ? targetPath : normalizedSourcePaths.FirstOrDefault() ?? targetPath,
|
||||
"One or more intermediate recording files were not found for MP4 finalization.");
|
||||
}
|
||||
|
||||
var tempPath = Path.Combine(
|
||||
@@ -47,9 +61,9 @@ public sealed partial class FfmpegService
|
||||
using var storageScope = _serviceScopeFactory.CreateScope();
|
||||
var settingsService = storageScope.ServiceProvider.GetRequiredService<LiveRecorder.Application.Abstractions.Settings.ISystemSettingsService>();
|
||||
var settings = await settingsService.GetAsync(cancellationToken);
|
||||
var sourceSizeBytes = new FileInfo(sourcePath).Length;
|
||||
var storageCheck = _storageGuardService.CheckCanStartOrResume(settings, sourceSizeBytes);
|
||||
return storageCheck.ShouldPauseActive
|
||||
var sourceSizeBytes = normalizedSourcePaths.Sum(path => new FileInfo(path).Length);
|
||||
var storageCheck = _storageGuardService.CheckCanFinalize(settings, sourceSizeBytes);
|
||||
return !storageCheck.HasEnoughSpace
|
||||
? $"{LowStoragePauseErrorPrefix} {storageCheck.Message}"
|
||||
: null;
|
||||
}
|
||||
@@ -58,7 +72,7 @@ public sealed partial class FfmpegService
|
||||
if (!string.IsNullOrWhiteSpace(lowStoragePauseMessage))
|
||||
{
|
||||
SetPostProcessState(recordSessionId, recordTaskId, "Paused: Low storage", null, lowStoragePauseMessage);
|
||||
return (sourcePath, lowStoragePauseMessage);
|
||||
return (normalizedSourcePaths[0], lowStoragePauseMessage);
|
||||
}
|
||||
|
||||
SetPostProcessState(
|
||||
@@ -74,7 +88,7 @@ public sealed partial class FfmpegService
|
||||
if (!string.IsNullOrWhiteSpace(lowStoragePauseMessage))
|
||||
{
|
||||
SetPostProcessState(recordSessionId, recordTaskId, "Paused: Low storage", null, lowStoragePauseMessage);
|
||||
return (sourcePath, lowStoragePauseMessage);
|
||||
return (normalizedSourcePaths[0], lowStoragePauseMessage);
|
||||
}
|
||||
|
||||
var stderrLines = new Queue<string>();
|
||||
@@ -178,6 +192,10 @@ public sealed partial class FfmpegService
|
||||
|
||||
SetPostProcessState(recordSessionId, recordTaskId, stage, 0, detail);
|
||||
|
||||
var concatInputPath = normalizedSourcePaths.Length > 1
|
||||
? await WriteFfconcatInputAsync(targetPath, recordTaskId, normalizedSourcePaths, cancellationToken)
|
||||
: null;
|
||||
|
||||
using var finalizeProcess = new Process
|
||||
{
|
||||
StartInfo = new ProcessStartInfo
|
||||
@@ -190,7 +208,11 @@ public sealed partial class FfmpegService
|
||||
}
|
||||
};
|
||||
|
||||
foreach (var argument in BuildMp4FinalizeArgumentList(sourcePath, tempPath, strategy))
|
||||
foreach (var argument in BuildMp4FinalizeArgumentList(
|
||||
concatInputPath ?? normalizedSourcePaths[0],
|
||||
tempPath,
|
||||
strategy,
|
||||
concatInputPath is not null))
|
||||
{
|
||||
finalizeProcess.StartInfo.ArgumentList.Add(argument);
|
||||
}
|
||||
@@ -238,6 +260,7 @@ public sealed partial class FfmpegService
|
||||
try
|
||||
{
|
||||
finalizeProcess.Start();
|
||||
_postProcessProcesses[recordTaskId] = finalizeProcess;
|
||||
finalizeProcess.BeginOutputReadLine();
|
||||
finalizeProcess.BeginErrorReadLine();
|
||||
|
||||
@@ -278,6 +301,22 @@ public sealed partial class FfmpegService
|
||||
|
||||
return ex.Message;
|
||||
}
|
||||
catch (OperationCanceledException) when (_shutdownCts.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!finalizeProcess.HasExited)
|
||||
{
|
||||
finalizeProcess.Kill(true);
|
||||
}
|
||||
}
|
||||
catch (Exception killEx)
|
||||
{
|
||||
_logger.LogWarning(killEx, "Shutdown-interrupted MP4 finalization process could not be killed for task {RecordTaskId}", recordTaskId);
|
||||
}
|
||||
|
||||
return ShutdownFinalizationPauseErrorPrefix;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
try
|
||||
@@ -294,6 +333,14 @@ public sealed partial class FfmpegService
|
||||
|
||||
return ex.Message;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_postProcessProcesses.TryRemove(recordTaskId, out _);
|
||||
if (!string.IsNullOrWhiteSpace(concatInputPath) && File.Exists(concatInputPath))
|
||||
{
|
||||
File.Delete(concatInputPath);
|
||||
}
|
||||
}
|
||||
|
||||
if (finalizeProcess.ExitCode == 0 && File.Exists(tempPath))
|
||||
{
|
||||
@@ -344,9 +391,17 @@ public sealed partial class FfmpegService
|
||||
File.Move(tempPath, targetPath);
|
||||
}
|
||||
|
||||
if (!string.Equals(sourcePath, targetPath, StringComparison.OrdinalIgnoreCase) && File.Exists(sourcePath))
|
||||
foreach (var sourcePath in normalizedSourcePaths)
|
||||
{
|
||||
File.Delete(sourcePath);
|
||||
if (!string.Equals(sourcePath, targetPath, StringComparison.OrdinalIgnoreCase) && File.Exists(sourcePath))
|
||||
{
|
||||
File.Delete(sourcePath);
|
||||
}
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(segmentsManifestPath) && File.Exists(segmentsManifestPath))
|
||||
{
|
||||
File.Delete(segmentsManifestPath);
|
||||
}
|
||||
|
||||
SetPostProcessState(recordSessionId, recordTaskId, "Completed", 100, "MP4 seek index is ready");
|
||||
@@ -358,7 +413,7 @@ public sealed partial class FfmpegService
|
||||
File.Delete(tempPath);
|
||||
}
|
||||
|
||||
var fallbackPath = File.Exists(targetPath) ? targetPath : sourcePath;
|
||||
var fallbackPath = File.Exists(targetPath) ? targetPath : normalizedSourcePaths[0];
|
||||
string? errorDetail;
|
||||
lock (stderrLines)
|
||||
{
|
||||
@@ -426,9 +481,10 @@ public sealed partial class FfmpegService
|
||||
mp4FinalizeTimeoutMinutes,
|
||||
recordSession.Id,
|
||||
recordTask.Id,
|
||||
recorderOutputPath,
|
||||
[recorderOutputPath],
|
||||
finalOutputPath,
|
||||
expectedDurationSeconds,
|
||||
segmentsManifestPath: null,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
@@ -462,100 +518,104 @@ public sealed partial class FfmpegService
|
||||
return (File.Exists(finalSegmentOutputPath) ? finalSegmentOutputPath : recordTask.OutputFilePath, null);
|
||||
}
|
||||
|
||||
string? materializedSourcePath = null;
|
||||
var cleanupMaterializedSource = false;
|
||||
var preserveMaterializedSource = false;
|
||||
try
|
||||
string? segmentsManifestPath = null;
|
||||
if (normalizedRecorderSegmentPaths.Length > 1)
|
||||
{
|
||||
materializedSourcePath = await MaterializeRecorderSegmentSourceAsync(
|
||||
segmentsManifestPath = GetRecorderSegmentsManifestPath(finalSegmentOutputPath);
|
||||
await WriteRecorderSegmentsManifestAsync(
|
||||
segmentsManifestPath,
|
||||
finalSegmentOutputPath,
|
||||
normalizedRecorderSegmentPaths,
|
||||
cancellationToken);
|
||||
if (string.IsNullOrWhiteSpace(materializedSourcePath))
|
||||
{
|
||||
return (File.Exists(finalSegmentOutputPath) ? finalSegmentOutputPath : recordTask.OutputFilePath, null);
|
||||
}
|
||||
|
||||
cleanupMaterializedSource =
|
||||
!string.Equals(materializedSourcePath, normalizedRecorderSegmentPaths[0], StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
var finalizationResult = await TryFinalizeMp4Async(
|
||||
ffmpegPath,
|
||||
maxConcurrentTranscodeTasks,
|
||||
mp4FinalizeTimeoutMinutes,
|
||||
recordSession.Id,
|
||||
recordTask.Id,
|
||||
materializedSourcePath,
|
||||
finalSegmentOutputPath,
|
||||
expectedDurationSeconds,
|
||||
cancellationToken);
|
||||
preserveMaterializedSource = IsLowStoragePauseError(finalizationResult.ErrorMessage);
|
||||
return finalizationResult;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (cleanupMaterializedSource &&
|
||||
!preserveMaterializedSource &&
|
||||
!string.IsNullOrWhiteSpace(materializedSourcePath) &&
|
||||
File.Exists(materializedSourcePath))
|
||||
{
|
||||
File.Delete(materializedSourcePath);
|
||||
}
|
||||
}
|
||||
|
||||
return await TryFinalizeMp4Async(
|
||||
ffmpegPath,
|
||||
maxConcurrentTranscodeTasks,
|
||||
mp4FinalizeTimeoutMinutes,
|
||||
recordSession.Id,
|
||||
recordTask.Id,
|
||||
normalizedRecorderSegmentPaths,
|
||||
finalSegmentOutputPath,
|
||||
expectedDurationSeconds,
|
||||
segmentsManifestPath,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task<string?> MaterializeRecorderSegmentSourceAsync(
|
||||
private static string GetRecorderSegmentsManifestPath(string finalOutputPath) =>
|
||||
$"{NormalizeAbsolutePath(finalOutputPath)}{RecorderSegmentsManifestSuffix}";
|
||||
|
||||
private static async Task WriteRecorderSegmentsManifestAsync(
|
||||
string manifestPath,
|
||||
string finalOutputPath,
|
||||
IReadOnlyList<string> recorderSegmentPaths,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (recorderSegmentPaths.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (recorderSegmentPaths.Count == 1)
|
||||
{
|
||||
return recorderSegmentPaths[0];
|
||||
}
|
||||
|
||||
var combinedPath = Path.Combine(
|
||||
Path.GetDirectoryName(finalOutputPath)!,
|
||||
$"{Path.GetFileNameWithoutExtension(finalOutputPath)}.concat.ts");
|
||||
|
||||
if (File.Exists(combinedPath))
|
||||
{
|
||||
File.Delete(combinedPath);
|
||||
}
|
||||
|
||||
await using var outputStream = new FileStream(
|
||||
combinedPath,
|
||||
FileMode.CreateNew,
|
||||
FileAccess.Write,
|
||||
FileShare.None,
|
||||
bufferSize: 1024 * 128,
|
||||
useAsync: true);
|
||||
|
||||
foreach (var path in recorderSegmentPaths)
|
||||
{
|
||||
await using var inputStream = new FileStream(
|
||||
path,
|
||||
FileMode.Open,
|
||||
FileAccess.Read,
|
||||
FileShare.ReadWrite,
|
||||
bufferSize: 1024 * 128,
|
||||
useAsync: true);
|
||||
await inputStream.CopyToAsync(outputStream, 1024 * 128, cancellationToken);
|
||||
}
|
||||
|
||||
await outputStream.FlushAsync(cancellationToken);
|
||||
return combinedPath;
|
||||
var payload = new RecorderSegmentsManifest(
|
||||
NormalizeAbsolutePath(finalOutputPath),
|
||||
recorderSegmentPaths.Select(NormalizeAbsolutePath).ToArray(),
|
||||
DateTimeOffset.UtcNow);
|
||||
var temporaryPath = $"{manifestPath}.{Guid.NewGuid():N}.tmp";
|
||||
await File.WriteAllTextAsync(temporaryPath, JsonSerializer.Serialize(payload), Encoding.UTF8, cancellationToken);
|
||||
File.Move(temporaryPath, manifestPath, overwrite: true);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> BuildMp4FinalizeArgumentList(
|
||||
internal static IReadOnlyList<string> ReadRecorderSegmentsManifest(string finalOutputPath)
|
||||
{
|
||||
var manifestPath = GetRecorderSegmentsManifestPath(finalOutputPath);
|
||||
if (!File.Exists(manifestPath))
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var manifest = JsonSerializer.Deserialize<RecorderSegmentsManifest>(File.ReadAllText(manifestPath));
|
||||
return manifest?.SourcePaths
|
||||
.Where(static path => !string.IsNullOrWhiteSpace(path))
|
||||
.Select(NormalizeAbsolutePath)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Where(File.Exists)
|
||||
.ToArray() ?? [];
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<string> WriteFfconcatInputAsync(
|
||||
string targetPath,
|
||||
Guid recordTaskId,
|
||||
IReadOnlyList<string> sourcePaths,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var concatPath = Path.Combine(
|
||||
Path.GetDirectoryName(targetPath)!,
|
||||
$".{Path.GetFileName(targetPath)}.{recordTaskId:N}.ffconcat");
|
||||
var content = new StringBuilder("ffconcat version 1.0\n");
|
||||
foreach (var sourcePath in sourcePaths)
|
||||
{
|
||||
content.Append("file '")
|
||||
.Append(NormalizeAbsolutePath(sourcePath)
|
||||
.Replace("\\", "\\\\", StringComparison.Ordinal)
|
||||
.Replace("'", "\\'", StringComparison.Ordinal))
|
||||
.Append("'\n");
|
||||
}
|
||||
|
||||
await File.WriteAllTextAsync(concatPath, content.ToString(), Encoding.UTF8, cancellationToken);
|
||||
return concatPath;
|
||||
}
|
||||
|
||||
internal static IReadOnlyList<string> BuildMp4FinalizeArgumentList(
|
||||
string sourcePath,
|
||||
string targetPath,
|
||||
Mp4FinalizeStrategy strategy)
|
||||
Mp4FinalizeStrategy strategy,
|
||||
bool useConcatDemuxer = false)
|
||||
{
|
||||
var arguments = new List<string>
|
||||
{
|
||||
@@ -571,16 +631,24 @@ public sealed partial class FfmpegService
|
||||
"-fflags",
|
||||
"+genpts+igndts+discardcorrupt",
|
||||
"-err_detect",
|
||||
"ignore_err",
|
||||
"-i",
|
||||
sourcePath,
|
||||
"ignore_err"
|
||||
};
|
||||
|
||||
if (useConcatDemuxer)
|
||||
{
|
||||
arguments.AddRange(["-f", "concat", "-safe", "0"]);
|
||||
}
|
||||
|
||||
arguments.AddRange(
|
||||
[
|
||||
"-i", sourcePath,
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-map",
|
||||
"0:a:0?",
|
||||
"-dn",
|
||||
"-sn"
|
||||
};
|
||||
]);
|
||||
|
||||
if (strategy == Mp4FinalizeStrategy.RepairTranscode)
|
||||
{
|
||||
@@ -610,7 +678,7 @@ public sealed partial class FfmpegService
|
||||
return arguments;
|
||||
}
|
||||
|
||||
private static bool IsRepairableMp4FinalizeError(string errorDetail)
|
||||
internal static bool IsRepairableMp4FinalizeError(string errorDetail)
|
||||
{
|
||||
if (IsLowStoragePauseError(errorDetail))
|
||||
{
|
||||
@@ -622,25 +690,37 @@ public sealed partial class FfmpegService
|
||||
errorDetail.Contains("incorrect codec parameters", StringComparison.OrdinalIgnoreCase) ||
|
||||
errorDetail.Contains("codec parameters", StringComparison.OrdinalIgnoreCase) ||
|
||||
errorDetail.Contains("Invalid data found when processing input", StringComparison.OrdinalIgnoreCase) ||
|
||||
errorDetail.Contains("Error initializing output stream", StringComparison.OrdinalIgnoreCase);
|
||||
errorDetail.Contains("Error initializing output stream", StringComparison.OrdinalIgnoreCase) ||
|
||||
errorDetail.Contains("Error writing trailer", StringComparison.OrdinalIgnoreCase) ||
|
||||
errorDetail.Contains("Error muxing a packet", StringComparison.OrdinalIgnoreCase) ||
|
||||
errorDetail.Contains("Conversion failed", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static bool IsLowStoragePauseError(string? errorDetail) =>
|
||||
!string.IsNullOrWhiteSpace(errorDetail) &&
|
||||
errorDetail.Contains(LowStoragePauseErrorPrefix, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static bool IsShutdownFinalizationPauseError(string? errorDetail) =>
|
||||
!string.IsNullOrWhiteSpace(errorDetail) &&
|
||||
errorDetail.Contains(ShutdownFinalizationPauseErrorPrefix, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static bool IsNoSpaceLeftError(string? errorDetail) =>
|
||||
!string.IsNullOrWhiteSpace(errorDetail) &&
|
||||
(errorDetail.Contains("No space left on device", StringComparison.OrdinalIgnoreCase) ||
|
||||
errorDetail.Contains("database or disk is full", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
private enum Mp4FinalizeStrategy
|
||||
internal enum Mp4FinalizeStrategy
|
||||
{
|
||||
StreamCopy,
|
||||
RepairTranscode
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> BuildArgumentList(
|
||||
private sealed record RecorderSegmentsManifest(
|
||||
string TargetPath,
|
||||
IReadOnlyList<string> SourcePaths,
|
||||
DateTimeOffset CreatedAt);
|
||||
|
||||
internal static IReadOnlyList<string> BuildArgumentList(
|
||||
string streamUrl,
|
||||
string outputFilePath,
|
||||
RecordOutputFormat outputFormat,
|
||||
@@ -654,13 +734,15 @@ public sealed partial class FfmpegService
|
||||
StreamInputHeaders? inputHeaders,
|
||||
string? selectedProtocol,
|
||||
string? selectedVideoCodec,
|
||||
FfmpegInputOptionProfile inputOptionProfile)
|
||||
FfmpegInputOptionProfile inputOptionProfile,
|
||||
RecoveryVideoEncoderSelection? recoveryVideoEncoder = null)
|
||||
{
|
||||
var arguments = new List<string> { "-hide_banner", "-y", "-progress", "pipe:1" };
|
||||
var arguments = new List<string> { "-hide_banner", "-n", "-progress", "pipe:1" };
|
||||
recoveryVideoEncoder ??= RecoveryVideoEncoderSelection.Software;
|
||||
var useIntermediateTransportStream = ShouldUseIntermediateTransportStream(outputFilePath, outputFormat, saveMode);
|
||||
|
||||
var effectiveStreamUrl = streamUrl;
|
||||
var isPipeInput = IsHttpInput(streamUrl);
|
||||
var isPipeInput = ShouldUseCurlPipe(streamUrl, selectedProtocol);
|
||||
if (isPipeInput)
|
||||
{
|
||||
// When the input is HTTP, use pipe:0 so that curl handles the HTTP connection.
|
||||
@@ -671,7 +753,7 @@ public sealed partial class FfmpegService
|
||||
}
|
||||
|
||||
if (enableReconnect &&
|
||||
inputOptionProfile == FfmpegInputOptionProfile.Baseline &&
|
||||
inputOptionProfile != FfmpegInputOptionProfile.Minimal &&
|
||||
!isPipeInput &&
|
||||
ShouldEnableReconnect(streamUrl, selectedProtocol))
|
||||
{
|
||||
@@ -684,8 +766,21 @@ public sealed partial class FfmpegService
|
||||
]);
|
||||
}
|
||||
|
||||
arguments.AddRange(["-rw_timeout", readWriteTimeoutMilliseconds.ToString(), "-fflags", "+discardcorrupt+genpts", "-i", effectiveStreamUrl]);
|
||||
arguments.AddRange(BuildCodecArguments(recordingTemplate));
|
||||
if (!isPipeInput)
|
||||
{
|
||||
AddNativeHttpInputHeaders(arguments, streamUrl, inputHeaders);
|
||||
}
|
||||
|
||||
if (inputOptionProfile == FfmpegInputOptionProfile.TimestampTranscode)
|
||||
{
|
||||
AddRecoveryEncoderDeviceArguments(arguments, recoveryVideoEncoder);
|
||||
}
|
||||
|
||||
var inputFlags = inputOptionProfile is FfmpegInputOptionProfile.TimestampRepair or FfmpegInputOptionProfile.TimestampTranscode
|
||||
? "+discardcorrupt+genpts+igndts"
|
||||
: "+discardcorrupt+genpts";
|
||||
arguments.AddRange(["-rw_timeout", readWriteTimeoutMilliseconds.ToString(), "-fflags", inputFlags, "-i", effectiveStreamUrl]);
|
||||
arguments.AddRange(BuildCodecArguments(recordingTemplate, inputOptionProfile, recoveryVideoEncoder));
|
||||
|
||||
var writesTransportStream = useIntermediateTransportStream || outputFormat == RecordOutputFormat.Ts;
|
||||
var bitstreamFilter = ResolveTransportStreamBitstreamFilter(recordingTemplate, writesTransportStream, selectedVideoCodec);
|
||||
@@ -728,8 +823,63 @@ public sealed partial class FfmpegService
|
||||
return arguments;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> BuildCodecArguments(RecordingTemplateType recordingTemplate) =>
|
||||
recordingTemplate switch
|
||||
private static IReadOnlyList<string> BuildCodecArguments(
|
||||
RecordingTemplateType recordingTemplate,
|
||||
FfmpegInputOptionProfile inputOptionProfile,
|
||||
RecoveryVideoEncoderSelection recoveryVideoEncoder)
|
||||
{
|
||||
if (inputOptionProfile == FfmpegInputOptionProfile.TimestampTranscode)
|
||||
{
|
||||
var arguments = new List<string>
|
||||
{
|
||||
"-map", "0:v:0",
|
||||
"-map", "0:a:0?",
|
||||
};
|
||||
arguments.AddRange(BuildRecoveryVideoCodecArguments(recoveryVideoEncoder));
|
||||
arguments.AddRange(
|
||||
[
|
||||
"-af", "aresample=async=1:first_pts=0,asetpts=PTS-STARTPTS",
|
||||
"-c:a", "aac",
|
||||
"-b:a", "128k",
|
||||
"-avoid_negative_ts", "make_zero"
|
||||
]);
|
||||
return arguments;
|
||||
}
|
||||
|
||||
if (inputOptionProfile == FfmpegInputOptionProfile.TimestampRepair)
|
||||
{
|
||||
return recordingTemplate switch
|
||||
{
|
||||
RecordingTemplateType.BalancedMp4 =>
|
||||
[
|
||||
"-c:v", "libx264",
|
||||
"-preset", "veryfast",
|
||||
"-crf", "23",
|
||||
"-c:a", "aac",
|
||||
"-af", "aresample=async=1:first_pts=0",
|
||||
"-b:a", "128k"
|
||||
],
|
||||
RecordingTemplateType.ArchiveTs =>
|
||||
[
|
||||
"-map", "0",
|
||||
"-c", "copy",
|
||||
"-c:a", "aac",
|
||||
"-af", "aresample=async=1:first_pts=0",
|
||||
"-b:a", "128k"
|
||||
],
|
||||
_ =>
|
||||
[
|
||||
"-map", "0:v:0",
|
||||
"-map", "0:a:0?",
|
||||
"-c:v", "copy",
|
||||
"-c:a", "aac",
|
||||
"-af", "aresample=async=1:first_pts=0",
|
||||
"-b:a", "128k"
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
return recordingTemplate switch
|
||||
{
|
||||
RecordingTemplateType.BalancedMp4 =>
|
||||
[
|
||||
@@ -749,6 +899,66 @@ public sealed partial class FfmpegService
|
||||
"-c", "copy"
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
internal static void AddRecoveryEncoderDeviceArguments(
|
||||
ICollection<string> arguments,
|
||||
RecoveryVideoEncoderSelection encoder)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(encoder.DevicePath))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (encoder.Kind == RecoveryVideoEncoderKind.Qsv)
|
||||
{
|
||||
arguments.Add("-qsv_device");
|
||||
arguments.Add(encoder.DevicePath);
|
||||
}
|
||||
else if (encoder.Kind == RecoveryVideoEncoderKind.Vaapi)
|
||||
{
|
||||
arguments.Add("-vaapi_device");
|
||||
arguments.Add(encoder.DevicePath);
|
||||
}
|
||||
}
|
||||
|
||||
internal static IReadOnlyList<string> BuildRecoveryVideoCodecArguments(
|
||||
RecoveryVideoEncoderSelection encoder) =>
|
||||
encoder.Kind switch
|
||||
{
|
||||
RecoveryVideoEncoderKind.Nvenc =>
|
||||
[
|
||||
"-vf", "settb=AVTB,setpts=PTS-STARTPTS,format=yuv420p",
|
||||
"-c:v", "h264_nvenc",
|
||||
"-preset", "p4",
|
||||
"-cq", "23",
|
||||
"-b:v", "0",
|
||||
"-fps_mode", "vfr"
|
||||
],
|
||||
RecoveryVideoEncoderKind.Qsv =>
|
||||
[
|
||||
"-vf", "settb=AVTB,setpts=PTS-STARTPTS,format=nv12,hwupload=extra_hw_frames=64",
|
||||
"-c:v", "h264_qsv",
|
||||
"-preset", "veryfast",
|
||||
"-global_quality", "23",
|
||||
"-fps_mode", "vfr"
|
||||
],
|
||||
RecoveryVideoEncoderKind.Vaapi =>
|
||||
[
|
||||
"-vf", "settb=AVTB,setpts=PTS-STARTPTS,format=nv12,hwupload",
|
||||
"-c:v", "h264_vaapi",
|
||||
"-qp", "23",
|
||||
"-fps_mode", "vfr"
|
||||
],
|
||||
_ =>
|
||||
[
|
||||
"-vf", "settb=AVTB,setpts=PTS-STARTPTS",
|
||||
"-c:v", "libx264",
|
||||
"-preset", "veryfast",
|
||||
"-crf", "23",
|
||||
"-fps_mode", "vfr"
|
||||
]
|
||||
};
|
||||
|
||||
private static string? ResolveTransportStreamBitstreamFilter(
|
||||
RecordingTemplateType recordingTemplate,
|
||||
@@ -821,6 +1031,51 @@ public sealed partial class FfmpegService
|
||||
return false;
|
||||
}
|
||||
|
||||
private async Task<MediaArtifactValidation> ValidateMediaArtifactAsync(
|
||||
string? outputPath,
|
||||
RecordOutputFormat outputFormat,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(outputPath) || !File.Exists(outputPath))
|
||||
{
|
||||
return MediaArtifactValidation.Invalid("The recorded media file does not exist.");
|
||||
}
|
||||
|
||||
var fileInfo = new FileInfo(outputPath);
|
||||
if (fileInfo.Length <= 0)
|
||||
{
|
||||
return MediaArtifactValidation.Invalid("The recorded media file is empty.");
|
||||
}
|
||||
|
||||
if (outputFormat == RecordOutputFormat.Mp4 &&
|
||||
!outputPath.EndsWith(".mp4", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return MediaArtifactValidation.Invalid(
|
||||
"MP4 finalization did not produce an MP4 file. The intermediate recording was kept locally.");
|
||||
}
|
||||
|
||||
using var scope = _serviceScopeFactory.CreateScope();
|
||||
var metadataService = scope.ServiceProvider.GetRequiredService<IVideoMetadataService>();
|
||||
var metadata = await metadataService.ExtractMetadataAsync(outputPath, cancellationToken);
|
||||
if (metadata is null)
|
||||
{
|
||||
return MediaArtifactValidation.Invalid(FfprobeUnreadableArtifactError);
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(metadata.VideoCodec))
|
||||
{
|
||||
return MediaArtifactValidation.Invalid("The recorded media file does not contain a video stream.", metadata.DurationSeconds);
|
||||
}
|
||||
|
||||
if (!metadata.DurationSeconds.HasValue ||
|
||||
metadata.DurationSeconds.Value < MinimumUnexpectedExitArtifactDuration.TotalSeconds)
|
||||
{
|
||||
return MediaArtifactValidation.Invalid(ShortUnexpectedExitArtifactError, metadata.DurationSeconds);
|
||||
}
|
||||
|
||||
return new MediaArtifactValidation(true, metadata.DurationSeconds, null);
|
||||
}
|
||||
|
||||
private static async Task UpsertRecordResultAsync(
|
||||
RecordTask recordTask,
|
||||
LiveRecorderDbContext dbContext,
|
||||
@@ -830,6 +1085,7 @@ public sealed partial class FfmpegService
|
||||
string? danmakuFilePath,
|
||||
int danmakuMessageCount,
|
||||
DateTimeOffset endedAt,
|
||||
bool mediaValidatedForDispatch = false,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(effectiveOutputPath))
|
||||
@@ -838,8 +1094,13 @@ public sealed partial class FfmpegService
|
||||
}
|
||||
|
||||
var resultId = Guid.NewGuid();
|
||||
var dispatchId = Guid.NewGuid();
|
||||
var normalizedDanmakuCount = Math.Max(0, danmakuMessageCount);
|
||||
var finalStatus = (int)recordTask.Status;
|
||||
var shouldCreateCompletionDispatch = IsCompletionDispatchEligible(
|
||||
recordTask,
|
||||
effectiveOutputPath,
|
||||
fileSize) && mediaValidatedForDispatch;
|
||||
|
||||
// Multiple background paths can reconcile the same segment after ffmpeg exits.
|
||||
// Use the database's atomic upsert instead of EF Add-or-Update to avoid
|
||||
@@ -859,9 +1120,32 @@ public sealed partial class FfmpegService
|
||||
"ErrorMessage" = excluded."ErrorMessage",
|
||||
"UploadStatus" = COALESCE("RecordResults"."UploadStatus", excluded."UploadStatus"),
|
||||
"DeletedLocalFilesAfterUpload" = COALESCE("RecordResults"."DeletedLocalFilesAfterUpload", excluded."DeletedLocalFilesAfterUpload");
|
||||
|
||||
INSERT INTO "RecordCompletionDispatches"
|
||||
("Id", "RecordTaskId", "ScriptDispatched", "UploadDispatched", "AttemptCount", "LastError", "CreatedAt", "UpdatedAt", "NextAttemptAt", "CompletedAt")
|
||||
SELECT
|
||||
{dispatchId}, {recordTask.Id}, FALSE, FALSE, 0, NULL, {endedAt}, {endedAt}, {endedAt}, NULL
|
||||
WHERE {shouldCreateCompletionDispatch}
|
||||
ON CONFLICT("RecordTaskId") DO NOTHING;
|
||||
""", cancellationToken);
|
||||
}
|
||||
|
||||
internal static bool IsCompletionDispatchEligible(
|
||||
RecordTask recordTask,
|
||||
string? effectiveOutputPath,
|
||||
long? fileSize)
|
||||
{
|
||||
if (recordTask.Status is not (RecordTaskStatus.Completed or RecordTaskStatus.Stopped) ||
|
||||
string.IsNullOrWhiteSpace(effectiveOutputPath) ||
|
||||
(recordTask.OutputFormat == RecordOutputFormat.Mp4 &&
|
||||
!effectiveOutputPath.EndsWith(".mp4", StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return HasUsableOutput(effectiveOutputPath, fileSize);
|
||||
}
|
||||
|
||||
private static Task<RecordResult?> LoadRecordResultAsync(
|
||||
LiveRecorderDbContext dbContext,
|
||||
Guid recordTaskId,
|
||||
@@ -931,6 +1215,43 @@ public sealed partial class FfmpegService
|
||||
streamUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ||
|
||||
streamUrl.StartsWith("https://", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
internal static bool ShouldUseCurlPipe(string streamUrl, string? selectedProtocol) =>
|
||||
IsHttpInput(streamUrl) && !IsHlsInput(streamUrl, selectedProtocol);
|
||||
|
||||
private static bool IsHlsInput(string streamUrl, string? selectedProtocol) =>
|
||||
string.Equals(selectedProtocol, "hls", StringComparison.OrdinalIgnoreCase) ||
|
||||
streamUrl.Contains(".m3u8", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static void AddNativeHttpInputHeaders(
|
||||
ICollection<string> arguments,
|
||||
string streamUrl,
|
||||
StreamInputHeaders? inputHeaders)
|
||||
{
|
||||
if (!IsHttpInput(streamUrl) || inputHeaders is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(inputHeaders.UserAgent))
|
||||
{
|
||||
arguments.Add("-user_agent");
|
||||
arguments.Add(inputHeaders.UserAgent.Trim());
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(inputHeaders.Referer))
|
||||
{
|
||||
arguments.Add("-referer");
|
||||
arguments.Add(inputHeaders.Referer.Trim());
|
||||
}
|
||||
|
||||
var customHeaders = BuildCustomHeaderArgument(inputHeaders);
|
||||
if (!string.IsNullOrWhiteSpace(customHeaders))
|
||||
{
|
||||
arguments.Add("-headers");
|
||||
arguments.Add(customHeaders);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool ShouldEnableReconnect(string streamUrl, string? selectedProtocol)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(selectedProtocol) &&
|
||||
@@ -1134,6 +1455,9 @@ public sealed partial class FfmpegService
|
||||
private static bool IsActiveSessionStatus(RecordSessionStatus status) =>
|
||||
status is RecordSessionStatus.Starting or RecordSessionStatus.Running or RecordSessionStatus.Stopping;
|
||||
|
||||
private static bool IsTerminalSessionStatus(RecordSessionStatus status) =>
|
||||
status is RecordSessionStatus.Completed or RecordSessionStatus.Failed or RecordSessionStatus.Stopped;
|
||||
|
||||
private static bool TryParseFfmpegProgressSeconds(string line, out double seconds)
|
||||
{
|
||||
if (line.StartsWith("out_time=", StringComparison.OrdinalIgnoreCase))
|
||||
@@ -1163,16 +1487,25 @@ public sealed partial class FfmpegService
|
||||
return false;
|
||||
}
|
||||
|
||||
private enum FfmpegInputOptionProfile
|
||||
internal enum FfmpegInputOptionProfile
|
||||
{
|
||||
Baseline = 0,
|
||||
Minimal = 1
|
||||
Minimal = 1,
|
||||
TimestampRepair = 2,
|
||||
TimestampTranscode = 3
|
||||
}
|
||||
|
||||
private sealed record MediaArtifactValidation(bool IsValid, double? DurationSeconds, string? ErrorMessage)
|
||||
{
|
||||
public static MediaArtifactValidation Invalid(string errorMessage, double? durationSeconds = null) =>
|
||||
new(false, durationSeconds, errorMessage);
|
||||
}
|
||||
|
||||
private enum StartupFailureKind
|
||||
{
|
||||
None = 0,
|
||||
InputOptionCompatibility = 1,
|
||||
StreamHandshake = 2
|
||||
StreamHandshake = 2,
|
||||
HardwareEncoderUnavailable = 3
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ using LiveRecorder.Application.Abstractions.Logging;
|
||||
using LiveRecorder.Application.Abstractions.Notifications;
|
||||
using LiveRecorder.Application.Abstractions.Platforms;
|
||||
using LiveRecorder.Application.Abstractions.Recording;
|
||||
using LiveRecorder.Application.Abstractions.Scripting;
|
||||
using LiveRecorder.Application.Abstractions.Settings;
|
||||
using LiveRecorder.Application.Abstractions.Storage;
|
||||
using LiveRecorder.Application.Models.Media;
|
||||
@@ -27,14 +28,23 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
private static readonly TimeSpan StartupFailureNotificationCooldown = TimeSpan.FromMinutes(30);
|
||||
private static readonly TimeSpan StartupFailureMaxBackoff = TimeSpan.FromMinutes(15);
|
||||
private static readonly TimeSpan StartupFailureBaseBackoff = TimeSpan.FromSeconds(30);
|
||||
private static readonly TimeSpan RuntimeFailureMaxBackoff = TimeSpan.FromMinutes(5);
|
||||
private static readonly TimeSpan RuntimeFailureBaseBackoff = TimeSpan.FromSeconds(15);
|
||||
|
||||
private readonly ConcurrentDictionary<Guid, SessionProcessRuntime> _processes = new();
|
||||
private readonly ConcurrentDictionary<Guid, PostProcessRuntimeEntry> _postProcessStates = new();
|
||||
private readonly ConcurrentDictionary<Guid, Process> _postProcessProcesses = new();
|
||||
private readonly ConcurrentDictionary<Guid, RoomStartupFailureState> _roomStartupFailureStates = new();
|
||||
private readonly ConcurrentDictionary<Guid, RoomStartupFailureState> _roomRuntimeFailureStates = new();
|
||||
private readonly SemaphoreSlim _recoveryEncoderProbeGate = new(1, 1);
|
||||
private readonly object _transcodeConcurrencyLock = new();
|
||||
private readonly SemaphoreSlim _orphanRecoveryGate = new(1, 1);
|
||||
private readonly Queue<TaskCompletionSource<IDisposable>> _transcodeWaiters = new();
|
||||
private readonly CancellationTokenSource _shutdownCts = new();
|
||||
private int _activeTranscodeTasks;
|
||||
private int _maxConcurrentTranscodeTasks = 1;
|
||||
private bool _hasProbedRecoveryVideoEncoder;
|
||||
private RecoveryVideoEncoderSelection _cachedRecoveryVideoEncoder = RecoveryVideoEncoderSelection.Software;
|
||||
private readonly IServiceScopeFactory _serviceScopeFactory;
|
||||
private readonly IStorageGuardService _storageGuardService;
|
||||
private readonly ILiveRoomPollingSignal _liveRoomPollingSignal;
|
||||
@@ -87,10 +97,7 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
initialTask,
|
||||
streamUrlResult,
|
||||
recordingSettings,
|
||||
inputOptionProfile: FfmpegInputOptionProfile.Baseline,
|
||||
hasRetriedWithCompatibilityProfile: false,
|
||||
hasRetriedWithRefreshedStream: false,
|
||||
retryAttemptCount: 0,
|
||||
InitialRecoveryContext,
|
||||
cancellationToken);
|
||||
|
||||
private async Task StartInternalAsync(
|
||||
@@ -98,10 +105,7 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
RecordTask initialTask,
|
||||
StreamUrlResult streamUrlResult,
|
||||
RecordingExecutionSettings recordingSettings,
|
||||
FfmpegInputOptionProfile inputOptionProfile,
|
||||
bool hasRetriedWithCompatibilityProfile,
|
||||
bool hasRetriedWithRefreshedStream,
|
||||
int retryAttemptCount,
|
||||
FfmpegRecoveryContext recoveryContext,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(recordSession);
|
||||
@@ -117,6 +121,9 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
using var settingsScope = _serviceScopeFactory.CreateScope();
|
||||
var settingsService = settingsScope.ServiceProvider.GetRequiredService<ISystemSettingsService>();
|
||||
var settings = await settingsService.GetAsync(cancellationToken);
|
||||
var recoveryVideoEncoder = recoveryContext.InputOptionProfile == FfmpegInputOptionProfile.TimestampTranscode
|
||||
? await ResolveRecoveryVideoEncoderAsync(settings.FfmpegPath, recoveryContext.ForceSoftwareEncoder, cancellationToken)
|
||||
: RecoveryVideoEncoderSelection.Software;
|
||||
|
||||
var outputPathPattern = Path.IsPathRooted(recordSession.OutputPathPattern)
|
||||
? recordSession.OutputPathPattern
|
||||
@@ -154,10 +161,8 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
streamUrlResult.SelectedVideoCodec,
|
||||
streamUrlResult.InputHeaders,
|
||||
recordingSettings,
|
||||
inputOptionProfile,
|
||||
hasRetriedWithCompatibilityProfile,
|
||||
hasRetriedWithRefreshedStream,
|
||||
retryAttemptCount);
|
||||
recoveryContext,
|
||||
recoveryVideoEncoder);
|
||||
|
||||
foreach (var argument in BuildArgumentList(
|
||||
streamUrlResult.SelectedUrl,
|
||||
@@ -173,7 +178,8 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
streamUrlResult.InputHeaders,
|
||||
streamUrlResult.SelectedProtocol,
|
||||
streamUrlResult.SelectedVideoCodec,
|
||||
inputOptionProfile))
|
||||
recoveryContext.InputOptionProfile,
|
||||
recoveryVideoEncoder))
|
||||
{
|
||||
process.StartInfo.ArgumentList.Add(argument);
|
||||
}
|
||||
@@ -199,7 +205,7 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
// pipe its stdout into FFmpeg's stdin. This bypasses FFmpeg's built-in HTTP
|
||||
// handler which has a hard-coded 4096-byte response header buffer that triggers
|
||||
// "overlong headers" errors with CDNs that return oversized headers.
|
||||
if (IsHttpInput(streamUrlResult.SelectedUrl))
|
||||
if (ShouldUseCurlPipe(streamUrlResult.SelectedUrl, streamUrlResult.SelectedProtocol))
|
||||
{
|
||||
var curlProcess = new Process
|
||||
{
|
||||
@@ -295,6 +301,21 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
}
|
||||
|
||||
var process = runtime.Process;
|
||||
var curlProcess = runtime.CurlProcess;
|
||||
if (curlProcess is not null)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!curlProcess.HasExited)
|
||||
{
|
||||
curlProcess.Kill(true);
|
||||
}
|
||||
}
|
||||
catch (InvalidOperationException)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
if (process is not null)
|
||||
{
|
||||
try
|
||||
@@ -313,7 +334,178 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
return await WaitForExitAsync(runtime, timeout, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<bool> TryReconcileInactiveSessionAsync(Guid recordSessionId, CancellationToken cancellationToken = default)
|
||||
public async Task<int> StopAllAndWaitAsync(
|
||||
TimeSpan gracefulTimeout,
|
||||
TimeSpan forceKillTimeout,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var runtimes = _processes.Values.ToArray();
|
||||
foreach (var runtime in runtimes)
|
||||
{
|
||||
// Mark the captured runtime before looking it up again. The process may exit
|
||||
// between the snapshot and the stop signal, but its exit handler must still
|
||||
// observe that this was an application shutdown rather than a stream failure.
|
||||
runtime.MarkShutdownRequested();
|
||||
}
|
||||
|
||||
await Task.WhenAll(runtimes.Select(runtime =>
|
||||
RequestStopAsync(
|
||||
runtime.RecordSessionId,
|
||||
markAsCompletedOnExit: true,
|
||||
cancellationToken,
|
||||
shutdownRequested: true)));
|
||||
|
||||
await WaitForRuntimeCompletionsAsync(runtimes, gracefulTimeout, cancellationToken);
|
||||
|
||||
foreach (var runtime in runtimes.Where(static runtime => !runtime.ExitCompletion.Task.IsCompleted))
|
||||
{
|
||||
var process = runtime.Process;
|
||||
try
|
||||
{
|
||||
if (process is not null && !process.HasExited)
|
||||
{
|
||||
runtime.MarkForceKilled();
|
||||
process.Kill(true);
|
||||
}
|
||||
}
|
||||
catch (InvalidOperationException)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
await WaitForRuntimeCompletionsAsync(runtimes, forceKillTimeout, CancellationToken.None);
|
||||
|
||||
_shutdownCts.Cancel();
|
||||
foreach (var process in _postProcessProcesses.Values.ToArray())
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!process.HasExited)
|
||||
{
|
||||
process.Kill(true);
|
||||
}
|
||||
}
|
||||
catch (InvalidOperationException)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
await WaitForAllProcessesAsync(forceKillTimeout, CancellationToken.None);
|
||||
return runtimes.Length;
|
||||
}
|
||||
|
||||
private static async Task WaitForRuntimeCompletionsAsync(
|
||||
IReadOnlyCollection<SessionProcessRuntime> runtimes,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (runtimes.Count == 0 || runtimes.All(static runtime => runtime.ExitCompletion.Task.IsCompleted))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var completionTask = Task.WhenAll(runtimes.Select(static runtime => runtime.ExitCompletion.Task));
|
||||
using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
var delayTask = Task.Delay(timeout, timeoutCts.Token);
|
||||
if (await Task.WhenAny(completionTask, delayTask) == completionTask)
|
||||
{
|
||||
timeoutCts.Cancel();
|
||||
await completionTask;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task WaitForAllProcessesAsync(TimeSpan timeout, CancellationToken cancellationToken)
|
||||
{
|
||||
var deadline = DateTimeOffset.UtcNow + timeout;
|
||||
while ((!_processes.IsEmpty || !_postProcessProcesses.IsEmpty || !_postProcessStates.IsEmpty) &&
|
||||
DateTimeOffset.UtcNow < deadline)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(200), cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
public Task<bool> TryReconcileInactiveSessionAsync(Guid recordSessionId, CancellationToken cancellationToken = default) =>
|
||||
ReconcileInactiveSessionAsync(recordSessionId, allowTerminalSession: false, cancellationToken);
|
||||
|
||||
public async Task<int> RecoverOrphanedTerminalSessionTasksAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (!await _orphanRecoveryGate.WaitAsync(0, cancellationToken))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using var scope = _serviceScopeFactory.CreateScope();
|
||||
var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
|
||||
var candidateIds = await dbContext.RecordSessions
|
||||
.AsNoTracking()
|
||||
.Where(session =>
|
||||
(session.Status == RecordSessionStatus.Completed ||
|
||||
session.Status == RecordSessionStatus.Failed ||
|
||||
session.Status == RecordSessionStatus.Stopped) &&
|
||||
session.RecordTasks.Any(task =>
|
||||
task.Status == RecordTaskStatus.Starting ||
|
||||
task.Status == RecordTaskStatus.Running ||
|
||||
task.Status == RecordTaskStatus.Stopping))
|
||||
.OrderBy(session => session.UpdatedAt)
|
||||
.Select(session => session.Id)
|
||||
.Take(20)
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
var recovered = 0;
|
||||
foreach (var candidateId in candidateIds)
|
||||
{
|
||||
if (await ReconcileInactiveSessionAsync(candidateId, allowTerminalSession: true, cancellationToken))
|
||||
{
|
||||
recovered++;
|
||||
}
|
||||
}
|
||||
|
||||
return recovered;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_orphanRecoveryGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<bool> TryRecoverOrphanedTerminalTaskAsync(
|
||||
Guid recordTaskId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await _orphanRecoveryGate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
using var lookupScope = _serviceScopeFactory.CreateScope();
|
||||
var lookupDbContext = lookupScope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
|
||||
var sessionId = await lookupDbContext.RecordTasks
|
||||
.AsNoTracking()
|
||||
.Where(task =>
|
||||
task.Id == recordTaskId &&
|
||||
(task.Status == RecordTaskStatus.Starting ||
|
||||
task.Status == RecordTaskStatus.Running ||
|
||||
task.Status == RecordTaskStatus.Stopping) &&
|
||||
task.RecordSession != null &&
|
||||
(task.RecordSession.Status == RecordSessionStatus.Completed ||
|
||||
task.RecordSession.Status == RecordSessionStatus.Failed ||
|
||||
task.RecordSession.Status == RecordSessionStatus.Stopped))
|
||||
.Select(task => (Guid?)task.RecordSessionId)
|
||||
.FirstOrDefaultAsync(cancellationToken);
|
||||
|
||||
return sessionId.HasValue &&
|
||||
await ReconcileInactiveSessionAsync(sessionId.Value, allowTerminalSession: true, cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_orphanRecoveryGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<bool> ReconcileInactiveSessionAsync(
|
||||
Guid recordSessionId,
|
||||
bool allowTerminalSession,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (IsRunning(recordSessionId))
|
||||
{
|
||||
@@ -330,11 +522,53 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
.ThenInclude(item => item.Result)
|
||||
.FirstOrDefaultAsync(item => item.Id == recordSessionId, cancellationToken);
|
||||
|
||||
if (recordSession is null || !IsActiveSessionStatus(recordSession.Status))
|
||||
if (recordSession is null ||
|
||||
(!IsActiveSessionStatus(recordSession.Status) &&
|
||||
!(allowTerminalSession && IsTerminalSessionStatus(recordSession.Status))))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var hasActiveTasks = recordSession.RecordTasks.Any(task => IsActiveTaskStatus(task.Status));
|
||||
if (allowTerminalSession && !hasActiveTasks)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var discoveredSegments = DiscoverRecoverableSegments(recordSession.OutputPathPattern, recordSession.OutputFormat, recordSession.SaveMode);
|
||||
var existingSegmentIndexes = recordSession.RecordTasks
|
||||
.Select(static task => task.SegmentIndex)
|
||||
.ToHashSet();
|
||||
var addedMissingTasks = false;
|
||||
foreach (var segment in discoveredSegments.Where(segment => !existingSegmentIndexes.Contains(segment.SegmentIndex)))
|
||||
{
|
||||
var discoveredAt = File.GetLastWriteTimeUtc(segment.RecorderPath);
|
||||
var createdAt = discoveredAt == DateTime.MinValue
|
||||
? DateTimeOffset.UtcNow
|
||||
: new DateTimeOffset(DateTime.SpecifyKind(discoveredAt, DateTimeKind.Utc));
|
||||
var missingTask = new RecordTask(
|
||||
recordSession.LiveRoomId,
|
||||
recordSession.Id,
|
||||
segment.SegmentIndex,
|
||||
recordSession.PreferredQuality,
|
||||
recordSession.OutputFormat,
|
||||
createdAt);
|
||||
var recoveryStartedAt = DateTimeOffset.UtcNow;
|
||||
missingTask.MarkStarting(recordSession.StreamUrl ?? string.Empty, segment.OutputPath, recoveryStartedAt);
|
||||
missingTask.MarkRunning(recoveryStartedAt);
|
||||
await dbContext.RecordTasks.AddAsync(missingTask, cancellationToken);
|
||||
recordSession.RecordTasks.Add(missingTask);
|
||||
existingSegmentIndexes.Add(segment.SegmentIndex);
|
||||
addedMissingTasks = true;
|
||||
}
|
||||
|
||||
// RecordResult is upserted with raw SQL below, so newly discovered tasks must
|
||||
// exist first to satisfy the RecordTaskId foreign key.
|
||||
if (addedMissingTasks)
|
||||
{
|
||||
await dbContext.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
var settings = await settingsService.GetAsync(cancellationToken);
|
||||
var endedAt = DateTimeOffset.UtcNow;
|
||||
var tasks = recordSession.RecordTasks
|
||||
@@ -345,6 +579,7 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
var anyUsableOutput = tasks.Any(static item => item.Status == RecordTaskStatus.Completed);
|
||||
var hasBackgroundPostProcessing = false;
|
||||
string? sessionFinalizationError = null;
|
||||
var newlyCompletedTasks = new List<(RecordTask Task, string OutputPath)>();
|
||||
|
||||
foreach (var task in tasks)
|
||||
{
|
||||
@@ -372,11 +607,21 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
cancellationToken);
|
||||
var effectiveOutputPath = finalizationResult.OutputPath;
|
||||
var taskFinalizationError = finalizationResult.ErrorMessage;
|
||||
var mediaValidation = await ValidateMediaArtifactAsync(
|
||||
effectiveOutputPath,
|
||||
recordSession.OutputFormat,
|
||||
cancellationToken);
|
||||
durationSeconds = mediaValidation.DurationSeconds;
|
||||
if (!IsLowStoragePauseError(taskFinalizationError))
|
||||
{
|
||||
sessionFinalizationError ??= taskFinalizationError;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(taskFinalizationError) && !mediaValidation.IsValid)
|
||||
{
|
||||
sessionFinalizationError ??= mediaValidation.ErrorMessage;
|
||||
}
|
||||
|
||||
var fileSize = CalculateFileSize(effectiveOutputPath);
|
||||
var danmakuPath = task.Result?.DanmakuFilePath ?? GuessDanmakuPath(task.OutputFilePath);
|
||||
var danmakuCount = CountDanmakuMessages(danmakuPath);
|
||||
@@ -388,23 +633,34 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
}
|
||||
else if (!string.IsNullOrWhiteSpace(taskFinalizationError))
|
||||
{
|
||||
task.MarkFailed(taskFinalizationError, endedAt);
|
||||
task.MarkFailed(taskFinalizationError, endedAt, durationSeconds);
|
||||
}
|
||||
else if (HasUsableOutput(effectiveOutputPath, fileSize))
|
||||
else if (mediaValidation.IsValid)
|
||||
{
|
||||
task.MarkCompleted(endedAt, durationSeconds);
|
||||
anyUsableOutput = true;
|
||||
newlyCompletedTasks.Add((task, effectiveOutputPath));
|
||||
}
|
||||
else
|
||||
{
|
||||
task.MarkStopped(endedAt, durationSeconds, "Recording process was no longer running when the session was reconciled.");
|
||||
task.MarkFailed(mediaValidation.ErrorMessage!, endedAt, durationSeconds);
|
||||
}
|
||||
|
||||
await UpsertRecordResultAsync(task, dbContext, effectiveOutputPath, fileSize, durationSeconds, danmakuPath, danmakuCount, endedAt, cancellationToken);
|
||||
await UpsertRecordResultAsync(
|
||||
task,
|
||||
dbContext,
|
||||
effectiveOutputPath,
|
||||
fileSize,
|
||||
durationSeconds,
|
||||
danmakuPath,
|
||||
danmakuCount,
|
||||
endedAt,
|
||||
mediaValidatedForDispatch: mediaValidation.IsValid,
|
||||
cancellationToken: cancellationToken);
|
||||
ClearPostProcessState(task.Id);
|
||||
}
|
||||
|
||||
recordSession.SyncSegmentCount(tasks.Count, endedAt);
|
||||
recordSession.SyncSegmentCount(tasks.Count == 0 ? 0 : tasks.Max(static task => task.SegmentIndex), endedAt);
|
||||
if (!string.IsNullOrWhiteSpace(sessionFinalizationError))
|
||||
{
|
||||
recordSession.MarkFailed(sessionFinalizationError, endedAt);
|
||||
@@ -425,6 +681,26 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
}
|
||||
|
||||
await dbContext.SaveChangesAsync(cancellationToken);
|
||||
|
||||
if (newlyCompletedTasks.Count > 0 && recordSession.LiveRoom is not null)
|
||||
{
|
||||
var completionDispatchService = scope.ServiceProvider.GetRequiredService<CompletionDispatchService>();
|
||||
foreach (var completed in newlyCompletedTasks)
|
||||
{
|
||||
try
|
||||
{
|
||||
await completionDispatchService.TryDispatchTaskAsync(completed.Task.Id, cancellationToken);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
ex,
|
||||
"Recovered segment completion hooks failed for task {RecordTaskId}",
|
||||
completed.Task.Id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -459,8 +735,8 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
return false;
|
||||
}
|
||||
|
||||
var recorderOutputPath = ResolveManualFinalizeSourcePath(recordTask, recordTask.RecordSession);
|
||||
if (!File.Exists(recorderOutputPath))
|
||||
var recorderOutputPaths = ResolveManualFinalizeSourcePaths(recordTask, recordTask.RecordSession);
|
||||
if (recorderOutputPaths.Count == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
@@ -548,9 +824,10 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
settings.Mp4FinalizeTimeoutMinutes,
|
||||
syntheticSessionId,
|
||||
syntheticTaskId,
|
||||
absoluteSourcePath,
|
||||
[absoluteSourcePath],
|
||||
absoluteTargetPath,
|
||||
expectedDurationSeconds: null,
|
||||
segmentsManifestPath: null,
|
||||
CancellationToken.None);
|
||||
|
||||
using var scope = _serviceScopeFactory.CreateScope();
|
||||
@@ -610,32 +887,30 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
var settings = await settingsService.GetAsync(cancellationToken);
|
||||
|
||||
var storageCheck = _storageGuardService.CheckCanStartOrResume(settings);
|
||||
if (storageCheck.ShouldPauseActive)
|
||||
{
|
||||
await logService.WriteAsync(
|
||||
SystemLogLevel.Warning,
|
||||
"Storage",
|
||||
"MP4 finalization remains paused because storage tier is Red.",
|
||||
storageCheck.Message,
|
||||
cancellationToken: cancellationToken);
|
||||
return 0;
|
||||
}
|
||||
|
||||
var candidates = await dbContext.RecordTasks
|
||||
.Include(item => item.RecordSession)
|
||||
.ThenInclude(item => item!.RecordTasks)
|
||||
.Include(item => item.Result)
|
||||
.Where(item => item.OutputFormat == RecordOutputFormat.Mp4 &&
|
||||
(item.Status == RecordTaskStatus.Processing ||
|
||||
item.Status == RecordTaskStatus.Completed))
|
||||
item.Status == RecordTaskStatus.Completed ||
|
||||
item.Status == RecordTaskStatus.Failed &&
|
||||
item.ErrorMessage == FfprobeUnreadableArtifactError))
|
||||
// Deployment-paused work must not be starved by a large backlog of
|
||||
// older legacy failures that may no longer have local source files.
|
||||
.OrderBy(static item => item.Status == RecordTaskStatus.Processing
|
||||
? 0
|
||||
: item.Status == RecordTaskStatus.Completed
|
||||
? 1
|
||||
: 2)
|
||||
.ThenBy(static item => item.UpdatedAt)
|
||||
.Take(100)
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
candidates = candidates
|
||||
.OrderBy(static item => item.UpdatedAt)
|
||||
.Take(100)
|
||||
.ToList();
|
||||
|
||||
var queuedTaskIds = new List<Guid>();
|
||||
var recoveredValidatedTaskIds = new List<Guid>();
|
||||
var repairedInterruptedTasks = 0;
|
||||
var repairedLegacyShutdownFailures = 0;
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
@@ -645,6 +920,58 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
continue;
|
||||
}
|
||||
|
||||
if (candidate.Status == RecordTaskStatus.Failed)
|
||||
{
|
||||
var recoverySources = candidate.RecordSession is null
|
||||
? Array.Empty<string>()
|
||||
: ResolveManualFinalizeSourcePaths(candidate, candidate.RecordSession);
|
||||
if (recoverySources.Count > 0)
|
||||
{
|
||||
candidate.MarkProcessing(
|
||||
"A deployment-interrupted MP4 finalization was recovered and queued after restart.",
|
||||
now);
|
||||
candidate.RecordSession!.MarkStopped(
|
||||
now,
|
||||
"A deployment-interrupted MP4 finalization is continuing in the background.");
|
||||
queuedTaskIds.Add(candidate.Id);
|
||||
repairedLegacyShutdownFailures++;
|
||||
continue;
|
||||
}
|
||||
|
||||
var existingOutputPath = candidate.Result?.FilePath ?? candidate.OutputFilePath;
|
||||
var mediaValidation = await ValidateMediaArtifactAsync(
|
||||
existingOutputPath,
|
||||
candidate.OutputFormat,
|
||||
cancellationToken);
|
||||
if (!mediaValidation.IsValid)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
candidate.MarkCompleted(now, mediaValidation.DurationSeconds);
|
||||
if (candidate.RecordSession is not null &&
|
||||
candidate.RecordSession.RecordTasks.All(static task =>
|
||||
task.Status is RecordTaskStatus.Completed or RecordTaskStatus.Stopped))
|
||||
{
|
||||
candidate.RecordSession.MarkCompleted(now);
|
||||
}
|
||||
|
||||
await UpsertRecordResultAsync(
|
||||
candidate,
|
||||
dbContext,
|
||||
existingOutputPath,
|
||||
CalculateFileSize(existingOutputPath),
|
||||
mediaValidation.DurationSeconds,
|
||||
candidate.Result?.DanmakuFilePath,
|
||||
candidate.Result?.DanmakuMessageCount ?? 0,
|
||||
now,
|
||||
mediaValidatedForDispatch: true,
|
||||
cancellationToken: cancellationToken);
|
||||
recoveredValidatedTaskIds.Add(candidate.Id);
|
||||
repairedLegacyShutdownFailures++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!NeedsInterruptedMp4Finalization(candidate))
|
||||
{
|
||||
continue;
|
||||
@@ -655,13 +982,24 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
repairedInterruptedTasks++;
|
||||
}
|
||||
|
||||
if (repairedInterruptedTasks > 0)
|
||||
if (repairedInterruptedTasks > 0 || repairedLegacyShutdownFailures > 0)
|
||||
{
|
||||
await dbContext.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
if (recoveredValidatedTaskIds.Count > 0)
|
||||
{
|
||||
var completionDispatchService = scope.ServiceProvider.GetRequiredService<CompletionDispatchService>();
|
||||
foreach (var taskId in recoveredValidatedTaskIds)
|
||||
{
|
||||
await completionDispatchService.TryDispatchTaskAsync(taskId, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
var started = 0;
|
||||
foreach (var taskId in queuedTaskIds.Take(20))
|
||||
foreach (var taskId in queuedTaskIds
|
||||
.Where(taskId => !IsTaskUnderPostProcessing(taskId))
|
||||
.Take(20))
|
||||
{
|
||||
if (await StartManualFinalizeTaskAsync(taskId, cancellationToken))
|
||||
{
|
||||
@@ -669,13 +1007,13 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
}
|
||||
}
|
||||
|
||||
if (started > 0)
|
||||
if (started > 0 || repairedLegacyShutdownFailures > 0)
|
||||
{
|
||||
await logService.WriteAsync(
|
||||
SystemLogLevel.Info,
|
||||
"Storage",
|
||||
"Storage is available. Resumed paused MP4 finalization tasks.",
|
||||
$"count={started}; repairedInterrupted={repairedInterruptedTasks}; {storageCheck.Message}",
|
||||
"Resumed paused MP4 finalizations and repaired deployment-interrupted recording results.",
|
||||
$"count={started}; repairedInterrupted={repairedInterruptedTasks}; repairedLegacyShutdown={repairedLegacyShutdownFailures}; {storageCheck.Message}",
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
@@ -699,12 +1037,23 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
return false;
|
||||
}
|
||||
|
||||
var recorderOutputPath = ResolveManualFinalizeSourcePath(recordTask, recordTask.RecordSession);
|
||||
return File.Exists(recorderOutputPath);
|
||||
return ResolveManualFinalizeSourcePaths(recordTask, recordTask.RecordSession).Count > 0;
|
||||
}
|
||||
|
||||
private static string ResolveManualFinalizeSourcePath(RecordTask recordTask, RecordSession recordSession)
|
||||
=> ResolveManualFinalizeSourcePaths(recordTask, recordSession).FirstOrDefault() ?? string.Empty;
|
||||
|
||||
private static IReadOnlyList<string> ResolveManualFinalizeSourcePaths(RecordTask recordTask, RecordSession recordSession)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(recordTask.OutputFilePath))
|
||||
{
|
||||
var manifestPaths = ReadRecorderSegmentsManifest(recordTask.OutputFilePath);
|
||||
if (manifestPaths.Count > 0)
|
||||
{
|
||||
return manifestPaths;
|
||||
}
|
||||
}
|
||||
|
||||
var resultPath = recordTask.Result?.FilePath;
|
||||
if (!string.IsNullOrWhiteSpace(resultPath) &&
|
||||
resultPath.EndsWith(".ts", StringComparison.OrdinalIgnoreCase))
|
||||
@@ -712,15 +1061,16 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
var normalizedResultPath = NormalizeAbsolutePath(resultPath);
|
||||
if (File.Exists(normalizedResultPath))
|
||||
{
|
||||
return normalizedResultPath;
|
||||
return [normalizedResultPath];
|
||||
}
|
||||
}
|
||||
|
||||
return NormalizeAbsolutePath(
|
||||
var defaultPath = NormalizeAbsolutePath(
|
||||
GetRecorderOutputPath(
|
||||
recordTask.OutputFilePath ?? resultPath ?? string.Empty,
|
||||
recordSession.OutputFormat,
|
||||
recordSession.SaveMode));
|
||||
return File.Exists(defaultPath) ? [defaultPath] : [];
|
||||
}
|
||||
|
||||
private async Task PersistStartupProfileAsync(SessionProcessRuntime runtime, CancellationToken cancellationToken)
|
||||
@@ -731,7 +1081,7 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
SystemLogLevel.Info,
|
||||
"FFmpeg",
|
||||
$"ffmpeg input profile={runtime.InputOptionProfile}.",
|
||||
detail: $"stream={runtime.SelectedProtocol}:{runtime.SelectedQuality}; videoCodec={runtime.SelectedVideoCodec ?? "unknown"}; compatibilityRetry={runtime.HasRetriedWithCompatibilityProfile}; refreshRetry={runtime.HasRetriedWithRefreshedStream}",
|
||||
detail: $"stream={runtime.SelectedProtocol}:{runtime.SelectedQuality}; videoCodec={runtime.SelectedVideoCodec ?? "unknown"}; recoveryEncoder={runtime.RecoveryVideoEncoder.Kind}; recoveryDevice={runtime.RecoveryVideoEncoder.DevicePath ?? "default"}; attempt={runtime.RetryAttemptCount}/{MaxInSessionRetryAttempts}; compatibilityRetry={runtime.HasRetriedWithCompatibilityProfile}; refreshRetry={runtime.HasRetriedWithRefreshedStream}; alternateProtocolRetry={runtime.HasRetriedWithAlternateProtocol}; softwareEncoderRetry={runtime.RecoveryContext.HasRetriedWithSoftwareEncoder}",
|
||||
liveRoomId: runtime.LiveRoomId,
|
||||
recordSessionId: runtime.RecordSessionId,
|
||||
recordTaskId: runtime.CurrentTaskId,
|
||||
@@ -756,14 +1106,25 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
return false;
|
||||
}
|
||||
|
||||
private async Task RequestStopAsync(Guid recordSessionId, bool markAsCompletedOnExit, CancellationToken cancellationToken)
|
||||
private async Task RequestStopAsync(
|
||||
Guid recordSessionId,
|
||||
bool markAsCompletedOnExit,
|
||||
CancellationToken cancellationToken,
|
||||
bool shutdownRequested = false)
|
||||
{
|
||||
if (!_processes.TryGetValue(recordSessionId, out var runtime))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
runtime.MarkStopRequested(markAsCompletedOnExit);
|
||||
if (shutdownRequested)
|
||||
{
|
||||
runtime.MarkShutdownRequested();
|
||||
}
|
||||
else
|
||||
{
|
||||
runtime.MarkStopRequested(markAsCompletedOnExit);
|
||||
}
|
||||
var process = runtime.Process;
|
||||
if (process is null)
|
||||
{
|
||||
@@ -919,6 +1280,41 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
}
|
||||
}
|
||||
|
||||
private TimeSpan RecordRuntimeFailure(Guid liveRoomId)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var state = _roomRuntimeFailureStates.AddOrUpdate(
|
||||
liveRoomId,
|
||||
_ => new RoomStartupFailureState { ConsecutiveFailures = 1, FirstFailureAt = now, LastFailureAt = now },
|
||||
(_, existing) =>
|
||||
{
|
||||
existing.ConsecutiveFailures++;
|
||||
existing.LastFailureAt = now;
|
||||
return existing;
|
||||
});
|
||||
|
||||
var backoffSeconds = RuntimeFailureBaseBackoff.TotalSeconds *
|
||||
Math.Pow(2, Math.Min(state.ConsecutiveFailures - 1, 5));
|
||||
var backoff = TimeSpan.FromSeconds(Math.Min(backoffSeconds, RuntimeFailureMaxBackoff.TotalSeconds));
|
||||
_logger.LogWarning(
|
||||
"Short runtime failure backoff for room {LiveRoomId}: {ConsecutiveFailures} consecutive failures, next poll delayed by {BackoffSeconds:F0}s",
|
||||
liveRoomId,
|
||||
state.ConsecutiveFailures,
|
||||
backoff.TotalSeconds);
|
||||
return backoff;
|
||||
}
|
||||
|
||||
private void ResetRuntimeFailureBackoff(Guid liveRoomId)
|
||||
{
|
||||
if (_roomRuntimeFailureStates.TryRemove(liveRoomId, out var state) && state.ConsecutiveFailures > 1)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Short runtime failure backoff reset for room {LiveRoomId} after {ConsecutiveFailures} failures",
|
||||
liveRoomId,
|
||||
state.ConsecutiveFailures);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns true if the failure notification for this room should be throttled
|
||||
/// (i.e., at most one notification per <see cref="StartupFailureNotificationCooldown"/>).
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
using LiveRecorder.Application.Abstractions.Recording;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace LiveRecorder.Infrastructure.Services;
|
||||
|
||||
public sealed class FfmpegShutdownHostedService : IHostedService
|
||||
{
|
||||
private static readonly TimeSpan GracefulTimeout = TimeSpan.FromSeconds(20);
|
||||
private static readonly TimeSpan ForceKillTimeout = TimeSpan.FromSeconds(10);
|
||||
private readonly IFfmpegService _ffmpegService;
|
||||
private readonly ILogger<FfmpegShutdownHostedService> _logger;
|
||||
|
||||
public FfmpegShutdownHostedService(
|
||||
IFfmpegService ffmpegService,
|
||||
ILogger<FfmpegShutdownHostedService> logger)
|
||||
{
|
||||
_ffmpegService = ffmpegService;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public Task StartAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public async Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
_logger.LogInformation("Stopping active recording and post-processing child processes.");
|
||||
var stopped = await _ffmpegService.StopAllAndWaitAsync(
|
||||
GracefulTimeout,
|
||||
ForceKillTimeout,
|
||||
cancellationToken);
|
||||
_logger.LogInformation("Recording shutdown coordination completed for {SessionCount} sessions.", stopped);
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,22 @@
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
using LiveRecorder.Application.Abstractions.Recording;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace LiveRecorder.Infrastructure.Services;
|
||||
|
||||
public sealed class FfmpegVideoMetadataService : IVideoMetadataService
|
||||
{
|
||||
private const string ThumbnailsSubDir = ".thumbnails";
|
||||
private static readonly TimeSpan ProbeTimeout = TimeSpan.FromSeconds(60);
|
||||
private readonly ILogger<FfmpegVideoMetadataService> _logger;
|
||||
|
||||
public FfmpegVideoMetadataService(ILogger<FfmpegVideoMetadataService> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task<VideoMetadata?> ExtractMetadataAsync(string filePath, CancellationToken cancellationToken = default)
|
||||
{
|
||||
@@ -17,12 +27,11 @@ public sealed class FfmpegVideoMetadataService : IVideoMetadataService
|
||||
|
||||
try
|
||||
{
|
||||
var process = new Process
|
||||
using var process = new Process
|
||||
{
|
||||
StartInfo = new ProcessStartInfo
|
||||
{
|
||||
FileName = GetFfprobePath(),
|
||||
Arguments = $"-v quiet -print_format json -show_format -show_streams \"{filePath}\"",
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false,
|
||||
@@ -30,21 +39,55 @@ public sealed class FfmpegVideoMetadataService : IVideoMetadataService
|
||||
}
|
||||
};
|
||||
|
||||
process.Start();
|
||||
var output = await process.StandardOutput.ReadToEndAsync(cancellationToken);
|
||||
await process.WaitForExitAsync(cancellationToken);
|
||||
|
||||
if (process.ExitCode != 0 || string.IsNullOrWhiteSpace(output))
|
||||
foreach (var argument in new[] { "-v", "error", "-print_format", "json", "-show_format", "-show_streams", filePath })
|
||||
{
|
||||
return null;
|
||||
process.StartInfo.ArgumentList.Add(argument);
|
||||
}
|
||||
|
||||
return ParseFfprobeOutput(output);
|
||||
SanitizeFfprobeProcessEnvironment(process.StartInfo);
|
||||
|
||||
process.Start();
|
||||
using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
timeoutCts.CancelAfter(ProbeTimeout);
|
||||
var outputTask = process.StandardOutput.ReadToEndAsync(timeoutCts.Token);
|
||||
var errorTask = process.StandardError.ReadToEndAsync(timeoutCts.Token);
|
||||
await process.WaitForExitAsync(timeoutCts.Token);
|
||||
var output = await outputTask;
|
||||
var error = await errorTask;
|
||||
|
||||
if (process.ExitCode == 0 && !string.IsNullOrWhiteSpace(output))
|
||||
{
|
||||
var metadata = ParseFfprobeOutput(output);
|
||||
if (metadata is not null)
|
||||
{
|
||||
return metadata;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"ffprobe failed for {FilePath}: exitCode={ExitCode}; stderr={Error}",
|
||||
filePath,
|
||||
process.ExitCode,
|
||||
string.IsNullOrWhiteSpace(error) ? "(empty)" : error.Trim());
|
||||
}
|
||||
}
|
||||
catch
|
||||
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
_logger.LogWarning("ffprobe timed out after {TimeoutSeconds}s for {FilePath}", ProbeTimeout.TotalSeconds, filePath);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "ffprobe could not be started or read media metadata for {FilePath}", filePath);
|
||||
}
|
||||
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
_logger.LogWarning("Falling back to ffmpeg header probing for {FilePath}", filePath);
|
||||
return await ExtractMetadataWithFfmpegAsync(filePath, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<string?> GenerateThumbnailAsync(string filePath, string outputDir, CancellationToken cancellationToken = default)
|
||||
@@ -93,7 +136,10 @@ public sealed class FfmpegVideoMetadataService : IVideoMetadataService
|
||||
};
|
||||
|
||||
process.Start();
|
||||
var outputTask = process.StandardOutput.ReadToEndAsync(cancellationToken);
|
||||
var errorTask = process.StandardError.ReadToEndAsync(cancellationToken);
|
||||
await process.WaitForExitAsync(cancellationToken);
|
||||
await Task.WhenAll(outputTask, errorTask);
|
||||
|
||||
if (process.ExitCode == 0 && File.Exists(thumbPath) && new FileInfo(thumbPath).Length > 0)
|
||||
{
|
||||
@@ -165,6 +211,155 @@ public sealed class FfmpegVideoMetadataService : IVideoMetadataService
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<VideoMetadata?> ExtractMetadataWithFfmpegAsync(
|
||||
string filePath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var process = new Process
|
||||
{
|
||||
StartInfo = new ProcessStartInfo
|
||||
{
|
||||
FileName = GetFfmpegPath(),
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false,
|
||||
CreateNoWindow = true
|
||||
}
|
||||
};
|
||||
|
||||
foreach (var argument in new[] { "-hide_banner", "-i", filePath, "-t", "0", "-f", "null", "-" })
|
||||
{
|
||||
process.StartInfo.ArgumentList.Add(argument);
|
||||
}
|
||||
|
||||
// The recorder already proves that fnOS can launch this ffmpeg with
|
||||
// the inherited application environment. Do not apply the ffprobe-
|
||||
// specific library cleanup to this compatibility fallback.
|
||||
process.Start();
|
||||
using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
timeoutCts.CancelAfter(ProbeTimeout);
|
||||
var outputTask = process.StandardOutput.ReadToEndAsync(timeoutCts.Token);
|
||||
var errorTask = process.StandardError.ReadToEndAsync(timeoutCts.Token);
|
||||
await process.WaitForExitAsync(timeoutCts.Token);
|
||||
await outputTask;
|
||||
var error = await errorTask;
|
||||
|
||||
var metadata = ParseFfmpegHeaderOutput(error);
|
||||
if (process.ExitCode == 0 && metadata is not null)
|
||||
{
|
||||
_logger.LogInformation("ffmpeg header probing succeeded for {FilePath}", filePath);
|
||||
return metadata;
|
||||
}
|
||||
|
||||
_logger.LogWarning(
|
||||
"ffmpeg header probing failed for {FilePath}: exitCode={ExitCode}; stderr={Error}",
|
||||
filePath,
|
||||
process.ExitCode,
|
||||
string.IsNullOrWhiteSpace(error) ? "(empty)" : error.Trim());
|
||||
}
|
||||
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"ffmpeg header probing timed out after {TimeoutSeconds}s for {FilePath}",
|
||||
ProbeTimeout.TotalSeconds,
|
||||
filePath);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "ffmpeg header probing could not read media metadata for {FilePath}", filePath);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
internal static VideoMetadata? ParseFfmpegHeaderOutput(string output)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(output))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var durationMatch = Regex.Match(
|
||||
output,
|
||||
@"Duration:\s*(?<hours>\d+):(?<minutes>\d{2}):(?<seconds>\d{2}(?:\.\d+)?)",
|
||||
RegexOptions.CultureInvariant);
|
||||
if (!durationMatch.Success ||
|
||||
!double.TryParse(durationMatch.Groups["hours"].Value, NumberStyles.None, CultureInfo.InvariantCulture, out var hours) ||
|
||||
!double.TryParse(durationMatch.Groups["minutes"].Value, NumberStyles.None, CultureInfo.InvariantCulture, out var minutes) ||
|
||||
!double.TryParse(durationMatch.Groups["seconds"].Value, NumberStyles.Float, CultureInfo.InvariantCulture, out var seconds))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var durationSeconds = hours * 3600 + minutes * 60 + seconds;
|
||||
if (durationSeconds <= 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
string? videoLine = null;
|
||||
string? audioLine = null;
|
||||
foreach (var line in output.Split('\n'))
|
||||
{
|
||||
if (line.Contains("Stream mapping:", StringComparison.Ordinal))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (videoLine is null && line.Contains("Video:", StringComparison.Ordinal))
|
||||
{
|
||||
videoLine = line;
|
||||
}
|
||||
else if (audioLine is null && line.Contains("Audio:", StringComparison.Ordinal))
|
||||
{
|
||||
audioLine = line;
|
||||
}
|
||||
}
|
||||
|
||||
var videoCodec = MatchValue(videoLine, @"Video:\s*(?<value>[^\s,(]+)");
|
||||
var audioCodec = MatchValue(audioLine, @"Audio:\s*(?<value>[^\s,(]+)");
|
||||
var dimensions = videoLine is null
|
||||
? Match.Empty
|
||||
: Regex.Match(videoLine, @"(?<!\d)(?<width>\d{2,5})x(?<height>\d{2,5})(?!\d)", RegexOptions.CultureInvariant);
|
||||
var frameRate = MatchDouble(videoLine, @",\s*(?<value>\d+(?:\.\d+)?)\s+fps(?:,|\s)");
|
||||
var bitRateKbps = MatchDouble(
|
||||
output,
|
||||
@"Duration:[^\r\n]*bitrate:\s*(?<value>\d+(?:\.\d+)?)\s*kb/s");
|
||||
|
||||
int? width = dimensions.Success && int.TryParse(dimensions.Groups["width"].Value, out var parsedWidth)
|
||||
? parsedWidth
|
||||
: null;
|
||||
int? height = dimensions.Success && int.TryParse(dimensions.Groups["height"].Value, out var parsedHeight)
|
||||
? parsedHeight
|
||||
: null;
|
||||
long? bitRate = bitRateKbps.HasValue
|
||||
? (long)Math.Round(bitRateKbps.Value * 1000, MidpointRounding.AwayFromZero)
|
||||
: null;
|
||||
|
||||
return new VideoMetadata(durationSeconds, width, height, videoCodec, audioCodec, frameRate, bitRate);
|
||||
}
|
||||
|
||||
private static string? MatchValue(string? input, string pattern)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(input))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var match = Regex.Match(input, pattern, RegexOptions.CultureInvariant);
|
||||
return match.Success ? match.Groups["value"].Value : null;
|
||||
}
|
||||
|
||||
private static double? MatchDouble(string? input, string pattern)
|
||||
{
|
||||
var value = MatchValue(input, pattern);
|
||||
return double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed)
|
||||
? parsed
|
||||
: null;
|
||||
}
|
||||
|
||||
private static double? ParseFrameRate(string fraction)
|
||||
{
|
||||
var parts = fraction.Split('/');
|
||||
@@ -181,4 +376,12 @@ public sealed class FfmpegVideoMetadataService : IVideoMetadataService
|
||||
|
||||
private static string GetFfmpegPath() => "ffmpeg";
|
||||
private static string GetFfprobePath() => "ffprobe";
|
||||
|
||||
internal static void SanitizeFfprobeProcessEnvironment(ProcessStartInfo startInfo)
|
||||
{
|
||||
// The fnOS package ships private Debian libraries for its bundled curl.
|
||||
// Inheriting that LD_LIBRARY_PATH into the host's ffmpeg/ffprobe can make
|
||||
// otherwise valid system binaries fail with incompatible shared libraries.
|
||||
startInfo.Environment.Remove("LD_LIBRARY_PATH");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,6 +74,13 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
|
||||
var settingsService = scope.ServiceProvider.GetRequiredService<ISystemSettingsService>();
|
||||
var settings = await settingsService.GetAsync(stoppingToken);
|
||||
var ffmpegService = scope.ServiceProvider.GetRequiredService<IFfmpegService>();
|
||||
var recoveredOrphanedSessions = await ffmpegService.RecoverOrphanedTerminalSessionTasksAsync(stoppingToken);
|
||||
if (recoveredOrphanedSessions > 0)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Recovered {SessionCount} terminal recording sessions that still contained active segment tasks.",
|
||||
recoveredOrphanedSessions);
|
||||
}
|
||||
// Always try to resume paused MP4 finalizations — even (especially) under the Red
|
||||
// tier. Finalization is what flips a task to Completed, which fires the
|
||||
// segment_completed script (upload + delete source) that frees disk space. Skipping
|
||||
@@ -431,28 +438,37 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
|
||||
return;
|
||||
}
|
||||
|
||||
var hasRunningSession = await dbContext.RecordSessions.AnyAsync(
|
||||
var activeSessionId = await dbContext.RecordSessions
|
||||
.Where(
|
||||
item => item.LiveRoomId == liveRoom.Id &&
|
||||
(item.Status == RecordSessionStatus.Starting ||
|
||||
item.Status == RecordSessionStatus.Running ||
|
||||
item.Status == RecordSessionStatus.Stopping),
|
||||
cancellationToken);
|
||||
item.Status == RecordSessionStatus.Stopping))
|
||||
.OrderByDescending(static item => item.CreatedAt)
|
||||
.Select(static item => (Guid?)item.Id)
|
||||
.FirstOrDefaultAsync(cancellationToken);
|
||||
|
||||
if (hasRunningSession)
|
||||
if (activeSessionId.HasValue)
|
||||
{
|
||||
await UpdateAutoStartDecisionAsync(
|
||||
var decisionChanged = await UpdateAutoStartDecisionIfChangedAsync(
|
||||
dbContext,
|
||||
liveRoom,
|
||||
AutoStartDecisionCodes.SkippedActiveSession,
|
||||
"Auto-start skipped because an active recording session already exists.",
|
||||
detail: null,
|
||||
$"activeSessionId={activeSessionId.Value}",
|
||||
cancellationToken);
|
||||
await logService.WriteAsync(
|
||||
SystemLogLevel.Info,
|
||||
"Scheduler",
|
||||
"Auto-start skipped because an active recording session already exists.",
|
||||
liveRoomId: liveRoom.Id,
|
||||
cancellationToken: cancellationToken);
|
||||
if (decisionChanged)
|
||||
{
|
||||
await logService.WriteAsync(
|
||||
SystemLogLevel.Info,
|
||||
"Scheduler",
|
||||
"Auto-start skipped because an active recording session already exists.",
|
||||
$"activeSessionId={activeSessionId.Value}",
|
||||
liveRoomId: liveRoom.Id,
|
||||
recordSessionId: activeSessionId.Value,
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -860,6 +876,40 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
|
||||
await SaveChangesWithRetryAsync(dbContext, cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task<bool> UpdateAutoStartDecisionIfChangedAsync(
|
||||
LiveRecorderDbContext dbContext,
|
||||
Domain.Entities.LiveRoom liveRoom,
|
||||
string code,
|
||||
string summary,
|
||||
string? detail,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedCode = Truncate(code, 64);
|
||||
var normalizedSummary = Truncate(summary, 256);
|
||||
var normalizedDetail = Truncate(detail, 2048);
|
||||
if (!HasAutoStartDecisionChanged(liveRoom, normalizedCode, normalizedSummary, normalizedDetail))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
liveRoom.SetLastAutoStartDecision(
|
||||
normalizedCode,
|
||||
normalizedSummary,
|
||||
normalizedDetail,
|
||||
DateTimeOffset.UtcNow);
|
||||
await SaveChangesWithRetryAsync(dbContext, cancellationToken);
|
||||
return true;
|
||||
}
|
||||
|
||||
internal static bool HasAutoStartDecisionChanged(
|
||||
Domain.Entities.LiveRoom liveRoom,
|
||||
string? code,
|
||||
string? summary,
|
||||
string? detail) =>
|
||||
!string.Equals(liveRoom.LastAutoStartDecisionCode, code, StringComparison.Ordinal) ||
|
||||
!string.Equals(liveRoom.LastAutoStartDecisionSummary, summary, StringComparison.Ordinal) ||
|
||||
!string.Equals(liveRoom.LastAutoStartDecisionDetail, detail, StringComparison.Ordinal);
|
||||
|
||||
private static async Task SaveChangesWithRetryAsync(LiveRecorderDbContext dbContext, CancellationToken cancellationToken)
|
||||
{
|
||||
for (var attempt = 1; attempt <= 5; attempt++)
|
||||
|
||||
@@ -38,6 +38,23 @@ public interface IOpenListClient
|
||||
OpenListConnectionRequest connection,
|
||||
string taskId,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
Task<bool> TryCancelCopyTaskAsync(
|
||||
OpenListConnectionRequest connection,
|
||||
string taskId,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
Task RenameAsync(
|
||||
OpenListConnectionRequest connection,
|
||||
string path,
|
||||
string newName,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
Task MoveAsync(
|
||||
OpenListConnectionRequest connection,
|
||||
string sourcePath,
|
||||
string targetDirectory,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
public sealed record OpenListObjectInfo(
|
||||
@@ -400,6 +417,67 @@ public sealed class OpenListClient : IOpenListClient
|
||||
return new OpenListTaskInfo(id, state, progress, status, error);
|
||||
}
|
||||
|
||||
public async Task<bool> TryCancelCopyTaskAsync(
|
||||
OpenListConnectionRequest connection,
|
||||
string taskId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(taskId))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
var encodedTaskId = Uri.EscapeDataString(taskId.Trim());
|
||||
var envelope = await SendAuthorizedAsync(
|
||||
connection,
|
||||
() => new HttpRequestMessage(HttpMethod.Post, $"/api/task/copy/cancel?tid={encodedTaskId}"),
|
||||
cancellationToken);
|
||||
return envelope.Code == 200 || envelope.Code == 404 || ContainsAny(envelope.Message, "not found", "finished");
|
||||
}
|
||||
|
||||
public async Task RenameAsync(
|
||||
OpenListConnectionRequest connection,
|
||||
string path,
|
||||
string newName,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
path = NormalizePath(path);
|
||||
if (string.IsNullOrWhiteSpace(newName) || newName.Contains('/') || newName.Contains('\\'))
|
||||
{
|
||||
throw new InvalidOperationException("OpenList 新文件名无效。");
|
||||
}
|
||||
|
||||
var envelope = await SendAuthorizedAsync(
|
||||
connection,
|
||||
() => CreateJsonRequest(HttpMethod.Post, "/api/fs/rename", new { path, name = newName.Trim() }),
|
||||
cancellationToken);
|
||||
EnsureSuccess(envelope, $"OpenList rename '{path}'");
|
||||
}
|
||||
|
||||
public async Task MoveAsync(
|
||||
OpenListConnectionRequest connection,
|
||||
string sourcePath,
|
||||
string targetDirectory,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
sourcePath = NormalizePath(sourcePath);
|
||||
targetDirectory = NormalizePath(targetDirectory);
|
||||
var envelope = await SendAuthorizedAsync(
|
||||
connection,
|
||||
() => CreateJsonRequest(
|
||||
HttpMethod.Post,
|
||||
"/api/fs/move",
|
||||
new
|
||||
{
|
||||
src_dir = GetDirectoryName(sourcePath),
|
||||
dst_dir = targetDirectory,
|
||||
names = new[] { GetFileName(sourcePath) },
|
||||
overwrite = false
|
||||
}),
|
||||
cancellationToken);
|
||||
EnsureSuccess(envelope, $"OpenList move '{sourcePath}' to '{targetDirectory}'");
|
||||
}
|
||||
|
||||
public static string NormalizeBaseUrl(string baseUrl)
|
||||
{
|
||||
if (!Uri.TryCreate(baseUrl?.Trim(), UriKind.Absolute, out var uri) ||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System.Security.Cryptography;
|
||||
using LiveRecorder.Application.Abstractions.Logging;
|
||||
using LiveRecorder.Application.Abstractions.Recording;
|
||||
using LiveRecorder.Application.Abstractions.Settings;
|
||||
using LiveRecorder.Application.Models.RecordTasks;
|
||||
using LiveRecorder.Application.Models.Settings;
|
||||
@@ -16,8 +17,11 @@ namespace LiveRecorder.Infrastructure.Services;
|
||||
public sealed class OpenListUploadQueueService
|
||||
{
|
||||
private const int MaxAttempts = 6;
|
||||
private const int MaxAutomaticRecoveryBatchSize = 100;
|
||||
private static readonly TimeSpan VerificationTimeout = TimeSpan.FromMinutes(2);
|
||||
private static readonly TimeSpan ExternalTaskTimeout = TimeSpan.FromHours(24);
|
||||
private static readonly TimeSpan ExternalTaskStallTimeout = TimeSpan.FromMinutes(60);
|
||||
private static readonly TimeSpan CleanupRetryDelay = TimeSpan.FromHours(1);
|
||||
private static readonly TimeSpan[] RetryDelays =
|
||||
[
|
||||
TimeSpan.FromMinutes(1),
|
||||
@@ -31,17 +35,20 @@ public sealed class OpenListUploadQueueService
|
||||
private readonly ISystemSettingsService _settingsService;
|
||||
private readonly IOpenListClient _openListClient;
|
||||
private readonly ISystemLogService _systemLogService;
|
||||
private readonly IVideoMetadataService _videoMetadataService;
|
||||
|
||||
public OpenListUploadQueueService(
|
||||
LiveRecorderDbContext dbContext,
|
||||
ISystemSettingsService settingsService,
|
||||
IOpenListClient openListClient,
|
||||
ISystemLogService systemLogService)
|
||||
ISystemLogService systemLogService,
|
||||
IVideoMetadataService videoMetadataService)
|
||||
{
|
||||
_dbContext = dbContext;
|
||||
_settingsService = settingsService;
|
||||
_openListClient = openListClient;
|
||||
_systemLogService = systemLogService;
|
||||
_videoMetadataService = videoMetadataService;
|
||||
}
|
||||
|
||||
public async Task<RecordArtifactUploadItemResultDto?> TryEnqueueAutomaticAsync(
|
||||
@@ -59,6 +66,69 @@ public sealed class OpenListUploadQueueService
|
||||
return await EnqueueInternalAsync(recordTaskId, settings, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<int> RecoverPendingAutomaticUploadsAsync(
|
||||
int take = MaxAutomaticRecoveryBatchSize,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var settings = await _settingsService.GetAsync(cancellationToken);
|
||||
if (!settings.EnableFileUpload ||
|
||||
!settings.EnableAutoUpload ||
|
||||
settings.UploadTarget != UploadTargetType.OpenList)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var completedBefore = DateTimeOffset.UtcNow.AddSeconds(-30);
|
||||
var taskIds = await _dbContext.RecordTasks
|
||||
.AsNoTracking()
|
||||
.Where(item =>
|
||||
(item.Status == RecordTaskStatus.Completed || item.Status == RecordTaskStatus.Stopped) &&
|
||||
item.Result != null &&
|
||||
item.Result.UploadStatus == RecordArtifactUploadStatus.NotUploaded &&
|
||||
item.UploadJob == null &&
|
||||
item.UpdatedAt <= completedBefore &&
|
||||
!string.IsNullOrWhiteSpace(item.Result.FilePath))
|
||||
.OrderBy(static item => item.UpdatedAt)
|
||||
.Select(static item => item.Id)
|
||||
.Take(Math.Clamp(take, 1, MaxAutomaticRecoveryBatchSize))
|
||||
.ToArrayAsync(cancellationToken);
|
||||
|
||||
var recovered = 0;
|
||||
foreach (var taskId in taskIds)
|
||||
{
|
||||
try
|
||||
{
|
||||
var result = await EnqueueInternalAsync(taskId, settings, cancellationToken);
|
||||
if (result.Success && result.UploadStatus == RecordArtifactUploadStatus.Queued)
|
||||
{
|
||||
recovered++;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _systemLogService.WriteAsync(
|
||||
SystemLogLevel.Warning,
|
||||
"Upload",
|
||||
"跳过一个无法恢复的 OpenList 自动上传任务。",
|
||||
$"recordTaskId={taskId}; error={ex.Message}",
|
||||
recordTaskId: taskId,
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
if (recovered > 0)
|
||||
{
|
||||
await _systemLogService.WriteAsync(
|
||||
SystemLogLevel.Warning,
|
||||
"Upload",
|
||||
"Recovered completed recording segments that had not entered the automatic OpenList upload queue.",
|
||||
$"count={recovered}",
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
return recovered;
|
||||
}
|
||||
|
||||
public async Task<RecordArtifactUploadItemResultDto> EnqueueAsync(
|
||||
Guid recordTaskId,
|
||||
CancellationToken cancellationToken = default)
|
||||
@@ -145,6 +215,16 @@ public sealed class OpenListUploadQueueService
|
||||
}
|
||||
|
||||
var result = job.RecordTask.Result;
|
||||
var validationError = await GetUploadValidationErrorAsync(
|
||||
job.RecordTask,
|
||||
NormalizeAbsolutePath(result.FilePath),
|
||||
cancellationToken);
|
||||
if (!string.IsNullOrWhiteSpace(validationError))
|
||||
{
|
||||
await MarkFailedAsync(job, result, validationError, cancellationToken);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (job.Status != RecordArtifactUploadStatus.Uploading)
|
||||
{
|
||||
job.BeginAttempt(now);
|
||||
@@ -206,6 +286,12 @@ public sealed class OpenListUploadQueueService
|
||||
return Failure(recordTaskId, "本地视频文件不存在,不能加入上传队列。", "openlist");
|
||||
}
|
||||
|
||||
var validationError = await GetUploadValidationErrorAsync(recordTask, localVideoPath, cancellationToken);
|
||||
if (!string.IsNullOrWhiteSpace(validationError))
|
||||
{
|
||||
return Failure(recordTaskId, validationError, "openlist");
|
||||
}
|
||||
|
||||
var outputRoot = Path.GetFullPath(settings.OutputRoot, AppContext.BaseDirectory);
|
||||
var videoRelativePath = GetSafeRelativePath(outputRoot, localVideoPath);
|
||||
var sourceVideoPath = OpenListClient.CombinePath(settings.OpenListUpload.SourcePath, videoRelativePath);
|
||||
@@ -285,6 +371,46 @@ public sealed class OpenListUploadQueueService
|
||||
return QueuedResult(recordTaskId, result, job, "已加入 OpenList 上传队列。");
|
||||
}
|
||||
|
||||
private async Task<string?> GetUploadValidationErrorAsync(
|
||||
RecordTask recordTask,
|
||||
string localVideoPath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (recordTask.Status is not (RecordTaskStatus.Completed or RecordTaskStatus.Stopped))
|
||||
{
|
||||
return "只有已完成或已停止且媒体有效的录制任务可以上传。";
|
||||
}
|
||||
|
||||
if (recordTask.OutputFormat == RecordOutputFormat.Mp4 &&
|
||||
!localVideoPath.EndsWith(".mp4", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return "MP4 收尾未完成,仅保留了中间文件,已阻止自动上传。";
|
||||
}
|
||||
|
||||
if (!File.Exists(localVideoPath) || new FileInfo(localVideoPath).Length <= 0)
|
||||
{
|
||||
return "本地视频文件不存在或为空,不能上传。";
|
||||
}
|
||||
|
||||
var metadata = await _videoMetadataService.ExtractMetadataAsync(localVideoPath, cancellationToken);
|
||||
if (metadata is null)
|
||||
{
|
||||
return "ffprobe 无法读取媒体信息,已阻止上传并保留本地文件。";
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(metadata.VideoCodec))
|
||||
{
|
||||
return "文件不包含视频流,已阻止上传并保留本地文件。";
|
||||
}
|
||||
|
||||
if (!metadata.DurationSeconds.HasValue || metadata.DurationSeconds.Value < 5)
|
||||
{
|
||||
return $"实际媒体时长不足 5 秒({metadata.DurationSeconds.GetValueOrDefault():0.###} 秒),已阻止上传并保留本地文件。";
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private async Task ProcessJobStepAsync(
|
||||
RecordUploadJob job,
|
||||
RecordResult result,
|
||||
@@ -299,21 +425,34 @@ public sealed class OpenListUploadQueueService
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var targetPath = job.GetCurrentTargetPath();
|
||||
var transferTargetPath = job.GetCurrentTransferTargetPath();
|
||||
var expectedSize = job.GetCurrentSizeBytes();
|
||||
var localPath = GetCurrentLocalPath(job, result);
|
||||
|
||||
if (job.VerificationStartedAt.HasValue)
|
||||
{
|
||||
var verification = await VerifyTargetAsync(connection, targetPath, localPath, expectedSize, cancellationToken);
|
||||
var verification = await VerifyTargetAsync(connection, transferTargetPath, localPath, expectedSize, allowSizeOnlyMatch: true, cancellationToken);
|
||||
if (verification == TargetVerification.Match)
|
||||
{
|
||||
if (!string.Equals(transferTargetPath, targetPath, StringComparison.Ordinal))
|
||||
{
|
||||
await PromoteStagedTransferAsync(job, connection, transferTargetPath, targetPath, now, cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
await CompleteCurrentArtifactAsync(job, result, cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
if (verification == TargetVerification.Conflict)
|
||||
{
|
||||
throw new OpenListUploadConflictException($"目标文件 '{targetPath}' 已存在但内容不一致。");
|
||||
throw new OpenListUploadConflictException($"目标文件 '{transferTargetPath}' 已存在但内容不一致。");
|
||||
}
|
||||
|
||||
if (!string.Equals(transferTargetPath, targetPath, StringComparison.Ordinal) &&
|
||||
await TryRecoverInterruptedPromotionAsync(job, connection, transferTargetPath, targetPath, localPath, expectedSize, now, cancellationToken))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (now - job.VerificationStartedAt.Value < VerificationTimeout)
|
||||
@@ -321,25 +460,49 @@ public sealed class OpenListUploadQueueService
|
||||
return;
|
||||
}
|
||||
|
||||
await ScheduleRetryOrFailAsync(job, result, $"OpenList 任务结束后两分钟内仍未发现目标文件 '{targetPath}'。", true, cancellationToken);
|
||||
await ScheduleRetryOrFailAsync(job, result, $"OpenList 任务结束后两分钟内仍未发现目标文件 '{transferTargetPath}'。", true, cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(job.ExternalTaskId))
|
||||
{
|
||||
var lastProgressAt = job.LastProgressAt ?? job.ExternalTaskStartedAt;
|
||||
if (lastProgressAt.HasValue && now - lastProgressAt.Value > ExternalTaskStallTimeout)
|
||||
{
|
||||
var cancelled = await _openListClient.TryCancelCopyTaskAsync(connection, job.ExternalTaskId, cancellationToken);
|
||||
if (cancelled)
|
||||
{
|
||||
await ScheduleRetryOrFailAsync(job, result, $"OpenList 复制任务连续 60 分钟没有进度,已取消并准备重试:{job.ExternalTaskId}", true, cancellationToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
await MarkFailedAsync(job, result, $"OpenList 复制任务连续 60 分钟没有进度且无法取消,需要人工检查:{job.ExternalTaskId}", cancellationToken);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (job.ExternalTaskStartedAt.HasValue && now - job.ExternalTaskStartedAt.Value > ExternalTaskTimeout)
|
||||
{
|
||||
await ScheduleRetryOrFailAsync(job, result, $"OpenList 复制任务运行超过 24 小时:{job.ExternalTaskId}", true, cancellationToken);
|
||||
_ = await _openListClient.TryCancelCopyTaskAsync(connection, job.ExternalTaskId, cancellationToken);
|
||||
await ScheduleRetryOrFailAsync(job, result, $"OpenList 复制任务运行超过 24 小时,已停止跟踪并准备重试:{job.ExternalTaskId}", true, cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
var task = await _openListClient.TryGetCopyTaskAsync(connection, job.ExternalTaskId, cancellationToken);
|
||||
if (task is null)
|
||||
{
|
||||
var missingTaskVerification = await VerifyTargetAsync(connection, targetPath, localPath, expectedSize, cancellationToken);
|
||||
var missingTaskVerification = await VerifyTargetAsync(connection, transferTargetPath, localPath, expectedSize, allowSizeOnlyMatch: true, cancellationToken);
|
||||
if (missingTaskVerification == TargetVerification.Match)
|
||||
{
|
||||
await CompleteCurrentArtifactAsync(job, result, cancellationToken);
|
||||
if (!string.Equals(transferTargetPath, targetPath, StringComparison.Ordinal))
|
||||
{
|
||||
await PromoteStagedTransferAsync(job, connection, transferTargetPath, targetPath, now, cancellationToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
await CompleteCurrentArtifactAsync(job, result, cancellationToken);
|
||||
}
|
||||
}
|
||||
else if (missingTaskVerification == TargetVerification.Conflict)
|
||||
{
|
||||
@@ -380,7 +543,7 @@ public sealed class OpenListUploadQueueService
|
||||
// first target probe so a new recording path can be uploaded normally.
|
||||
await _openListClient.EnsureDirectoryAsync(connection, GetDirectoryName(targetPath), cancellationToken);
|
||||
|
||||
var existingTarget = await VerifyTargetAsync(connection, targetPath, localPath, expectedSize, cancellationToken);
|
||||
var existingTarget = await VerifyTargetAsync(connection, targetPath, localPath, expectedSize, allowSizeOnlyMatch: false, cancellationToken);
|
||||
if (existingTarget == TargetVerification.Match)
|
||||
{
|
||||
await CompleteCurrentArtifactAsync(job, result, cancellationToken);
|
||||
@@ -389,7 +552,20 @@ public sealed class OpenListUploadQueueService
|
||||
|
||||
if (existingTarget == TargetVerification.Conflict)
|
||||
{
|
||||
throw new OpenListUploadConflictException($"目标文件 '{targetPath}' 已存在但内容不一致。");
|
||||
if (HasTaskConflictSuffix(targetPath, job.RecordTaskId))
|
||||
{
|
||||
throw new OpenListUploadConflictException($"目标文件 '{targetPath}' 已存在但内容不一致。");
|
||||
}
|
||||
|
||||
var conflictTargetPath = AppendTaskConflictSuffix(targetPath, job.RecordTaskId);
|
||||
var stagingDirectory = OpenListClient.CombinePath(
|
||||
GetDirectoryName(targetPath),
|
||||
$".liverecorder-staging-{job.Id.ToString("N")[..8]}");
|
||||
var stagedTargetPath = OpenListClient.CombinePath(stagingDirectory, GetFileName(job.GetCurrentSourcePath()));
|
||||
await _openListClient.EnsureDirectoryAsync(connection, stagingDirectory, cancellationToken);
|
||||
job.ResolveCurrentTargetConflict(conflictTargetPath, stagedTargetPath, now);
|
||||
await _dbContext.SaveChangesAsync(cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
result.MarkUploadStarted("openlist", now);
|
||||
@@ -407,7 +583,11 @@ public sealed class OpenListUploadQueueService
|
||||
throw new InvalidOperationException($"OpenList 源文件大小不一致:期望 {expectedSize},实际 {sourceObject.Size},路径 {sourcePath}");
|
||||
}
|
||||
|
||||
var copyResult = await _openListClient.CopyFileAsync(connection, sourcePath, targetPath, cancellationToken);
|
||||
if (!string.Equals(GetDirectoryName(transferTargetPath), GetDirectoryName(targetPath), StringComparison.Ordinal))
|
||||
{
|
||||
await _openListClient.EnsureDirectoryAsync(connection, GetDirectoryName(transferTargetPath), cancellationToken);
|
||||
}
|
||||
var copyResult = await _openListClient.CopyFileAsync(connection, sourcePath, transferTargetPath, cancellationToken);
|
||||
if (copyResult.TaskIds.Count == 0)
|
||||
{
|
||||
job.StartVerification(DateTimeOffset.UtcNow);
|
||||
@@ -420,6 +600,74 @@ public sealed class OpenListUploadQueueService
|
||||
await _dbContext.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
private async Task PromoteStagedTransferAsync(
|
||||
RecordUploadJob job,
|
||||
OpenListConnectionRequest connection,
|
||||
string stagedPath,
|
||||
string targetPath,
|
||||
DateTimeOffset now,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var targetFileName = GetFileName(targetPath);
|
||||
var renamedStagedPath = stagedPath;
|
||||
if (!string.Equals(GetFileName(stagedPath), targetFileName, StringComparison.Ordinal))
|
||||
{
|
||||
await _openListClient.RenameAsync(connection, stagedPath, targetFileName, cancellationToken);
|
||||
renamedStagedPath = OpenListClient.CombinePath(GetDirectoryName(stagedPath), targetFileName);
|
||||
job.UpdateTransferTargetPath(renamedStagedPath, DateTimeOffset.UtcNow);
|
||||
await _dbContext.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
await _openListClient.MoveAsync(connection, renamedStagedPath, GetDirectoryName(targetPath), cancellationToken);
|
||||
job.CompleteTransferPromotion(now);
|
||||
job.StartVerification(now);
|
||||
await _dbContext.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<bool> TryRecoverInterruptedPromotionAsync(
|
||||
RecordUploadJob job,
|
||||
OpenListConnectionRequest connection,
|
||||
string stagedPath,
|
||||
string targetPath,
|
||||
string localPath,
|
||||
long expectedSize,
|
||||
DateTimeOffset now,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var finalVerification = await VerifyTargetAsync(
|
||||
connection,
|
||||
targetPath,
|
||||
localPath,
|
||||
expectedSize,
|
||||
allowSizeOnlyMatch: true,
|
||||
cancellationToken);
|
||||
if (finalVerification == TargetVerification.Match)
|
||||
{
|
||||
job.CompleteTransferPromotion(now);
|
||||
job.StartVerification(now);
|
||||
await _dbContext.SaveChangesAsync(cancellationToken);
|
||||
return true;
|
||||
}
|
||||
|
||||
var renamedStagedPath = OpenListClient.CombinePath(GetDirectoryName(stagedPath), GetFileName(targetPath));
|
||||
var renamedVerification = await VerifyTargetAsync(
|
||||
connection,
|
||||
renamedStagedPath,
|
||||
localPath,
|
||||
expectedSize,
|
||||
allowSizeOnlyMatch: true,
|
||||
cancellationToken);
|
||||
if (renamedVerification != TargetVerification.Match)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
job.UpdateTransferTargetPath(renamedStagedPath, now);
|
||||
await _dbContext.SaveChangesAsync(cancellationToken);
|
||||
await PromoteStagedTransferAsync(job, connection, renamedStagedPath, targetPath, now, cancellationToken);
|
||||
return true;
|
||||
}
|
||||
|
||||
private async Task CompleteCurrentArtifactAsync(
|
||||
RecordUploadJob job,
|
||||
RecordResult result,
|
||||
@@ -461,6 +709,20 @@ public sealed class OpenListUploadQueueService
|
||||
{
|
||||
cleanupWarning = ex.Message;
|
||||
}
|
||||
|
||||
if (!deletedLocalFiles && cleanupWarning is null)
|
||||
{
|
||||
cleanupWarning = "本地文件删除后仍然存在。";
|
||||
}
|
||||
}
|
||||
|
||||
if (cleanupWarning is not null)
|
||||
{
|
||||
var nextAttemptAt = now.Add(CleanupRetryDelay);
|
||||
job.ScheduleRetry($"远端上传已完成,本地清理失败:{cleanupWarning}", nextAttemptAt, now, clearExternalTask: true);
|
||||
result.MarkUploadWaitingRetry("openlist", job.ErrorMessage, now);
|
||||
await _dbContext.SaveChangesAsync(cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
job.MarkSucceeded(now);
|
||||
@@ -542,6 +804,7 @@ public sealed class OpenListUploadQueueService
|
||||
string targetPath,
|
||||
string localPath,
|
||||
long expectedSize,
|
||||
bool allowSizeOnlyMatch,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var remote = await _openListClient.TryGetObjectAsync(connection, targetPath, cancellationToken);
|
||||
@@ -561,7 +824,7 @@ public sealed class OpenListUploadQueueService
|
||||
pair.Key.Equals("md5", StringComparison.OrdinalIgnoreCase));
|
||||
if (string.IsNullOrWhiteSpace(comparableHash.Key) || string.IsNullOrWhiteSpace(comparableHash.Value))
|
||||
{
|
||||
return TargetVerification.Match;
|
||||
return allowSizeOnlyMatch ? TargetVerification.Match : TargetVerification.Conflict;
|
||||
}
|
||||
|
||||
var localHash = await ComputeFileHashAsync(localPath, comparableHash.Key, cancellationToken);
|
||||
@@ -679,6 +942,26 @@ public sealed class OpenListUploadQueueService
|
||||
return index <= 0 ? "/" : normalized[..index];
|
||||
}
|
||||
|
||||
private static string GetFileName(string path)
|
||||
{
|
||||
var normalized = OpenListClient.NormalizePath(path);
|
||||
var index = normalized.LastIndexOf('/');
|
||||
return index < 0 ? normalized : normalized[(index + 1)..];
|
||||
}
|
||||
|
||||
private static string AppendTaskConflictSuffix(string path, Guid recordTaskId)
|
||||
{
|
||||
var directory = GetDirectoryName(path);
|
||||
var fileName = GetFileName(path);
|
||||
var extension = Path.GetExtension(fileName);
|
||||
var stem = extension.Length == 0 ? fileName : fileName[..^extension.Length];
|
||||
return OpenListClient.CombinePath(directory, $"{stem}_{recordTaskId.ToString("N")[..8]}{extension}");
|
||||
}
|
||||
|
||||
private static bool HasTaskConflictSuffix(string path, Guid recordTaskId) =>
|
||||
Path.GetFileNameWithoutExtension(GetFileName(path))
|
||||
.EndsWith($"_{recordTaskId.ToString("N")[..8]}", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static RecordArtifactUploadItemResultDto Failure(Guid recordTaskId, string message, string provider) => new()
|
||||
{
|
||||
RecordTaskId = recordTaskId,
|
||||
@@ -738,6 +1021,7 @@ public sealed class OpenListUploadQueueService
|
||||
public sealed class OpenListUploadBackgroundService : BackgroundService
|
||||
{
|
||||
private static readonly TimeSpan IdleDelay = TimeSpan.FromSeconds(2);
|
||||
private static readonly TimeSpan RecoveryInterval = TimeSpan.FromMinutes(1);
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
private readonly ILogger<OpenListUploadBackgroundService> _logger;
|
||||
|
||||
@@ -751,21 +1035,21 @@ public sealed class OpenListUploadBackgroundService : BackgroundService
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
var nextRecoveryAt = DateTimeOffset.MinValue;
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var queue = scope.ServiceProvider.GetRequiredService<OpenListUploadQueueService>();
|
||||
var processed = await queue.ProcessNextAsync(stoppingToken);
|
||||
if (!processed)
|
||||
_ = await queue.ProcessNextAsync(stoppingToken);
|
||||
if (DateTimeOffset.UtcNow >= nextRecoveryAt)
|
||||
{
|
||||
await Task.Delay(IdleDelay, stoppingToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
await Task.Delay(IdleDelay, stoppingToken);
|
||||
await queue.RecoverPendingAutomaticUploadsAsync(cancellationToken: stoppingToken);
|
||||
nextRecoveryAt = DateTimeOffset.UtcNow.Add(RecoveryInterval);
|
||||
}
|
||||
|
||||
await Task.Delay(IdleDelay, stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
|
||||
@@ -48,6 +48,17 @@ public sealed class RecordSessionCleanupResolver
|
||||
{
|
||||
var cutoff = DateTimeOffset.UtcNow.AddDays(-Math.Max(1, filters.RetentionDays));
|
||||
var nonEmptySessionIds = await ResolveConditionalSessionIdsAsync(filters, cutoff, cancellationToken);
|
||||
if (filters.RequireUploadSuccess && nonEmptySessionIds.Count > 0)
|
||||
{
|
||||
nonEmptySessionIds = await _dbContext.RecordSessions
|
||||
.AsNoTracking()
|
||||
.Where(session => nonEmptySessionIds.Contains(session.Id) &&
|
||||
_dbContext.RecordTasks
|
||||
.Where(task => task.RecordSessionId == session.Id)
|
||||
.All(task => task.Result != null && task.Result.UploadStatus == RecordArtifactUploadStatus.Succeeded))
|
||||
.Select(static session => session.Id)
|
||||
.ToArrayAsync(cancellationToken);
|
||||
}
|
||||
var emptySessionIds = await ResolveEmptySessionIdsAsync(cutoff, cancellationToken);
|
||||
|
||||
return nonEmptySessionIds
|
||||
@@ -75,8 +86,14 @@ public sealed class RecordSessionCleanupResolver
|
||||
IQueryable<RecordSession> query = _dbContext.RecordSessions
|
||||
.AsNoTracking()
|
||||
.Where(static item => item.Status != RecordSessionStatus.Starting &&
|
||||
item.Status != RecordSessionStatus.Pending &&
|
||||
item.Status != RecordSessionStatus.Running &&
|
||||
item.Status != RecordSessionStatus.Stopping)
|
||||
.Where(item => !_dbContext.RecordTasks.Any(task =>
|
||||
task.RecordSessionId == item.Id && task.UploadJob != null &&
|
||||
(task.UploadJob.Status == RecordArtifactUploadStatus.Queued ||
|
||||
task.UploadJob.Status == RecordArtifactUploadStatus.Uploading ||
|
||||
task.UploadJob.Status == RecordArtifactUploadStatus.WaitingRetry)))
|
||||
.Where(item => _dbContext.RecordTasks.Any(task => task.RecordSessionId == item.Id));
|
||||
|
||||
if (createdBeforeUtc.HasValue)
|
||||
@@ -133,6 +150,7 @@ public sealed class RecordSessionCleanupResolver
|
||||
IQueryable<RecordSession> query = _dbContext.RecordSessions
|
||||
.AsNoTracking()
|
||||
.Where(static item => item.Status != RecordSessionStatus.Starting &&
|
||||
item.Status != RecordSessionStatus.Pending &&
|
||||
item.Status != RecordSessionStatus.Running &&
|
||||
item.Status != RecordSessionStatus.Stopping)
|
||||
.Where(item => !_dbContext.RecordTasks.Any(task => task.RecordSessionId == item.Id));
|
||||
@@ -153,6 +171,7 @@ public sealed class RecordSessionCleanupResolver
|
||||
var activeSessionIds = await _dbContext.RecordSessions
|
||||
.AsNoTracking()
|
||||
.Where(static item => item.Status == RecordSessionStatus.Starting ||
|
||||
item.Status == RecordSessionStatus.Pending ||
|
||||
item.Status == RecordSessionStatus.Running ||
|
||||
item.Status == RecordSessionStatus.Stopping)
|
||||
.Select(static item => item.Id)
|
||||
|
||||
@@ -46,13 +46,19 @@ public sealed class RecoveryService
|
||||
Storage = new StorageGuardStatusDto
|
||||
{
|
||||
IsEnabled = storage.IsEnabled,
|
||||
IsAvailable = storage.IsAvailable,
|
||||
HasEnoughSpace = storage.HasEnoughSpace,
|
||||
CheckedPath = storage.CheckedPath,
|
||||
TotalBytes = storage.TotalBytes,
|
||||
UsedBytes = storage.UsedBytes,
|
||||
AvailableBytes = storage.AvailableBytes,
|
||||
RequiredBytes = storage.RequiredBytes,
|
||||
Message = storage.Message,
|
||||
Tier = storage.Tier.ToString(),
|
||||
UsagePercent = storage.UsagePercent
|
||||
UsagePercent = storage.UsagePercent,
|
||||
FreePercent = storage.FreePercent,
|
||||
GreenThresholdPercent = storage.GreenThresholdPercent,
|
||||
RedThresholdPercent = storage.RedThresholdPercent
|
||||
},
|
||||
LiveRooms = liveRooms,
|
||||
Finalizations = finalizations
|
||||
@@ -64,17 +70,17 @@ public sealed class RecoveryService
|
||||
var room = await _dbContext.LiveRooms.FirstOrDefaultAsync(item => item.Id == liveRoomId, cancellationToken);
|
||||
if (room is null)
|
||||
{
|
||||
return FailureResult("Live room was not found.");
|
||||
return FailureResult("未找到该直播间。");
|
||||
}
|
||||
|
||||
if (!room.IsEnabled)
|
||||
{
|
||||
return FailureResult("Live room is disabled and cannot be retried.");
|
||||
return FailureResult("该直播间已禁用,无法重试开录。");
|
||||
}
|
||||
|
||||
if (room.AvailabilityStatus != LiveRoomAvailabilityStatus.Live)
|
||||
{
|
||||
return FailureResult("Live room is not currently online.");
|
||||
return FailureResult("该直播间当前未开播,无法重试开录。");
|
||||
}
|
||||
|
||||
var hasActiveSession = await _dbContext.RecordSessions.AnyAsync(
|
||||
@@ -88,11 +94,11 @@ public sealed class RecoveryService
|
||||
{
|
||||
room.SetLastAutoStartDecision(
|
||||
AutoStartDecisionCodes.SkippedActiveSession,
|
||||
"Auto-start skipped because an active recording session already exists.",
|
||||
"已有活动录制会话,本次自动开录已跳过。",
|
||||
null,
|
||||
DateTimeOffset.UtcNow);
|
||||
await _dbContext.SaveChangesAsync(cancellationToken);
|
||||
return FailureResult("An active recording session already exists for this live room.");
|
||||
return FailureResult("该直播间已经存在活动录制会话。");
|
||||
}
|
||||
|
||||
try
|
||||
@@ -114,14 +120,14 @@ public sealed class RecoveryService
|
||||
Messages =
|
||||
[
|
||||
started
|
||||
? $"Recording retry started for room {room.RoomId}."
|
||||
: $"Recording retry did not start for room {room.RoomId}. Status={task.Status}; Error={task.ErrorMessage ?? "n/a"}"
|
||||
? $"直播间 {room.RoomId} 已开始重试录制。"
|
||||
: $"直播间 {room.RoomId} 未能开始录制。状态={task.Status};错误={task.ErrorMessage ?? "无"}"
|
||||
]
|
||||
};
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return FailureResult($"Recording retry failed for room {room.RoomId}: {ex.Message}");
|
||||
return FailureResult($"直播间 {room.RoomId} 重试录制失败:{ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,7 +141,7 @@ public sealed class RecoveryService
|
||||
RequestedCount = 0,
|
||||
SuccessCount = 0,
|
||||
FailedCount = 0,
|
||||
Messages = ["No live rooms currently require retry."]
|
||||
Messages = ["当前没有需要重试开录的直播间。"]
|
||||
};
|
||||
}
|
||||
|
||||
@@ -159,7 +165,8 @@ public sealed class RecoveryService
|
||||
|
||||
public async Task<RecoveryActionResultDto> ResumeFinalizationAsync(Guid recordTaskId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var started = await _ffmpegService.StartManualFinalizeTaskAsync(recordTaskId, cancellationToken);
|
||||
var recoveredOrphan = await _ffmpegService.TryRecoverOrphanedTerminalTaskAsync(recordTaskId, cancellationToken);
|
||||
var started = recoveredOrphan || await _ffmpegService.StartManualFinalizeTaskAsync(recordTaskId, cancellationToken);
|
||||
return new RecoveryActionResultDto
|
||||
{
|
||||
RequestedCount = 1,
|
||||
@@ -168,8 +175,10 @@ public sealed class RecoveryService
|
||||
Messages =
|
||||
[
|
||||
started
|
||||
? $"MP4 finalization resumed for task {recordTaskId}."
|
||||
: $"MP4 finalization could not be resumed for task {recordTaskId}."
|
||||
? recoveredOrphan
|
||||
? $"任务 {recordTaskId} 的遗留分片已恢复,并已进入后续上传流程。"
|
||||
: $"任务 {recordTaskId} 已恢复 MP4 转码。"
|
||||
: $"任务 {recordTaskId} 无法恢复 MP4 转码。"
|
||||
]
|
||||
};
|
||||
}
|
||||
@@ -184,7 +193,7 @@ public sealed class RecoveryService
|
||||
RequestedCount = 0,
|
||||
SuccessCount = 0,
|
||||
FailedCount = 0,
|
||||
Messages = ["No MP4 finalization tasks currently require recovery."]
|
||||
Messages = ["当前没有需要恢复的 MP4 转码任务。"]
|
||||
};
|
||||
}
|
||||
|
||||
@@ -192,7 +201,8 @@ public sealed class RecoveryService
|
||||
var messages = new List<string>();
|
||||
foreach (var item in finalizations)
|
||||
{
|
||||
var started = await _ffmpegService.StartManualFinalizeTaskAsync(item.RecordTaskId, cancellationToken);
|
||||
var recoveredOrphan = await _ffmpegService.TryRecoverOrphanedTerminalTaskAsync(item.RecordTaskId, cancellationToken);
|
||||
var started = recoveredOrphan || await _ffmpegService.StartManualFinalizeTaskAsync(item.RecordTaskId, cancellationToken);
|
||||
if (started)
|
||||
{
|
||||
successCount++;
|
||||
@@ -200,8 +210,10 @@ public sealed class RecoveryService
|
||||
|
||||
messages.Add(
|
||||
started
|
||||
? $"MP4 finalization resumed for task {item.RecordTaskId}."
|
||||
: $"MP4 finalization could not be resumed for task {item.RecordTaskId}.");
|
||||
? recoveredOrphan
|
||||
? $"任务 {item.RecordTaskId} 的遗留分片已恢复。"
|
||||
: $"任务 {item.RecordTaskId} 已恢复 MP4 转码。"
|
||||
: $"任务 {item.RecordTaskId} 无法恢复 MP4 转码。");
|
||||
}
|
||||
|
||||
return new RecoveryActionResultDto
|
||||
@@ -290,16 +302,6 @@ public sealed class RecoveryService
|
||||
return false;
|
||||
}
|
||||
|
||||
if (recordTask.Status == RecordTaskStatus.Processing)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (recordTask.Status is RecordTaskStatus.Starting or RecordTaskStatus.Running or RecordTaskStatus.Stopping)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (recordTask.RecordSession.Status is RecordSessionStatus.Starting or RecordSessionStatus.Running or RecordSessionStatus.Stopping)
|
||||
{
|
||||
return false;
|
||||
@@ -310,8 +312,9 @@ public sealed class RecoveryService
|
||||
return false;
|
||||
}
|
||||
|
||||
var manifestSources = FfmpegService.ReadRecorderSegmentsManifest(recordTask.OutputFilePath);
|
||||
var recorderOutputPath = ResolveManualFinalizeSourcePath(recordTask, recordTask.RecordSession);
|
||||
if (!File.Exists(recorderOutputPath))
|
||||
if (manifestSources.Count == 0 && !File.Exists(recorderOutputPath))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
@@ -330,16 +333,21 @@ public sealed class RecoveryService
|
||||
if (recordTask.Status == RecordTaskStatus.Processing)
|
||||
{
|
||||
return string.IsNullOrWhiteSpace(recordTask.ErrorMessage)
|
||||
? "MP4 finalization is queued or paused and can be resumed."
|
||||
? "MP4 转码正在排队或已暂停,可以继续恢复。"
|
||||
: recordTask.ErrorMessage!;
|
||||
}
|
||||
|
||||
if (recordTask.Status is RecordTaskStatus.Starting or RecordTaskStatus.Running or RecordTaskStatus.Stopping)
|
||||
{
|
||||
return "父会话已经结束,但该分片仍显示为录制中;可安全对账并恢复转码、上传。";
|
||||
}
|
||||
|
||||
if (recordTask.Status == RecordTaskStatus.Completed)
|
||||
{
|
||||
return "The final MP4 output is missing, but the intermediate recording file is still available.";
|
||||
return "最终 MP4 文件缺失,但中间录制文件仍然存在。";
|
||||
}
|
||||
|
||||
return "Manual MP4 finalization can be retried from the intermediate recording file.";
|
||||
return "可以使用保留的中间录制文件重新执行 MP4 转码。";
|
||||
}
|
||||
|
||||
private static string ResolveManualFinalizeSourcePath(RecordTask recordTask, RecordSession recordSession)
|
||||
|
||||
@@ -15,77 +15,92 @@ public sealed class StorageGuardService : IStorageGuardService
|
||||
}
|
||||
|
||||
public StorageGuardResult CheckCanStartOrResume(SystemSettingsDto settings, long additionalRequiredBytes = 0) =>
|
||||
Check(settings, Math.Max(settings.PauseRecordingWhenFreeSpaceBelowMegabytes, settings.ResumeRecordingWhenFreeSpaceAboveMegabytes), additionalRequiredBytes);
|
||||
Check(
|
||||
settings,
|
||||
Math.Max(settings.PauseRecordingWhenFreeSpaceBelowMegabytes, settings.ResumeRecordingWhenFreeSpaceAboveMegabytes),
|
||||
additionalRequiredBytes);
|
||||
|
||||
public StorageGuardResult CheckShouldPause(SystemSettingsDto settings) =>
|
||||
Check(settings, settings.PauseRecordingWhenFreeSpaceBelowMegabytes, additionalRequiredBytes: 0);
|
||||
|
||||
public StorageGuardResult CheckCanFinalize(SystemSettingsDto settings, long estimatedTemporaryBytes) =>
|
||||
Check(settings, settings.PauseRecordingWhenFreeSpaceBelowMegabytes, Math.Max(0, estimatedTemporaryBytes));
|
||||
|
||||
private StorageGuardResult Check(SystemSettingsDto settings, int freeSpaceThresholdMegabytes, long additionalRequiredBytes)
|
||||
{
|
||||
if (!settings.EnableStorageGuard)
|
||||
{
|
||||
return new StorageGuardResult(false, true, ResolveOutputRoot(settings.OutputRoot), long.MaxValue, 0, "Storage guard is disabled.")
|
||||
{
|
||||
Tier = StorageTier.Green,
|
||||
UsagePercent = 0
|
||||
};
|
||||
}
|
||||
|
||||
var checkedPath = ResolveOutputRoot(settings.OutputRoot);
|
||||
var thresholdBytes = Math.Max(0, freeSpaceThresholdMegabytes) * Megabyte;
|
||||
var requiredBytes = thresholdBytes + Math.Max(0, additionalRequiredBytes);
|
||||
var isEnabled = settings.EnableStorageGuard;
|
||||
var pauseThresholdBytes = isEnabled
|
||||
? Math.Max(0, settings.PauseRecordingWhenFreeSpaceBelowMegabytes) * Megabyte
|
||||
: 0;
|
||||
var resumeThresholdBytes = isEnabled
|
||||
? Math.Max(settings.PauseRecordingWhenFreeSpaceBelowMegabytes, settings.ResumeRecordingWhenFreeSpaceAboveMegabytes) * Megabyte
|
||||
: 0;
|
||||
var thresholdBytes = isEnabled ? Math.Max(0, freeSpaceThresholdMegabytes) * Megabyte : 0;
|
||||
var requiredBytes = thresholdBytes + (isEnabled ? Math.Max(0, additionalRequiredBytes) : 0);
|
||||
var greenThreshold = Math.Clamp(settings.StorageGreenThresholdPercent, 10, 90);
|
||||
var redThreshold = Math.Clamp(settings.StorageRedThresholdPercent, 5, greenThreshold - 5);
|
||||
var checkedPath = settings.OutputRoot?.Trim() ?? string.Empty;
|
||||
|
||||
try
|
||||
{
|
||||
checkedPath = ResolveOutputRoot(settings.OutputRoot ?? string.Empty);
|
||||
var drive = new DriveInfo(Path.GetPathRoot(checkedPath) ?? checkedPath);
|
||||
var availableBytes = drive.AvailableFreeSpace;
|
||||
var totalBytes = drive.TotalSize;
|
||||
var usedBytes = Math.Max(0, totalBytes - availableBytes);
|
||||
var usagePercent = totalBytes > 0 ? (double)usedBytes / totalBytes * 100.0 : 0;
|
||||
var freePercent = 100.0 - usagePercent;
|
||||
|
||||
// Determine tier using configurable thresholds
|
||||
var greenThreshold = Math.Clamp(settings.StorageGreenThresholdPercent, 5, 90);
|
||||
var redThreshold = Math.Clamp(settings.StorageRedThresholdPercent, 1, greenThreshold - 1);
|
||||
var freePercent = totalBytes > 0 ? (double)availableBytes / totalBytes * 100.0 : 0;
|
||||
|
||||
StorageTier tier;
|
||||
if (freePercent >= greenThreshold)
|
||||
{
|
||||
tier = StorageTier.Green;
|
||||
}
|
||||
else if (freePercent >= redThreshold)
|
||||
{
|
||||
tier = StorageTier.Yellow;
|
||||
}
|
||||
else
|
||||
if (freePercent < redThreshold || availableBytes < pauseThresholdBytes)
|
||||
{
|
||||
tier = StorageTier.Red;
|
||||
}
|
||||
|
||||
var hasEnoughSpace = availableBytes >= requiredBytes;
|
||||
var message = tier switch
|
||||
else if (freePercent >= greenThreshold && availableBytes >= resumeThresholdBytes)
|
||||
{
|
||||
StorageTier.Green => $"Storage is healthy. free={FormatBytes(availableBytes)} ({freePercent:F1}%), required={FormatBytes(requiredBytes)}, path={checkedPath}",
|
||||
StorageTier.Yellow => $"Storage is low. free={FormatBytes(availableBytes)} ({freePercent:F1}%), required={FormatBytes(requiredBytes)}, path={checkedPath}. New recordings paused, existing recordings continue.",
|
||||
StorageTier.Red => $"Storage is critically low. free={FormatBytes(availableBytes)} ({freePercent:F1}%), required={FormatBytes(requiredBytes)}, path={checkedPath}. All recordings paused, uploads continue.",
|
||||
tier = StorageTier.Green;
|
||||
}
|
||||
else
|
||||
{
|
||||
tier = StorageTier.Yellow;
|
||||
}
|
||||
|
||||
var hasEnoughSpace = !isEnabled || availableBytes >= requiredBytes;
|
||||
var message = !isEnabled
|
||||
? $"存储保护已关闭。可用={FormatBytes(availableBytes)}({freePercent:F1}%),路径={checkedPath}"
|
||||
: tier switch
|
||||
{
|
||||
StorageTier.Green => $"存储空间正常。可用={FormatBytes(availableBytes)}({freePercent:F1}%),本次需要={FormatBytes(requiredBytes)},路径={checkedPath}",
|
||||
StorageTier.Yellow => $"存储空间偏低。可用={FormatBytes(availableBytes)}({freePercent:F1}%),本次需要={FormatBytes(requiredBytes)},路径={checkedPath}。暂停新录制,已有录制继续。",
|
||||
StorageTier.Red => $"存储空间严重不足。可用={FormatBytes(availableBytes)}({freePercent:F1}%),本次需要={FormatBytes(requiredBytes)},路径={checkedPath}。暂停录制;空间足够保留安全余量时仍允许 MP4 收尾,上传继续。",
|
||||
_ => hasEnoughSpace
|
||||
? $"Storage is available. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}"
|
||||
: $"Storage is below threshold. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}"
|
||||
? $"存储空间可用。可用={FormatBytes(availableBytes)},本次需要={FormatBytes(requiredBytes)},路径={checkedPath}"
|
||||
: $"存储空间低于阈值。可用={FormatBytes(availableBytes)},本次需要={FormatBytes(requiredBytes)},路径={checkedPath}"
|
||||
};
|
||||
|
||||
return new StorageGuardResult(true, hasEnoughSpace, checkedPath, availableBytes, requiredBytes, message)
|
||||
return new StorageGuardResult(isEnabled, hasEnoughSpace, checkedPath, availableBytes, requiredBytes, message)
|
||||
{
|
||||
IsAvailable = totalBytes > 0,
|
||||
Tier = tier,
|
||||
UsagePercent = Math.Round(usagePercent, 1)
|
||||
TotalBytes = totalBytes,
|
||||
UsedBytes = usedBytes,
|
||||
UsagePercent = Math.Round(Math.Clamp(usagePercent, 0, 100), 1),
|
||||
FreePercent = Math.Round(Math.Clamp(freePercent, 0, 100), 1),
|
||||
GreenThresholdPercent = greenThreshold,
|
||||
RedThresholdPercent = redThreshold
|
||||
};
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Storage guard failed to inspect output root {OutputRoot}", checkedPath);
|
||||
return new StorageGuardResult(true, false, checkedPath, 0, requiredBytes, $"Unable to inspect storage path {checkedPath}: {ex.Message}")
|
||||
return new StorageGuardResult(isEnabled, !isEnabled, checkedPath, 0, requiredBytes, $"无法检查存储路径 {checkedPath}:{ex.Message}")
|
||||
{
|
||||
IsAvailable = false,
|
||||
Tier = StorageTier.Red,
|
||||
UsagePercent = 0
|
||||
UsagePercent = 0,
|
||||
FreePercent = 0,
|
||||
GreenThresholdPercent = greenThreshold,
|
||||
RedThresholdPercent = redThreshold
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
using LiveRecorder.Infrastructure.Persistence;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace LiveRecorder.Infrastructure.Services;
|
||||
|
||||
public sealed class SystemLogRetentionBackgroundService : BackgroundService
|
||||
{
|
||||
private const int BatchSize = 1000;
|
||||
private static readonly TimeSpan Retention = TimeSpan.FromDays(90);
|
||||
private static readonly TimeSpan Interval = TimeSpan.FromHours(24);
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
private readonly ILogger<SystemLogRetentionBackgroundService> _logger;
|
||||
|
||||
public SystemLogRetentionBackgroundService(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
ILogger<SystemLogRetentionBackgroundService> logger)
|
||||
{
|
||||
_scopeFactory = scopeFactory;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
var deleted = await DeleteExpiredLogsAsync(stoppingToken);
|
||||
if (deleted > 0)
|
||||
{
|
||||
_logger.LogInformation("Deleted {LogCount} system log entries older than 90 days.", deleted);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "System log retention cleanup failed.");
|
||||
}
|
||||
|
||||
await Task.Delay(Interval, stoppingToken);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<int> DeleteExpiredLogsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var cutoff = DateTimeOffset.UtcNow.Subtract(Retention);
|
||||
var total = 0;
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
|
||||
var ids = await dbContext.SystemLogEntries
|
||||
.AsNoTracking()
|
||||
.Where(log => log.CreatedAt < cutoff)
|
||||
.OrderBy(static log => log.CreatedAt)
|
||||
.Select(static log => log.Id)
|
||||
.Take(BatchSize)
|
||||
.ToArrayAsync(cancellationToken);
|
||||
if (ids.Length == 0)
|
||||
{
|
||||
return total;
|
||||
}
|
||||
|
||||
total += await dbContext.SystemLogEntries
|
||||
.Where(log => ids.Contains(log.Id))
|
||||
.ExecuteDeleteAsync(cancellationToken);
|
||||
if (ids.Length < BatchSize)
|
||||
{
|
||||
return total;
|
||||
}
|
||||
}
|
||||
|
||||
return total;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user