feat: harden recording lifecycle and refresh fnOS UI

This commit is contained in:
2026-08-03 23:45:26 +08:00
parent e5b50ea85c
commit ecc737f0bd
90 changed files with 6995 additions and 1826 deletions
@@ -119,6 +119,7 @@ public sealed class CleanupOperationCoordinator
new RetentionCleanupOperationFilters
{
RetentionDays = settings.RetentionDays,
RequireUploadSuccess = settings.RetentionRequireUploadSuccess,
VideoFileCondition = settings.RetentionVideoFileCondition,
TaskStatuses = settings.RetentionTaskStatuses
},
@@ -85,6 +85,8 @@ internal class ConditionalCleanupOperationFilters
internal sealed class RetentionCleanupOperationFilters : ConditionalCleanupOperationFilters
{
public int RetentionDays { get; init; } = 30;
public bool RequireUploadSuccess { get; init; }
}
internal sealed class EmptyCleanupOperationFilters
@@ -0,0 +1,144 @@
using LiveRecorder.Application.Abstractions.Scripting;
using LiveRecorder.Domain.Enums;
using LiveRecorder.Infrastructure.Persistence;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace LiveRecorder.Infrastructure.Services;
public sealed class CompletionDispatchService
{
private static readonly TimeSpan RetryDelay = TimeSpan.FromMinutes(1);
private readonly LiveRecorderDbContext _dbContext;
private readonly IEventScriptService _eventScriptService;
private readonly RecordUploadService _recordUploadService;
public CompletionDispatchService(
LiveRecorderDbContext dbContext,
IEventScriptService eventScriptService,
RecordUploadService recordUploadService)
{
_dbContext = dbContext;
_eventScriptService = eventScriptService;
_recordUploadService = recordUploadService;
}
public async Task<bool> TryDispatchNextAsync(CancellationToken cancellationToken = default)
{
var now = DateTimeOffset.UtcNow;
var dispatch = await _dbContext.RecordCompletionDispatches
.Include(item => item.RecordTask)!.ThenInclude(task => task!.LiveRoom)
.Include(item => item.RecordTask)!.ThenInclude(task => task!.RecordSession)
.Include(item => item.RecordTask)!.ThenInclude(task => task!.Result)
.Where(item => item.CompletedAt == null && (!item.NextAttemptAt.HasValue || item.NextAttemptAt <= now))
.OrderBy(static item => item.CreatedAt)
.FirstOrDefaultAsync(cancellationToken);
if (dispatch is null)
{
return false;
}
await DispatchAsync(dispatch, cancellationToken);
return true;
}
public async Task TryDispatchTaskAsync(Guid recordTaskId, CancellationToken cancellationToken = default)
{
var dispatch = await _dbContext.RecordCompletionDispatches
.Include(item => item.RecordTask)!.ThenInclude(task => task!.LiveRoom)
.Include(item => item.RecordTask)!.ThenInclude(task => task!.RecordSession)
.Include(item => item.RecordTask)!.ThenInclude(task => task!.Result)
.FirstOrDefaultAsync(item => item.RecordTaskId == recordTaskId, cancellationToken);
if (dispatch is not null && dispatch.CompletedAt is null)
{
await DispatchAsync(dispatch, cancellationToken);
}
}
private async Task DispatchAsync(Domain.Entities.RecordCompletionDispatch dispatch, CancellationToken cancellationToken)
{
var task = dispatch.RecordTask;
if (task?.RecordSession is null || task.Result is null)
{
dispatch.ScheduleRetry("录制任务上下文尚未准备完成。", DateTimeOffset.UtcNow.Add(RetryDelay), DateTimeOffset.UtcNow);
await _dbContext.SaveChangesAsync(cancellationToken);
return;
}
try
{
if (!dispatch.ScriptDispatched)
{
var scriptResult = await _eventScriptService.RunSegmentCompletedAsync(
task.LiveRoom,
task.RecordSession,
task,
task.Result,
task.Result.FilePath,
task.EndedAt ?? DateTimeOffset.UtcNow,
eventId: dispatch.Id,
cancellationToken: cancellationToken);
if (scriptResult is null || scriptResult.Success)
{
dispatch.MarkScriptDispatched(DateTimeOffset.UtcNow);
await _dbContext.SaveChangesAsync(cancellationToken);
}
else
{
throw new InvalidOperationException(scriptResult.Message);
}
}
if (!dispatch.UploadDispatched)
{
_ = await _recordUploadService.TryAutoUploadTaskAsync(task.Id, cancellationToken);
dispatch.MarkUploadDispatched(DateTimeOffset.UtcNow);
await _dbContext.SaveChangesAsync(cancellationToken);
}
}
catch (Exception ex)
{
dispatch.ScheduleRetry(ex.Message, DateTimeOffset.UtcNow.Add(RetryDelay), DateTimeOffset.UtcNow);
await _dbContext.SaveChangesAsync(cancellationToken);
}
}
}
public sealed class CompletionDispatchBackgroundService : BackgroundService
{
private readonly IServiceScopeFactory _scopeFactory;
private readonly ILogger<CompletionDispatchBackgroundService> _logger;
public CompletionDispatchBackgroundService(
IServiceScopeFactory scopeFactory,
ILogger<CompletionDispatchBackgroundService> logger)
{
_scopeFactory = scopeFactory;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
try
{
using var scope = _scopeFactory.CreateScope();
var service = scope.ServiceProvider.GetRequiredService<CompletionDispatchService>();
var processed = await service.TryDispatchNextAsync(stoppingToken);
await Task.Delay(processed ? TimeSpan.FromMilliseconds(200) : TimeSpan.FromSeconds(2), stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Completion dispatch worker iteration failed.");
await Task.Delay(TimeSpan.FromSeconds(2), stoppingToken);
}
}
}
}
@@ -100,11 +100,13 @@ public sealed class EventScriptService : IEventScriptService
string segmentFilePath,
DateTimeOffset occurredAt,
bool forceRun = false,
Guid? eventId = null,
CancellationToken cancellationToken = default)
{
var settings = await _settingsService.GetAsync(cancellationToken);
var environment = BuildLiveRoomEnvironment(liveRoom, occurredAt);
environment["LIVE_RECORDER_EVENT"] = "segment_completed";
environment["LIVE_RECORDER_EVENT_ID"] = (eventId ?? recordTask.Id).ToString();
environment["LIVE_RECORDER_RECORD_SESSION_ID"] = recordSession.Id.ToString();
environment["LIVE_RECORDER_RECORD_TASK_ID"] = recordTask.Id.ToString();
environment["LIVE_RECORDER_SEGMENT_INDEX"] = recordTask.SegmentIndex.ToString();
@@ -0,0 +1,297 @@
using System.Diagnostics;
using System.Text.RegularExpressions;
using LiveRecorder.Domain.Enums;
using Microsoft.Extensions.Logging;
namespace LiveRecorder.Infrastructure.Services;
public sealed partial class FfmpegService
{
internal static readonly FfmpegRecoveryContext InitialRecoveryContext = new(
FfmpegInputOptionProfile.Baseline,
AttemptCount: 0,
HasRetriedWithCompatibilityProfile: false,
HasRetriedWithRefreshedStream: false,
HasRetriedWithAlternateProtocol: false,
ForceSoftwareEncoder: false,
HasRetriedWithSoftwareEncoder: false);
internal static FfmpegRecoveryContext AdvanceRecoveryContext(
FfmpegRecoveryContext current,
FfmpegInputOptionProfile inputOptionProfile,
string currentProtocol,
string nextProtocol,
bool refreshedStream = false) =>
current with
{
InputOptionProfile = inputOptionProfile,
AttemptCount = current.AttemptCount + 1,
HasRetriedWithRefreshedStream = current.HasRetriedWithRefreshedStream || refreshedStream,
HasRetriedWithAlternateProtocol = current.HasRetriedWithAlternateProtocol ||
!string.Equals(currentProtocol, nextProtocol, StringComparison.OrdinalIgnoreCase)
};
internal static bool ShouldImmediatelyFallbackFromHls(
string selectedProtocol,
bool hasHlsOverlongHeadersFailure) =>
hasHlsOverlongHeadersFailure &&
string.Equals(selectedProtocol, "hls", StringComparison.OrdinalIgnoreCase);
internal static bool ShouldApplyRuntimeFailureBackoff(
RecordSessionStatus status,
TimeSpan processRuntime) =>
status == RecordSessionStatus.Failed && processRuntime < StableRuntimeResetThreshold;
private async Task<RecoveryVideoEncoderSelection> ResolveRecoveryVideoEncoderAsync(
string ffmpegPath,
bool forceSoftware,
CancellationToken cancellationToken)
{
if (forceSoftware)
{
return RecoveryVideoEncoderSelection.Software;
}
if (_hasProbedRecoveryVideoEncoder)
{
return _cachedRecoveryVideoEncoder;
}
await _recoveryEncoderProbeGate.WaitAsync(cancellationToken);
try
{
if (_hasProbedRecoveryVideoEncoder)
{
return _cachedRecoveryVideoEncoder;
}
var candidates = new List<RecoveryVideoEncoderSelection>
{
new(RecoveryVideoEncoderKind.Nvenc, null)
};
IReadOnlyList<string> renderDevices = [];
try
{
if (Directory.Exists("/dev/dri"))
{
renderDevices = Directory.EnumerateFileSystemEntries("/dev/dri", "renderD*")
.OrderBy(static path => path)
.ToList();
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
_logger.LogDebug(ex, "Unable to enumerate /dev/dri render devices for recovery encoding.");
}
foreach (var devicePath in renderDevices)
{
candidates.Add(new RecoveryVideoEncoderSelection(RecoveryVideoEncoderKind.Qsv, devicePath));
}
foreach (var devicePath in renderDevices)
{
candidates.Add(new RecoveryVideoEncoderSelection(RecoveryVideoEncoderKind.Vaapi, devicePath));
}
foreach (var candidate in candidates)
{
if (await ProbeRecoveryVideoEncoderAsync(ffmpegPath, candidate, cancellationToken))
{
_cachedRecoveryVideoEncoder = candidate;
_hasProbedRecoveryVideoEncoder = true;
_logger.LogInformation(
"Recovery video encoder probe selected {EncoderKind} using device {DevicePath}",
candidate.Kind,
candidate.DevicePath ?? "default");
return candidate;
}
}
_cachedRecoveryVideoEncoder = RecoveryVideoEncoderSelection.Software;
_hasProbedRecoveryVideoEncoder = true;
_logger.LogInformation("No usable hardware recovery encoder was detected; libx264 will be used.");
return _cachedRecoveryVideoEncoder;
}
finally
{
_recoveryEncoderProbeGate.Release();
}
}
private async Task<bool> ProbeRecoveryVideoEncoderAsync(
string ffmpegPath,
RecoveryVideoEncoderSelection encoder,
CancellationToken cancellationToken)
{
using var process = new Process
{
StartInfo = new ProcessStartInfo
{
FileName = ffmpegPath,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true
}
};
foreach (var argument in BuildRecoveryEncoderProbeArgumentList(encoder))
{
process.StartInfo.ArgumentList.Add(argument);
}
try
{
if (!process.Start())
{
return false;
}
var outputTask = process.StandardOutput.ReadToEndAsync(cancellationToken);
var errorTask = process.StandardError.ReadToEndAsync(cancellationToken);
using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
timeoutCts.CancelAfter(TimeSpan.FromSeconds(8));
await process.WaitForExitAsync(timeoutCts.Token);
await Task.WhenAll(outputTask, errorTask);
return process.ExitCode == 0;
}
catch (Exception ex) when (ex is InvalidOperationException or System.ComponentModel.Win32Exception or OperationCanceledException)
{
try
{
if (!process.HasExited)
{
process.Kill(true);
}
}
catch
{
}
_logger.LogDebug(
ex,
"Recovery video encoder probe failed for {EncoderKind} using device {DevicePath}",
encoder.Kind,
encoder.DevicePath ?? "default");
return false;
}
}
private void DisableRecoveryVideoEncoder(RecoveryVideoEncoderSelection encoder)
{
if (encoder.Kind == RecoveryVideoEncoderKind.Software ||
_cachedRecoveryVideoEncoder != encoder)
{
return;
}
_cachedRecoveryVideoEncoder = RecoveryVideoEncoderSelection.Software;
_hasProbedRecoveryVideoEncoder = true;
_logger.LogWarning(
"Recovery video encoder {EncoderKind} failed with a live input and was disabled until the service restarts.",
encoder.Kind);
}
internal static IReadOnlyList<string> BuildRecoveryEncoderProbeArgumentList(
RecoveryVideoEncoderSelection encoder)
{
var arguments = new List<string> { "-hide_banner", "-loglevel", "error" };
AddRecoveryEncoderDeviceArguments(arguments, encoder);
arguments.AddRange(["-f", "lavfi", "-i", "color=c=black:s=128x128:r=1", "-frames:v", "1"]);
arguments.AddRange(BuildRecoveryVideoCodecArguments(encoder));
arguments.AddRange(["-an", "-f", "null", "-"]);
return arguments;
}
internal static IReadOnlyList<RecoverableRecorderSegment> DiscoverRecoverableSegments(
string? outputPathPattern,
RecordOutputFormat outputFormat,
RecordSaveMode saveMode)
{
if (string.IsNullOrWhiteSpace(outputPathPattern) ||
outputFormat != RecordOutputFormat.Mp4 ||
saveMode != RecordSaveMode.Segmented ||
!outputPathPattern.Contains("%05d", StringComparison.OrdinalIgnoreCase))
{
return [];
}
var absoluteOutputPattern = NormalizeAbsolutePath(outputPathPattern);
var recorderPattern = GetRecorderOutputPath(absoluteOutputPattern, outputFormat, saveMode);
var directory = Path.GetDirectoryName(recorderPattern);
var filePattern = Path.GetFileName(recorderPattern);
if (string.IsNullOrWhiteSpace(directory) ||
string.IsNullOrWhiteSpace(filePattern) ||
!Directory.Exists(directory))
{
return [];
}
var tokenIndex = filePattern.IndexOf("%05d", StringComparison.OrdinalIgnoreCase);
if (tokenIndex < 0)
{
return [];
}
var prefix = filePattern[..tokenIndex];
var suffix = filePattern[(tokenIndex + 4)..];
var matcher = new Regex(
$"^{Regex.Escape(prefix)}(?<segment>[0-9]{{5}}){Regex.Escape(suffix)}$",
RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
try
{
return Directory.EnumerateFiles(directory, $"{prefix}*{suffix}", SearchOption.TopDirectoryOnly)
.Select(path => (Path: path, Match: matcher.Match(Path.GetFileName(path))))
.Where(item => item.Match.Success && new FileInfo(item.Path).Length > 0)
.Select(item => new
{
RecorderPath = NormalizeAbsolutePath(item.Path),
SegmentIndex = int.Parse(item.Match.Groups["segment"].Value)
})
.Where(item => item.SegmentIndex > 0)
.GroupBy(item => item.SegmentIndex)
.Select(group => group.First())
.OrderBy(item => item.SegmentIndex)
.Select(item => new RecoverableRecorderSegment(
item.SegmentIndex,
item.RecorderPath,
NormalizeAbsolutePath(ResolveSegmentOutputPath(absoluteOutputPattern, saveMode, item.SegmentIndex))))
.ToList();
}
catch (IOException)
{
return [];
}
catch (UnauthorizedAccessException)
{
return [];
}
}
}
internal sealed record FfmpegRecoveryContext(
FfmpegService.FfmpegInputOptionProfile InputOptionProfile,
int AttemptCount,
bool HasRetriedWithCompatibilityProfile,
bool HasRetriedWithRefreshedStream,
bool HasRetriedWithAlternateProtocol,
bool ForceSoftwareEncoder,
bool HasRetriedWithSoftwareEncoder);
internal enum RecoveryVideoEncoderKind
{
Software = 0,
Nvenc = 1,
Qsv = 2,
Vaapi = 3
}
internal sealed record RecoveryVideoEncoderSelection(RecoveryVideoEncoderKind Kind, string? DevicePath)
{
public static RecoveryVideoEncoderSelection Software { get; } = new(RecoveryVideoEncoderKind.Software, null);
}
internal sealed record RecoverableRecorderSegment(int SegmentIndex, string RecorderPath, string OutputPath);
File diff suppressed because it is too large Load Diff
@@ -1,7 +1,9 @@
using System.Diagnostics;
using System.Globalization;
using System.Text;
using System.Text.Json;
using LiveRecorder.Application.Abstractions.Platforms;
using LiveRecorder.Application.Abstractions.Recording;
using LiveRecorder.Domain.Entities;
using LiveRecorder.Domain.Enums;
using LiveRecorder.Infrastructure.Persistence;
@@ -13,7 +15,9 @@ namespace LiveRecorder.Infrastructure.Services;
public sealed partial class FfmpegService
{
private const string LowStoragePauseErrorPrefix = "MP4 finalization paused because storage tier is Red (critically low).";
private const string LowStoragePauseErrorPrefix = "MP4 finalization paused because there is not enough temporary disk space.";
private const string ShutdownFinalizationPauseErrorPrefix = "MP4 finalization paused because the application is shutting down.";
private const string RecorderSegmentsManifestSuffix = ".segments.json";
private static readonly TimeSpan Mp4FinalizeInactivityTimeout = TimeSpan.FromMinutes(10);
private static readonly TimeSpan Mp4FinalizePollInterval = TimeSpan.FromSeconds(1);
@@ -23,14 +27,24 @@ public sealed partial class FfmpegService
int mp4FinalizeTimeoutMinutes,
Guid recordSessionId,
Guid recordTaskId,
string sourcePath,
IReadOnlyList<string> sourcePaths,
string targetPath,
double? expectedDurationSeconds,
string? segmentsManifestPath,
CancellationToken cancellationToken)
{
if (!File.Exists(sourcePath))
using var shutdownLinkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _shutdownCts.Token);
cancellationToken = shutdownLinkedCts.Token;
var normalizedSourcePaths = sourcePaths
.Where(static path => !string.IsNullOrWhiteSpace(path))
.Select(NormalizeAbsolutePath)
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray();
if (normalizedSourcePaths.Length == 0 || normalizedSourcePaths.Any(path => !File.Exists(path)))
{
return (File.Exists(targetPath) ? targetPath : sourcePath, "The intermediate recording file was not found for MP4 finalization.");
return (
File.Exists(targetPath) ? targetPath : normalizedSourcePaths.FirstOrDefault() ?? targetPath,
"One or more intermediate recording files were not found for MP4 finalization.");
}
var tempPath = Path.Combine(
@@ -47,9 +61,9 @@ public sealed partial class FfmpegService
using var storageScope = _serviceScopeFactory.CreateScope();
var settingsService = storageScope.ServiceProvider.GetRequiredService<LiveRecorder.Application.Abstractions.Settings.ISystemSettingsService>();
var settings = await settingsService.GetAsync(cancellationToken);
var sourceSizeBytes = new FileInfo(sourcePath).Length;
var storageCheck = _storageGuardService.CheckCanStartOrResume(settings, sourceSizeBytes);
return storageCheck.ShouldPauseActive
var sourceSizeBytes = normalizedSourcePaths.Sum(path => new FileInfo(path).Length);
var storageCheck = _storageGuardService.CheckCanFinalize(settings, sourceSizeBytes);
return !storageCheck.HasEnoughSpace
? $"{LowStoragePauseErrorPrefix} {storageCheck.Message}"
: null;
}
@@ -58,7 +72,7 @@ public sealed partial class FfmpegService
if (!string.IsNullOrWhiteSpace(lowStoragePauseMessage))
{
SetPostProcessState(recordSessionId, recordTaskId, "Paused: Low storage", null, lowStoragePauseMessage);
return (sourcePath, lowStoragePauseMessage);
return (normalizedSourcePaths[0], lowStoragePauseMessage);
}
SetPostProcessState(
@@ -74,7 +88,7 @@ public sealed partial class FfmpegService
if (!string.IsNullOrWhiteSpace(lowStoragePauseMessage))
{
SetPostProcessState(recordSessionId, recordTaskId, "Paused: Low storage", null, lowStoragePauseMessage);
return (sourcePath, lowStoragePauseMessage);
return (normalizedSourcePaths[0], lowStoragePauseMessage);
}
var stderrLines = new Queue<string>();
@@ -178,6 +192,10 @@ public sealed partial class FfmpegService
SetPostProcessState(recordSessionId, recordTaskId, stage, 0, detail);
var concatInputPath = normalizedSourcePaths.Length > 1
? await WriteFfconcatInputAsync(targetPath, recordTaskId, normalizedSourcePaths, cancellationToken)
: null;
using var finalizeProcess = new Process
{
StartInfo = new ProcessStartInfo
@@ -190,7 +208,11 @@ public sealed partial class FfmpegService
}
};
foreach (var argument in BuildMp4FinalizeArgumentList(sourcePath, tempPath, strategy))
foreach (var argument in BuildMp4FinalizeArgumentList(
concatInputPath ?? normalizedSourcePaths[0],
tempPath,
strategy,
concatInputPath is not null))
{
finalizeProcess.StartInfo.ArgumentList.Add(argument);
}
@@ -238,6 +260,7 @@ public sealed partial class FfmpegService
try
{
finalizeProcess.Start();
_postProcessProcesses[recordTaskId] = finalizeProcess;
finalizeProcess.BeginOutputReadLine();
finalizeProcess.BeginErrorReadLine();
@@ -278,6 +301,22 @@ public sealed partial class FfmpegService
return ex.Message;
}
catch (OperationCanceledException) when (_shutdownCts.IsCancellationRequested)
{
try
{
if (!finalizeProcess.HasExited)
{
finalizeProcess.Kill(true);
}
}
catch (Exception killEx)
{
_logger.LogWarning(killEx, "Shutdown-interrupted MP4 finalization process could not be killed for task {RecordTaskId}", recordTaskId);
}
return ShutdownFinalizationPauseErrorPrefix;
}
catch (Exception ex)
{
try
@@ -294,6 +333,14 @@ public sealed partial class FfmpegService
return ex.Message;
}
finally
{
_postProcessProcesses.TryRemove(recordTaskId, out _);
if (!string.IsNullOrWhiteSpace(concatInputPath) && File.Exists(concatInputPath))
{
File.Delete(concatInputPath);
}
}
if (finalizeProcess.ExitCode == 0 && File.Exists(tempPath))
{
@@ -344,9 +391,17 @@ public sealed partial class FfmpegService
File.Move(tempPath, targetPath);
}
if (!string.Equals(sourcePath, targetPath, StringComparison.OrdinalIgnoreCase) && File.Exists(sourcePath))
foreach (var sourcePath in normalizedSourcePaths)
{
File.Delete(sourcePath);
if (!string.Equals(sourcePath, targetPath, StringComparison.OrdinalIgnoreCase) && File.Exists(sourcePath))
{
File.Delete(sourcePath);
}
}
if (!string.IsNullOrWhiteSpace(segmentsManifestPath) && File.Exists(segmentsManifestPath))
{
File.Delete(segmentsManifestPath);
}
SetPostProcessState(recordSessionId, recordTaskId, "Completed", 100, "MP4 seek index is ready");
@@ -358,7 +413,7 @@ public sealed partial class FfmpegService
File.Delete(tempPath);
}
var fallbackPath = File.Exists(targetPath) ? targetPath : sourcePath;
var fallbackPath = File.Exists(targetPath) ? targetPath : normalizedSourcePaths[0];
string? errorDetail;
lock (stderrLines)
{
@@ -426,9 +481,10 @@ public sealed partial class FfmpegService
mp4FinalizeTimeoutMinutes,
recordSession.Id,
recordTask.Id,
recorderOutputPath,
[recorderOutputPath],
finalOutputPath,
expectedDurationSeconds,
segmentsManifestPath: null,
cancellationToken);
}
@@ -462,100 +518,104 @@ public sealed partial class FfmpegService
return (File.Exists(finalSegmentOutputPath) ? finalSegmentOutputPath : recordTask.OutputFilePath, null);
}
string? materializedSourcePath = null;
var cleanupMaterializedSource = false;
var preserveMaterializedSource = false;
try
string? segmentsManifestPath = null;
if (normalizedRecorderSegmentPaths.Length > 1)
{
materializedSourcePath = await MaterializeRecorderSegmentSourceAsync(
segmentsManifestPath = GetRecorderSegmentsManifestPath(finalSegmentOutputPath);
await WriteRecorderSegmentsManifestAsync(
segmentsManifestPath,
finalSegmentOutputPath,
normalizedRecorderSegmentPaths,
cancellationToken);
if (string.IsNullOrWhiteSpace(materializedSourcePath))
{
return (File.Exists(finalSegmentOutputPath) ? finalSegmentOutputPath : recordTask.OutputFilePath, null);
}
cleanupMaterializedSource =
!string.Equals(materializedSourcePath, normalizedRecorderSegmentPaths[0], StringComparison.OrdinalIgnoreCase);
var finalizationResult = await TryFinalizeMp4Async(
ffmpegPath,
maxConcurrentTranscodeTasks,
mp4FinalizeTimeoutMinutes,
recordSession.Id,
recordTask.Id,
materializedSourcePath,
finalSegmentOutputPath,
expectedDurationSeconds,
cancellationToken);
preserveMaterializedSource = IsLowStoragePauseError(finalizationResult.ErrorMessage);
return finalizationResult;
}
finally
{
if (cleanupMaterializedSource &&
!preserveMaterializedSource &&
!string.IsNullOrWhiteSpace(materializedSourcePath) &&
File.Exists(materializedSourcePath))
{
File.Delete(materializedSourcePath);
}
}
return await TryFinalizeMp4Async(
ffmpegPath,
maxConcurrentTranscodeTasks,
mp4FinalizeTimeoutMinutes,
recordSession.Id,
recordTask.Id,
normalizedRecorderSegmentPaths,
finalSegmentOutputPath,
expectedDurationSeconds,
segmentsManifestPath,
cancellationToken);
}
private static async Task<string?> MaterializeRecorderSegmentSourceAsync(
private static string GetRecorderSegmentsManifestPath(string finalOutputPath) =>
$"{NormalizeAbsolutePath(finalOutputPath)}{RecorderSegmentsManifestSuffix}";
private static async Task WriteRecorderSegmentsManifestAsync(
string manifestPath,
string finalOutputPath,
IReadOnlyList<string> recorderSegmentPaths,
CancellationToken cancellationToken)
{
if (recorderSegmentPaths.Count == 0)
{
return null;
}
if (recorderSegmentPaths.Count == 1)
{
return recorderSegmentPaths[0];
}
var combinedPath = Path.Combine(
Path.GetDirectoryName(finalOutputPath)!,
$"{Path.GetFileNameWithoutExtension(finalOutputPath)}.concat.ts");
if (File.Exists(combinedPath))
{
File.Delete(combinedPath);
}
await using var outputStream = new FileStream(
combinedPath,
FileMode.CreateNew,
FileAccess.Write,
FileShare.None,
bufferSize: 1024 * 128,
useAsync: true);
foreach (var path in recorderSegmentPaths)
{
await using var inputStream = new FileStream(
path,
FileMode.Open,
FileAccess.Read,
FileShare.ReadWrite,
bufferSize: 1024 * 128,
useAsync: true);
await inputStream.CopyToAsync(outputStream, 1024 * 128, cancellationToken);
}
await outputStream.FlushAsync(cancellationToken);
return combinedPath;
var payload = new RecorderSegmentsManifest(
NormalizeAbsolutePath(finalOutputPath),
recorderSegmentPaths.Select(NormalizeAbsolutePath).ToArray(),
DateTimeOffset.UtcNow);
var temporaryPath = $"{manifestPath}.{Guid.NewGuid():N}.tmp";
await File.WriteAllTextAsync(temporaryPath, JsonSerializer.Serialize(payload), Encoding.UTF8, cancellationToken);
File.Move(temporaryPath, manifestPath, overwrite: true);
}
private static IReadOnlyList<string> BuildMp4FinalizeArgumentList(
internal static IReadOnlyList<string> ReadRecorderSegmentsManifest(string finalOutputPath)
{
var manifestPath = GetRecorderSegmentsManifestPath(finalOutputPath);
if (!File.Exists(manifestPath))
{
return [];
}
try
{
var manifest = JsonSerializer.Deserialize<RecorderSegmentsManifest>(File.ReadAllText(manifestPath));
return manifest?.SourcePaths
.Where(static path => !string.IsNullOrWhiteSpace(path))
.Select(NormalizeAbsolutePath)
.Distinct(StringComparer.OrdinalIgnoreCase)
.Where(File.Exists)
.ToArray() ?? [];
}
catch (JsonException)
{
return [];
}
catch (IOException)
{
return [];
}
}
private static async Task<string> WriteFfconcatInputAsync(
string targetPath,
Guid recordTaskId,
IReadOnlyList<string> sourcePaths,
CancellationToken cancellationToken)
{
var concatPath = Path.Combine(
Path.GetDirectoryName(targetPath)!,
$".{Path.GetFileName(targetPath)}.{recordTaskId:N}.ffconcat");
var content = new StringBuilder("ffconcat version 1.0\n");
foreach (var sourcePath in sourcePaths)
{
content.Append("file '")
.Append(NormalizeAbsolutePath(sourcePath)
.Replace("\\", "\\\\", StringComparison.Ordinal)
.Replace("'", "\\'", StringComparison.Ordinal))
.Append("'\n");
}
await File.WriteAllTextAsync(concatPath, content.ToString(), Encoding.UTF8, cancellationToken);
return concatPath;
}
internal static IReadOnlyList<string> BuildMp4FinalizeArgumentList(
string sourcePath,
string targetPath,
Mp4FinalizeStrategy strategy)
Mp4FinalizeStrategy strategy,
bool useConcatDemuxer = false)
{
var arguments = new List<string>
{
@@ -571,16 +631,24 @@ public sealed partial class FfmpegService
"-fflags",
"+genpts+igndts+discardcorrupt",
"-err_detect",
"ignore_err",
"-i",
sourcePath,
"ignore_err"
};
if (useConcatDemuxer)
{
arguments.AddRange(["-f", "concat", "-safe", "0"]);
}
arguments.AddRange(
[
"-i", sourcePath,
"-map",
"0:v:0",
"-map",
"0:a:0?",
"-dn",
"-sn"
};
]);
if (strategy == Mp4FinalizeStrategy.RepairTranscode)
{
@@ -610,7 +678,7 @@ public sealed partial class FfmpegService
return arguments;
}
private static bool IsRepairableMp4FinalizeError(string errorDetail)
internal static bool IsRepairableMp4FinalizeError(string errorDetail)
{
if (IsLowStoragePauseError(errorDetail))
{
@@ -622,25 +690,37 @@ public sealed partial class FfmpegService
errorDetail.Contains("incorrect codec parameters", StringComparison.OrdinalIgnoreCase) ||
errorDetail.Contains("codec parameters", StringComparison.OrdinalIgnoreCase) ||
errorDetail.Contains("Invalid data found when processing input", StringComparison.OrdinalIgnoreCase) ||
errorDetail.Contains("Error initializing output stream", StringComparison.OrdinalIgnoreCase);
errorDetail.Contains("Error initializing output stream", StringComparison.OrdinalIgnoreCase) ||
errorDetail.Contains("Error writing trailer", StringComparison.OrdinalIgnoreCase) ||
errorDetail.Contains("Error muxing a packet", StringComparison.OrdinalIgnoreCase) ||
errorDetail.Contains("Conversion failed", StringComparison.OrdinalIgnoreCase);
}
private static bool IsLowStoragePauseError(string? errorDetail) =>
!string.IsNullOrWhiteSpace(errorDetail) &&
errorDetail.Contains(LowStoragePauseErrorPrefix, StringComparison.OrdinalIgnoreCase);
private static bool IsShutdownFinalizationPauseError(string? errorDetail) =>
!string.IsNullOrWhiteSpace(errorDetail) &&
errorDetail.Contains(ShutdownFinalizationPauseErrorPrefix, StringComparison.OrdinalIgnoreCase);
private static bool IsNoSpaceLeftError(string? errorDetail) =>
!string.IsNullOrWhiteSpace(errorDetail) &&
(errorDetail.Contains("No space left on device", StringComparison.OrdinalIgnoreCase) ||
errorDetail.Contains("database or disk is full", StringComparison.OrdinalIgnoreCase));
private enum Mp4FinalizeStrategy
internal enum Mp4FinalizeStrategy
{
StreamCopy,
RepairTranscode
}
private static IReadOnlyList<string> BuildArgumentList(
private sealed record RecorderSegmentsManifest(
string TargetPath,
IReadOnlyList<string> SourcePaths,
DateTimeOffset CreatedAt);
internal static IReadOnlyList<string> BuildArgumentList(
string streamUrl,
string outputFilePath,
RecordOutputFormat outputFormat,
@@ -654,13 +734,15 @@ public sealed partial class FfmpegService
StreamInputHeaders? inputHeaders,
string? selectedProtocol,
string? selectedVideoCodec,
FfmpegInputOptionProfile inputOptionProfile)
FfmpegInputOptionProfile inputOptionProfile,
RecoveryVideoEncoderSelection? recoveryVideoEncoder = null)
{
var arguments = new List<string> { "-hide_banner", "-y", "-progress", "pipe:1" };
var arguments = new List<string> { "-hide_banner", "-n", "-progress", "pipe:1" };
recoveryVideoEncoder ??= RecoveryVideoEncoderSelection.Software;
var useIntermediateTransportStream = ShouldUseIntermediateTransportStream(outputFilePath, outputFormat, saveMode);
var effectiveStreamUrl = streamUrl;
var isPipeInput = IsHttpInput(streamUrl);
var isPipeInput = ShouldUseCurlPipe(streamUrl, selectedProtocol);
if (isPipeInput)
{
// When the input is HTTP, use pipe:0 so that curl handles the HTTP connection.
@@ -671,7 +753,7 @@ public sealed partial class FfmpegService
}
if (enableReconnect &&
inputOptionProfile == FfmpegInputOptionProfile.Baseline &&
inputOptionProfile != FfmpegInputOptionProfile.Minimal &&
!isPipeInput &&
ShouldEnableReconnect(streamUrl, selectedProtocol))
{
@@ -684,8 +766,21 @@ public sealed partial class FfmpegService
]);
}
arguments.AddRange(["-rw_timeout", readWriteTimeoutMilliseconds.ToString(), "-fflags", "+discardcorrupt+genpts", "-i", effectiveStreamUrl]);
arguments.AddRange(BuildCodecArguments(recordingTemplate));
if (!isPipeInput)
{
AddNativeHttpInputHeaders(arguments, streamUrl, inputHeaders);
}
if (inputOptionProfile == FfmpegInputOptionProfile.TimestampTranscode)
{
AddRecoveryEncoderDeviceArguments(arguments, recoveryVideoEncoder);
}
var inputFlags = inputOptionProfile is FfmpegInputOptionProfile.TimestampRepair or FfmpegInputOptionProfile.TimestampTranscode
? "+discardcorrupt+genpts+igndts"
: "+discardcorrupt+genpts";
arguments.AddRange(["-rw_timeout", readWriteTimeoutMilliseconds.ToString(), "-fflags", inputFlags, "-i", effectiveStreamUrl]);
arguments.AddRange(BuildCodecArguments(recordingTemplate, inputOptionProfile, recoveryVideoEncoder));
var writesTransportStream = useIntermediateTransportStream || outputFormat == RecordOutputFormat.Ts;
var bitstreamFilter = ResolveTransportStreamBitstreamFilter(recordingTemplate, writesTransportStream, selectedVideoCodec);
@@ -728,8 +823,63 @@ public sealed partial class FfmpegService
return arguments;
}
private static IReadOnlyList<string> BuildCodecArguments(RecordingTemplateType recordingTemplate) =>
recordingTemplate switch
private static IReadOnlyList<string> BuildCodecArguments(
RecordingTemplateType recordingTemplate,
FfmpegInputOptionProfile inputOptionProfile,
RecoveryVideoEncoderSelection recoveryVideoEncoder)
{
if (inputOptionProfile == FfmpegInputOptionProfile.TimestampTranscode)
{
var arguments = new List<string>
{
"-map", "0:v:0",
"-map", "0:a:0?",
};
arguments.AddRange(BuildRecoveryVideoCodecArguments(recoveryVideoEncoder));
arguments.AddRange(
[
"-af", "aresample=async=1:first_pts=0,asetpts=PTS-STARTPTS",
"-c:a", "aac",
"-b:a", "128k",
"-avoid_negative_ts", "make_zero"
]);
return arguments;
}
if (inputOptionProfile == FfmpegInputOptionProfile.TimestampRepair)
{
return recordingTemplate switch
{
RecordingTemplateType.BalancedMp4 =>
[
"-c:v", "libx264",
"-preset", "veryfast",
"-crf", "23",
"-c:a", "aac",
"-af", "aresample=async=1:first_pts=0",
"-b:a", "128k"
],
RecordingTemplateType.ArchiveTs =>
[
"-map", "0",
"-c", "copy",
"-c:a", "aac",
"-af", "aresample=async=1:first_pts=0",
"-b:a", "128k"
],
_ =>
[
"-map", "0:v:0",
"-map", "0:a:0?",
"-c:v", "copy",
"-c:a", "aac",
"-af", "aresample=async=1:first_pts=0",
"-b:a", "128k"
]
};
}
return recordingTemplate switch
{
RecordingTemplateType.BalancedMp4 =>
[
@@ -749,6 +899,66 @@ public sealed partial class FfmpegService
"-c", "copy"
]
};
}
internal static void AddRecoveryEncoderDeviceArguments(
ICollection<string> arguments,
RecoveryVideoEncoderSelection encoder)
{
if (string.IsNullOrWhiteSpace(encoder.DevicePath))
{
return;
}
if (encoder.Kind == RecoveryVideoEncoderKind.Qsv)
{
arguments.Add("-qsv_device");
arguments.Add(encoder.DevicePath);
}
else if (encoder.Kind == RecoveryVideoEncoderKind.Vaapi)
{
arguments.Add("-vaapi_device");
arguments.Add(encoder.DevicePath);
}
}
internal static IReadOnlyList<string> BuildRecoveryVideoCodecArguments(
RecoveryVideoEncoderSelection encoder) =>
encoder.Kind switch
{
RecoveryVideoEncoderKind.Nvenc =>
[
"-vf", "settb=AVTB,setpts=PTS-STARTPTS,format=yuv420p",
"-c:v", "h264_nvenc",
"-preset", "p4",
"-cq", "23",
"-b:v", "0",
"-fps_mode", "vfr"
],
RecoveryVideoEncoderKind.Qsv =>
[
"-vf", "settb=AVTB,setpts=PTS-STARTPTS,format=nv12,hwupload=extra_hw_frames=64",
"-c:v", "h264_qsv",
"-preset", "veryfast",
"-global_quality", "23",
"-fps_mode", "vfr"
],
RecoveryVideoEncoderKind.Vaapi =>
[
"-vf", "settb=AVTB,setpts=PTS-STARTPTS,format=nv12,hwupload",
"-c:v", "h264_vaapi",
"-qp", "23",
"-fps_mode", "vfr"
],
_ =>
[
"-vf", "settb=AVTB,setpts=PTS-STARTPTS",
"-c:v", "libx264",
"-preset", "veryfast",
"-crf", "23",
"-fps_mode", "vfr"
]
};
private static string? ResolveTransportStreamBitstreamFilter(
RecordingTemplateType recordingTemplate,
@@ -821,6 +1031,51 @@ public sealed partial class FfmpegService
return false;
}
private async Task<MediaArtifactValidation> ValidateMediaArtifactAsync(
string? outputPath,
RecordOutputFormat outputFormat,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(outputPath) || !File.Exists(outputPath))
{
return MediaArtifactValidation.Invalid("The recorded media file does not exist.");
}
var fileInfo = new FileInfo(outputPath);
if (fileInfo.Length <= 0)
{
return MediaArtifactValidation.Invalid("The recorded media file is empty.");
}
if (outputFormat == RecordOutputFormat.Mp4 &&
!outputPath.EndsWith(".mp4", StringComparison.OrdinalIgnoreCase))
{
return MediaArtifactValidation.Invalid(
"MP4 finalization did not produce an MP4 file. The intermediate recording was kept locally.");
}
using var scope = _serviceScopeFactory.CreateScope();
var metadataService = scope.ServiceProvider.GetRequiredService<IVideoMetadataService>();
var metadata = await metadataService.ExtractMetadataAsync(outputPath, cancellationToken);
if (metadata is null)
{
return MediaArtifactValidation.Invalid(FfprobeUnreadableArtifactError);
}
if (string.IsNullOrWhiteSpace(metadata.VideoCodec))
{
return MediaArtifactValidation.Invalid("The recorded media file does not contain a video stream.", metadata.DurationSeconds);
}
if (!metadata.DurationSeconds.HasValue ||
metadata.DurationSeconds.Value < MinimumUnexpectedExitArtifactDuration.TotalSeconds)
{
return MediaArtifactValidation.Invalid(ShortUnexpectedExitArtifactError, metadata.DurationSeconds);
}
return new MediaArtifactValidation(true, metadata.DurationSeconds, null);
}
private static async Task UpsertRecordResultAsync(
RecordTask recordTask,
LiveRecorderDbContext dbContext,
@@ -830,6 +1085,7 @@ public sealed partial class FfmpegService
string? danmakuFilePath,
int danmakuMessageCount,
DateTimeOffset endedAt,
bool mediaValidatedForDispatch = false,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(effectiveOutputPath))
@@ -838,8 +1094,13 @@ public sealed partial class FfmpegService
}
var resultId = Guid.NewGuid();
var dispatchId = Guid.NewGuid();
var normalizedDanmakuCount = Math.Max(0, danmakuMessageCount);
var finalStatus = (int)recordTask.Status;
var shouldCreateCompletionDispatch = IsCompletionDispatchEligible(
recordTask,
effectiveOutputPath,
fileSize) && mediaValidatedForDispatch;
// Multiple background paths can reconcile the same segment after ffmpeg exits.
// Use the database's atomic upsert instead of EF Add-or-Update to avoid
@@ -859,9 +1120,32 @@ public sealed partial class FfmpegService
"ErrorMessage" = excluded."ErrorMessage",
"UploadStatus" = COALESCE("RecordResults"."UploadStatus", excluded."UploadStatus"),
"DeletedLocalFilesAfterUpload" = COALESCE("RecordResults"."DeletedLocalFilesAfterUpload", excluded."DeletedLocalFilesAfterUpload");
INSERT INTO "RecordCompletionDispatches"
("Id", "RecordTaskId", "ScriptDispatched", "UploadDispatched", "AttemptCount", "LastError", "CreatedAt", "UpdatedAt", "NextAttemptAt", "CompletedAt")
SELECT
{dispatchId}, {recordTask.Id}, FALSE, FALSE, 0, NULL, {endedAt}, {endedAt}, {endedAt}, NULL
WHERE {shouldCreateCompletionDispatch}
ON CONFLICT("RecordTaskId") DO NOTHING;
""", cancellationToken);
}
internal static bool IsCompletionDispatchEligible(
RecordTask recordTask,
string? effectiveOutputPath,
long? fileSize)
{
if (recordTask.Status is not (RecordTaskStatus.Completed or RecordTaskStatus.Stopped) ||
string.IsNullOrWhiteSpace(effectiveOutputPath) ||
(recordTask.OutputFormat == RecordOutputFormat.Mp4 &&
!effectiveOutputPath.EndsWith(".mp4", StringComparison.OrdinalIgnoreCase)))
{
return false;
}
return HasUsableOutput(effectiveOutputPath, fileSize);
}
private static Task<RecordResult?> LoadRecordResultAsync(
LiveRecorderDbContext dbContext,
Guid recordTaskId,
@@ -931,6 +1215,43 @@ public sealed partial class FfmpegService
streamUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ||
streamUrl.StartsWith("https://", StringComparison.OrdinalIgnoreCase);
internal static bool ShouldUseCurlPipe(string streamUrl, string? selectedProtocol) =>
IsHttpInput(streamUrl) && !IsHlsInput(streamUrl, selectedProtocol);
private static bool IsHlsInput(string streamUrl, string? selectedProtocol) =>
string.Equals(selectedProtocol, "hls", StringComparison.OrdinalIgnoreCase) ||
streamUrl.Contains(".m3u8", StringComparison.OrdinalIgnoreCase);
private static void AddNativeHttpInputHeaders(
ICollection<string> arguments,
string streamUrl,
StreamInputHeaders? inputHeaders)
{
if (!IsHttpInput(streamUrl) || inputHeaders is null)
{
return;
}
if (!string.IsNullOrWhiteSpace(inputHeaders.UserAgent))
{
arguments.Add("-user_agent");
arguments.Add(inputHeaders.UserAgent.Trim());
}
if (!string.IsNullOrWhiteSpace(inputHeaders.Referer))
{
arguments.Add("-referer");
arguments.Add(inputHeaders.Referer.Trim());
}
var customHeaders = BuildCustomHeaderArgument(inputHeaders);
if (!string.IsNullOrWhiteSpace(customHeaders))
{
arguments.Add("-headers");
arguments.Add(customHeaders);
}
}
private static bool ShouldEnableReconnect(string streamUrl, string? selectedProtocol)
{
if (!string.IsNullOrWhiteSpace(selectedProtocol) &&
@@ -1134,6 +1455,9 @@ public sealed partial class FfmpegService
private static bool IsActiveSessionStatus(RecordSessionStatus status) =>
status is RecordSessionStatus.Starting or RecordSessionStatus.Running or RecordSessionStatus.Stopping;
private static bool IsTerminalSessionStatus(RecordSessionStatus status) =>
status is RecordSessionStatus.Completed or RecordSessionStatus.Failed or RecordSessionStatus.Stopped;
private static bool TryParseFfmpegProgressSeconds(string line, out double seconds)
{
if (line.StartsWith("out_time=", StringComparison.OrdinalIgnoreCase))
@@ -1163,16 +1487,25 @@ public sealed partial class FfmpegService
return false;
}
private enum FfmpegInputOptionProfile
internal enum FfmpegInputOptionProfile
{
Baseline = 0,
Minimal = 1
Minimal = 1,
TimestampRepair = 2,
TimestampTranscode = 3
}
private sealed record MediaArtifactValidation(bool IsValid, double? DurationSeconds, string? ErrorMessage)
{
public static MediaArtifactValidation Invalid(string errorMessage, double? durationSeconds = null) =>
new(false, durationSeconds, errorMessage);
}
private enum StartupFailureKind
{
None = 0,
InputOptionCompatibility = 1,
StreamHandshake = 2
StreamHandshake = 2,
HardwareEncoderUnavailable = 3
}
}
@@ -5,6 +5,7 @@ using LiveRecorder.Application.Abstractions.Logging;
using LiveRecorder.Application.Abstractions.Notifications;
using LiveRecorder.Application.Abstractions.Platforms;
using LiveRecorder.Application.Abstractions.Recording;
using LiveRecorder.Application.Abstractions.Scripting;
using LiveRecorder.Application.Abstractions.Settings;
using LiveRecorder.Application.Abstractions.Storage;
using LiveRecorder.Application.Models.Media;
@@ -27,14 +28,23 @@ public sealed partial class FfmpegService : IFfmpegService
private static readonly TimeSpan StartupFailureNotificationCooldown = TimeSpan.FromMinutes(30);
private static readonly TimeSpan StartupFailureMaxBackoff = TimeSpan.FromMinutes(15);
private static readonly TimeSpan StartupFailureBaseBackoff = TimeSpan.FromSeconds(30);
private static readonly TimeSpan RuntimeFailureMaxBackoff = TimeSpan.FromMinutes(5);
private static readonly TimeSpan RuntimeFailureBaseBackoff = TimeSpan.FromSeconds(15);
private readonly ConcurrentDictionary<Guid, SessionProcessRuntime> _processes = new();
private readonly ConcurrentDictionary<Guid, PostProcessRuntimeEntry> _postProcessStates = new();
private readonly ConcurrentDictionary<Guid, Process> _postProcessProcesses = new();
private readonly ConcurrentDictionary<Guid, RoomStartupFailureState> _roomStartupFailureStates = new();
private readonly ConcurrentDictionary<Guid, RoomStartupFailureState> _roomRuntimeFailureStates = new();
private readonly SemaphoreSlim _recoveryEncoderProbeGate = new(1, 1);
private readonly object _transcodeConcurrencyLock = new();
private readonly SemaphoreSlim _orphanRecoveryGate = new(1, 1);
private readonly Queue<TaskCompletionSource<IDisposable>> _transcodeWaiters = new();
private readonly CancellationTokenSource _shutdownCts = new();
private int _activeTranscodeTasks;
private int _maxConcurrentTranscodeTasks = 1;
private bool _hasProbedRecoveryVideoEncoder;
private RecoveryVideoEncoderSelection _cachedRecoveryVideoEncoder = RecoveryVideoEncoderSelection.Software;
private readonly IServiceScopeFactory _serviceScopeFactory;
private readonly IStorageGuardService _storageGuardService;
private readonly ILiveRoomPollingSignal _liveRoomPollingSignal;
@@ -87,10 +97,7 @@ public sealed partial class FfmpegService : IFfmpegService
initialTask,
streamUrlResult,
recordingSettings,
inputOptionProfile: FfmpegInputOptionProfile.Baseline,
hasRetriedWithCompatibilityProfile: false,
hasRetriedWithRefreshedStream: false,
retryAttemptCount: 0,
InitialRecoveryContext,
cancellationToken);
private async Task StartInternalAsync(
@@ -98,10 +105,7 @@ public sealed partial class FfmpegService : IFfmpegService
RecordTask initialTask,
StreamUrlResult streamUrlResult,
RecordingExecutionSettings recordingSettings,
FfmpegInputOptionProfile inputOptionProfile,
bool hasRetriedWithCompatibilityProfile,
bool hasRetriedWithRefreshedStream,
int retryAttemptCount,
FfmpegRecoveryContext recoveryContext,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(recordSession);
@@ -117,6 +121,9 @@ public sealed partial class FfmpegService : IFfmpegService
using var settingsScope = _serviceScopeFactory.CreateScope();
var settingsService = settingsScope.ServiceProvider.GetRequiredService<ISystemSettingsService>();
var settings = await settingsService.GetAsync(cancellationToken);
var recoveryVideoEncoder = recoveryContext.InputOptionProfile == FfmpegInputOptionProfile.TimestampTranscode
? await ResolveRecoveryVideoEncoderAsync(settings.FfmpegPath, recoveryContext.ForceSoftwareEncoder, cancellationToken)
: RecoveryVideoEncoderSelection.Software;
var outputPathPattern = Path.IsPathRooted(recordSession.OutputPathPattern)
? recordSession.OutputPathPattern
@@ -154,10 +161,8 @@ public sealed partial class FfmpegService : IFfmpegService
streamUrlResult.SelectedVideoCodec,
streamUrlResult.InputHeaders,
recordingSettings,
inputOptionProfile,
hasRetriedWithCompatibilityProfile,
hasRetriedWithRefreshedStream,
retryAttemptCount);
recoveryContext,
recoveryVideoEncoder);
foreach (var argument in BuildArgumentList(
streamUrlResult.SelectedUrl,
@@ -173,7 +178,8 @@ public sealed partial class FfmpegService : IFfmpegService
streamUrlResult.InputHeaders,
streamUrlResult.SelectedProtocol,
streamUrlResult.SelectedVideoCodec,
inputOptionProfile))
recoveryContext.InputOptionProfile,
recoveryVideoEncoder))
{
process.StartInfo.ArgumentList.Add(argument);
}
@@ -199,7 +205,7 @@ public sealed partial class FfmpegService : IFfmpegService
// pipe its stdout into FFmpeg's stdin. This bypasses FFmpeg's built-in HTTP
// handler which has a hard-coded 4096-byte response header buffer that triggers
// "overlong headers" errors with CDNs that return oversized headers.
if (IsHttpInput(streamUrlResult.SelectedUrl))
if (ShouldUseCurlPipe(streamUrlResult.SelectedUrl, streamUrlResult.SelectedProtocol))
{
var curlProcess = new Process
{
@@ -295,6 +301,21 @@ public sealed partial class FfmpegService : IFfmpegService
}
var process = runtime.Process;
var curlProcess = runtime.CurlProcess;
if (curlProcess is not null)
{
try
{
if (!curlProcess.HasExited)
{
curlProcess.Kill(true);
}
}
catch (InvalidOperationException)
{
}
}
if (process is not null)
{
try
@@ -313,7 +334,178 @@ public sealed partial class FfmpegService : IFfmpegService
return await WaitForExitAsync(runtime, timeout, cancellationToken);
}
public async Task<bool> TryReconcileInactiveSessionAsync(Guid recordSessionId, CancellationToken cancellationToken = default)
public async Task<int> StopAllAndWaitAsync(
TimeSpan gracefulTimeout,
TimeSpan forceKillTimeout,
CancellationToken cancellationToken = default)
{
var runtimes = _processes.Values.ToArray();
foreach (var runtime in runtimes)
{
// Mark the captured runtime before looking it up again. The process may exit
// between the snapshot and the stop signal, but its exit handler must still
// observe that this was an application shutdown rather than a stream failure.
runtime.MarkShutdownRequested();
}
await Task.WhenAll(runtimes.Select(runtime =>
RequestStopAsync(
runtime.RecordSessionId,
markAsCompletedOnExit: true,
cancellationToken,
shutdownRequested: true)));
await WaitForRuntimeCompletionsAsync(runtimes, gracefulTimeout, cancellationToken);
foreach (var runtime in runtimes.Where(static runtime => !runtime.ExitCompletion.Task.IsCompleted))
{
var process = runtime.Process;
try
{
if (process is not null && !process.HasExited)
{
runtime.MarkForceKilled();
process.Kill(true);
}
}
catch (InvalidOperationException)
{
}
}
await WaitForRuntimeCompletionsAsync(runtimes, forceKillTimeout, CancellationToken.None);
_shutdownCts.Cancel();
foreach (var process in _postProcessProcesses.Values.ToArray())
{
try
{
if (!process.HasExited)
{
process.Kill(true);
}
}
catch (InvalidOperationException)
{
}
}
await WaitForAllProcessesAsync(forceKillTimeout, CancellationToken.None);
return runtimes.Length;
}
private static async Task WaitForRuntimeCompletionsAsync(
IReadOnlyCollection<SessionProcessRuntime> runtimes,
TimeSpan timeout,
CancellationToken cancellationToken)
{
if (runtimes.Count == 0 || runtimes.All(static runtime => runtime.ExitCompletion.Task.IsCompleted))
{
return;
}
var completionTask = Task.WhenAll(runtimes.Select(static runtime => runtime.ExitCompletion.Task));
using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
var delayTask = Task.Delay(timeout, timeoutCts.Token);
if (await Task.WhenAny(completionTask, delayTask) == completionTask)
{
timeoutCts.Cancel();
await completionTask;
}
}
private async Task WaitForAllProcessesAsync(TimeSpan timeout, CancellationToken cancellationToken)
{
var deadline = DateTimeOffset.UtcNow + timeout;
while ((!_processes.IsEmpty || !_postProcessProcesses.IsEmpty || !_postProcessStates.IsEmpty) &&
DateTimeOffset.UtcNow < deadline)
{
await Task.Delay(TimeSpan.FromMilliseconds(200), cancellationToken);
}
}
public Task<bool> TryReconcileInactiveSessionAsync(Guid recordSessionId, CancellationToken cancellationToken = default) =>
ReconcileInactiveSessionAsync(recordSessionId, allowTerminalSession: false, cancellationToken);
public async Task<int> RecoverOrphanedTerminalSessionTasksAsync(CancellationToken cancellationToken = default)
{
if (!await _orphanRecoveryGate.WaitAsync(0, cancellationToken))
{
return 0;
}
try
{
using var scope = _serviceScopeFactory.CreateScope();
var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
var candidateIds = await dbContext.RecordSessions
.AsNoTracking()
.Where(session =>
(session.Status == RecordSessionStatus.Completed ||
session.Status == RecordSessionStatus.Failed ||
session.Status == RecordSessionStatus.Stopped) &&
session.RecordTasks.Any(task =>
task.Status == RecordTaskStatus.Starting ||
task.Status == RecordTaskStatus.Running ||
task.Status == RecordTaskStatus.Stopping))
.OrderBy(session => session.UpdatedAt)
.Select(session => session.Id)
.Take(20)
.ToListAsync(cancellationToken);
var recovered = 0;
foreach (var candidateId in candidateIds)
{
if (await ReconcileInactiveSessionAsync(candidateId, allowTerminalSession: true, cancellationToken))
{
recovered++;
}
}
return recovered;
}
finally
{
_orphanRecoveryGate.Release();
}
}
public async Task<bool> TryRecoverOrphanedTerminalTaskAsync(
Guid recordTaskId,
CancellationToken cancellationToken = default)
{
await _orphanRecoveryGate.WaitAsync(cancellationToken);
try
{
using var lookupScope = _serviceScopeFactory.CreateScope();
var lookupDbContext = lookupScope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
var sessionId = await lookupDbContext.RecordTasks
.AsNoTracking()
.Where(task =>
task.Id == recordTaskId &&
(task.Status == RecordTaskStatus.Starting ||
task.Status == RecordTaskStatus.Running ||
task.Status == RecordTaskStatus.Stopping) &&
task.RecordSession != null &&
(task.RecordSession.Status == RecordSessionStatus.Completed ||
task.RecordSession.Status == RecordSessionStatus.Failed ||
task.RecordSession.Status == RecordSessionStatus.Stopped))
.Select(task => (Guid?)task.RecordSessionId)
.FirstOrDefaultAsync(cancellationToken);
return sessionId.HasValue &&
await ReconcileInactiveSessionAsync(sessionId.Value, allowTerminalSession: true, cancellationToken);
}
finally
{
_orphanRecoveryGate.Release();
}
}
private async Task<bool> ReconcileInactiveSessionAsync(
Guid recordSessionId,
bool allowTerminalSession,
CancellationToken cancellationToken)
{
if (IsRunning(recordSessionId))
{
@@ -330,11 +522,53 @@ public sealed partial class FfmpegService : IFfmpegService
.ThenInclude(item => item.Result)
.FirstOrDefaultAsync(item => item.Id == recordSessionId, cancellationToken);
if (recordSession is null || !IsActiveSessionStatus(recordSession.Status))
if (recordSession is null ||
(!IsActiveSessionStatus(recordSession.Status) &&
!(allowTerminalSession && IsTerminalSessionStatus(recordSession.Status))))
{
return false;
}
var hasActiveTasks = recordSession.RecordTasks.Any(task => IsActiveTaskStatus(task.Status));
if (allowTerminalSession && !hasActiveTasks)
{
return false;
}
var discoveredSegments = DiscoverRecoverableSegments(recordSession.OutputPathPattern, recordSession.OutputFormat, recordSession.SaveMode);
var existingSegmentIndexes = recordSession.RecordTasks
.Select(static task => task.SegmentIndex)
.ToHashSet();
var addedMissingTasks = false;
foreach (var segment in discoveredSegments.Where(segment => !existingSegmentIndexes.Contains(segment.SegmentIndex)))
{
var discoveredAt = File.GetLastWriteTimeUtc(segment.RecorderPath);
var createdAt = discoveredAt == DateTime.MinValue
? DateTimeOffset.UtcNow
: new DateTimeOffset(DateTime.SpecifyKind(discoveredAt, DateTimeKind.Utc));
var missingTask = new RecordTask(
recordSession.LiveRoomId,
recordSession.Id,
segment.SegmentIndex,
recordSession.PreferredQuality,
recordSession.OutputFormat,
createdAt);
var recoveryStartedAt = DateTimeOffset.UtcNow;
missingTask.MarkStarting(recordSession.StreamUrl ?? string.Empty, segment.OutputPath, recoveryStartedAt);
missingTask.MarkRunning(recoveryStartedAt);
await dbContext.RecordTasks.AddAsync(missingTask, cancellationToken);
recordSession.RecordTasks.Add(missingTask);
existingSegmentIndexes.Add(segment.SegmentIndex);
addedMissingTasks = true;
}
// RecordResult is upserted with raw SQL below, so newly discovered tasks must
// exist first to satisfy the RecordTaskId foreign key.
if (addedMissingTasks)
{
await dbContext.SaveChangesAsync(cancellationToken);
}
var settings = await settingsService.GetAsync(cancellationToken);
var endedAt = DateTimeOffset.UtcNow;
var tasks = recordSession.RecordTasks
@@ -345,6 +579,7 @@ public sealed partial class FfmpegService : IFfmpegService
var anyUsableOutput = tasks.Any(static item => item.Status == RecordTaskStatus.Completed);
var hasBackgroundPostProcessing = false;
string? sessionFinalizationError = null;
var newlyCompletedTasks = new List<(RecordTask Task, string OutputPath)>();
foreach (var task in tasks)
{
@@ -372,11 +607,21 @@ public sealed partial class FfmpegService : IFfmpegService
cancellationToken);
var effectiveOutputPath = finalizationResult.OutputPath;
var taskFinalizationError = finalizationResult.ErrorMessage;
var mediaValidation = await ValidateMediaArtifactAsync(
effectiveOutputPath,
recordSession.OutputFormat,
cancellationToken);
durationSeconds = mediaValidation.DurationSeconds;
if (!IsLowStoragePauseError(taskFinalizationError))
{
sessionFinalizationError ??= taskFinalizationError;
}
if (string.IsNullOrWhiteSpace(taskFinalizationError) && !mediaValidation.IsValid)
{
sessionFinalizationError ??= mediaValidation.ErrorMessage;
}
var fileSize = CalculateFileSize(effectiveOutputPath);
var danmakuPath = task.Result?.DanmakuFilePath ?? GuessDanmakuPath(task.OutputFilePath);
var danmakuCount = CountDanmakuMessages(danmakuPath);
@@ -388,23 +633,34 @@ public sealed partial class FfmpegService : IFfmpegService
}
else if (!string.IsNullOrWhiteSpace(taskFinalizationError))
{
task.MarkFailed(taskFinalizationError, endedAt);
task.MarkFailed(taskFinalizationError, endedAt, durationSeconds);
}
else if (HasUsableOutput(effectiveOutputPath, fileSize))
else if (mediaValidation.IsValid)
{
task.MarkCompleted(endedAt, durationSeconds);
anyUsableOutput = true;
newlyCompletedTasks.Add((task, effectiveOutputPath));
}
else
{
task.MarkStopped(endedAt, durationSeconds, "Recording process was no longer running when the session was reconciled.");
task.MarkFailed(mediaValidation.ErrorMessage!, endedAt, durationSeconds);
}
await UpsertRecordResultAsync(task, dbContext, effectiveOutputPath, fileSize, durationSeconds, danmakuPath, danmakuCount, endedAt, cancellationToken);
await UpsertRecordResultAsync(
task,
dbContext,
effectiveOutputPath,
fileSize,
durationSeconds,
danmakuPath,
danmakuCount,
endedAt,
mediaValidatedForDispatch: mediaValidation.IsValid,
cancellationToken: cancellationToken);
ClearPostProcessState(task.Id);
}
recordSession.SyncSegmentCount(tasks.Count, endedAt);
recordSession.SyncSegmentCount(tasks.Count == 0 ? 0 : tasks.Max(static task => task.SegmentIndex), endedAt);
if (!string.IsNullOrWhiteSpace(sessionFinalizationError))
{
recordSession.MarkFailed(sessionFinalizationError, endedAt);
@@ -425,6 +681,26 @@ public sealed partial class FfmpegService : IFfmpegService
}
await dbContext.SaveChangesAsync(cancellationToken);
if (newlyCompletedTasks.Count > 0 && recordSession.LiveRoom is not null)
{
var completionDispatchService = scope.ServiceProvider.GetRequiredService<CompletionDispatchService>();
foreach (var completed in newlyCompletedTasks)
{
try
{
await completionDispatchService.TryDispatchTaskAsync(completed.Task.Id, cancellationToken);
}
catch (Exception ex)
{
_logger.LogWarning(
ex,
"Recovered segment completion hooks failed for task {RecordTaskId}",
completed.Task.Id);
}
}
}
return true;
}
@@ -459,8 +735,8 @@ public sealed partial class FfmpegService : IFfmpegService
return false;
}
var recorderOutputPath = ResolveManualFinalizeSourcePath(recordTask, recordTask.RecordSession);
if (!File.Exists(recorderOutputPath))
var recorderOutputPaths = ResolveManualFinalizeSourcePaths(recordTask, recordTask.RecordSession);
if (recorderOutputPaths.Count == 0)
{
return false;
}
@@ -548,9 +824,10 @@ public sealed partial class FfmpegService : IFfmpegService
settings.Mp4FinalizeTimeoutMinutes,
syntheticSessionId,
syntheticTaskId,
absoluteSourcePath,
[absoluteSourcePath],
absoluteTargetPath,
expectedDurationSeconds: null,
segmentsManifestPath: null,
CancellationToken.None);
using var scope = _serviceScopeFactory.CreateScope();
@@ -610,32 +887,30 @@ public sealed partial class FfmpegService : IFfmpegService
var settings = await settingsService.GetAsync(cancellationToken);
var storageCheck = _storageGuardService.CheckCanStartOrResume(settings);
if (storageCheck.ShouldPauseActive)
{
await logService.WriteAsync(
SystemLogLevel.Warning,
"Storage",
"MP4 finalization remains paused because storage tier is Red.",
storageCheck.Message,
cancellationToken: cancellationToken);
return 0;
}
var candidates = await dbContext.RecordTasks
.Include(item => item.RecordSession)
.ThenInclude(item => item!.RecordTasks)
.Include(item => item.Result)
.Where(item => item.OutputFormat == RecordOutputFormat.Mp4 &&
(item.Status == RecordTaskStatus.Processing ||
item.Status == RecordTaskStatus.Completed))
item.Status == RecordTaskStatus.Completed ||
item.Status == RecordTaskStatus.Failed &&
item.ErrorMessage == FfprobeUnreadableArtifactError))
// Deployment-paused work must not be starved by a large backlog of
// older legacy failures that may no longer have local source files.
.OrderBy(static item => item.Status == RecordTaskStatus.Processing
? 0
: item.Status == RecordTaskStatus.Completed
? 1
: 2)
.ThenBy(static item => item.UpdatedAt)
.Take(100)
.ToListAsync(cancellationToken);
candidates = candidates
.OrderBy(static item => item.UpdatedAt)
.Take(100)
.ToList();
var queuedTaskIds = new List<Guid>();
var recoveredValidatedTaskIds = new List<Guid>();
var repairedInterruptedTasks = 0;
var repairedLegacyShutdownFailures = 0;
var now = DateTimeOffset.UtcNow;
foreach (var candidate in candidates)
{
@@ -645,6 +920,58 @@ public sealed partial class FfmpegService : IFfmpegService
continue;
}
if (candidate.Status == RecordTaskStatus.Failed)
{
var recoverySources = candidate.RecordSession is null
? Array.Empty<string>()
: ResolveManualFinalizeSourcePaths(candidate, candidate.RecordSession);
if (recoverySources.Count > 0)
{
candidate.MarkProcessing(
"A deployment-interrupted MP4 finalization was recovered and queued after restart.",
now);
candidate.RecordSession!.MarkStopped(
now,
"A deployment-interrupted MP4 finalization is continuing in the background.");
queuedTaskIds.Add(candidate.Id);
repairedLegacyShutdownFailures++;
continue;
}
var existingOutputPath = candidate.Result?.FilePath ?? candidate.OutputFilePath;
var mediaValidation = await ValidateMediaArtifactAsync(
existingOutputPath,
candidate.OutputFormat,
cancellationToken);
if (!mediaValidation.IsValid)
{
continue;
}
candidate.MarkCompleted(now, mediaValidation.DurationSeconds);
if (candidate.RecordSession is not null &&
candidate.RecordSession.RecordTasks.All(static task =>
task.Status is RecordTaskStatus.Completed or RecordTaskStatus.Stopped))
{
candidate.RecordSession.MarkCompleted(now);
}
await UpsertRecordResultAsync(
candidate,
dbContext,
existingOutputPath,
CalculateFileSize(existingOutputPath),
mediaValidation.DurationSeconds,
candidate.Result?.DanmakuFilePath,
candidate.Result?.DanmakuMessageCount ?? 0,
now,
mediaValidatedForDispatch: true,
cancellationToken: cancellationToken);
recoveredValidatedTaskIds.Add(candidate.Id);
repairedLegacyShutdownFailures++;
continue;
}
if (!NeedsInterruptedMp4Finalization(candidate))
{
continue;
@@ -655,13 +982,24 @@ public sealed partial class FfmpegService : IFfmpegService
repairedInterruptedTasks++;
}
if (repairedInterruptedTasks > 0)
if (repairedInterruptedTasks > 0 || repairedLegacyShutdownFailures > 0)
{
await dbContext.SaveChangesAsync(cancellationToken);
}
if (recoveredValidatedTaskIds.Count > 0)
{
var completionDispatchService = scope.ServiceProvider.GetRequiredService<CompletionDispatchService>();
foreach (var taskId in recoveredValidatedTaskIds)
{
await completionDispatchService.TryDispatchTaskAsync(taskId, cancellationToken);
}
}
var started = 0;
foreach (var taskId in queuedTaskIds.Take(20))
foreach (var taskId in queuedTaskIds
.Where(taskId => !IsTaskUnderPostProcessing(taskId))
.Take(20))
{
if (await StartManualFinalizeTaskAsync(taskId, cancellationToken))
{
@@ -669,13 +1007,13 @@ public sealed partial class FfmpegService : IFfmpegService
}
}
if (started > 0)
if (started > 0 || repairedLegacyShutdownFailures > 0)
{
await logService.WriteAsync(
SystemLogLevel.Info,
"Storage",
"Storage is available. Resumed paused MP4 finalization tasks.",
$"count={started}; repairedInterrupted={repairedInterruptedTasks}; {storageCheck.Message}",
"Resumed paused MP4 finalizations and repaired deployment-interrupted recording results.",
$"count={started}; repairedInterrupted={repairedInterruptedTasks}; repairedLegacyShutdown={repairedLegacyShutdownFailures}; {storageCheck.Message}",
cancellationToken: cancellationToken);
}
@@ -699,12 +1037,23 @@ public sealed partial class FfmpegService : IFfmpegService
return false;
}
var recorderOutputPath = ResolveManualFinalizeSourcePath(recordTask, recordTask.RecordSession);
return File.Exists(recorderOutputPath);
return ResolveManualFinalizeSourcePaths(recordTask, recordTask.RecordSession).Count > 0;
}
private static string ResolveManualFinalizeSourcePath(RecordTask recordTask, RecordSession recordSession)
=> ResolveManualFinalizeSourcePaths(recordTask, recordSession).FirstOrDefault() ?? string.Empty;
private static IReadOnlyList<string> ResolveManualFinalizeSourcePaths(RecordTask recordTask, RecordSession recordSession)
{
if (!string.IsNullOrWhiteSpace(recordTask.OutputFilePath))
{
var manifestPaths = ReadRecorderSegmentsManifest(recordTask.OutputFilePath);
if (manifestPaths.Count > 0)
{
return manifestPaths;
}
}
var resultPath = recordTask.Result?.FilePath;
if (!string.IsNullOrWhiteSpace(resultPath) &&
resultPath.EndsWith(".ts", StringComparison.OrdinalIgnoreCase))
@@ -712,15 +1061,16 @@ public sealed partial class FfmpegService : IFfmpegService
var normalizedResultPath = NormalizeAbsolutePath(resultPath);
if (File.Exists(normalizedResultPath))
{
return normalizedResultPath;
return [normalizedResultPath];
}
}
return NormalizeAbsolutePath(
var defaultPath = NormalizeAbsolutePath(
GetRecorderOutputPath(
recordTask.OutputFilePath ?? resultPath ?? string.Empty,
recordSession.OutputFormat,
recordSession.SaveMode));
return File.Exists(defaultPath) ? [defaultPath] : [];
}
private async Task PersistStartupProfileAsync(SessionProcessRuntime runtime, CancellationToken cancellationToken)
@@ -731,7 +1081,7 @@ public sealed partial class FfmpegService : IFfmpegService
SystemLogLevel.Info,
"FFmpeg",
$"ffmpeg input profile={runtime.InputOptionProfile}.",
detail: $"stream={runtime.SelectedProtocol}:{runtime.SelectedQuality}; videoCodec={runtime.SelectedVideoCodec ?? "unknown"}; compatibilityRetry={runtime.HasRetriedWithCompatibilityProfile}; refreshRetry={runtime.HasRetriedWithRefreshedStream}",
detail: $"stream={runtime.SelectedProtocol}:{runtime.SelectedQuality}; videoCodec={runtime.SelectedVideoCodec ?? "unknown"}; recoveryEncoder={runtime.RecoveryVideoEncoder.Kind}; recoveryDevice={runtime.RecoveryVideoEncoder.DevicePath ?? "default"}; attempt={runtime.RetryAttemptCount}/{MaxInSessionRetryAttempts}; compatibilityRetry={runtime.HasRetriedWithCompatibilityProfile}; refreshRetry={runtime.HasRetriedWithRefreshedStream}; alternateProtocolRetry={runtime.HasRetriedWithAlternateProtocol}; softwareEncoderRetry={runtime.RecoveryContext.HasRetriedWithSoftwareEncoder}",
liveRoomId: runtime.LiveRoomId,
recordSessionId: runtime.RecordSessionId,
recordTaskId: runtime.CurrentTaskId,
@@ -756,14 +1106,25 @@ public sealed partial class FfmpegService : IFfmpegService
return false;
}
private async Task RequestStopAsync(Guid recordSessionId, bool markAsCompletedOnExit, CancellationToken cancellationToken)
private async Task RequestStopAsync(
Guid recordSessionId,
bool markAsCompletedOnExit,
CancellationToken cancellationToken,
bool shutdownRequested = false)
{
if (!_processes.TryGetValue(recordSessionId, out var runtime))
{
return;
}
runtime.MarkStopRequested(markAsCompletedOnExit);
if (shutdownRequested)
{
runtime.MarkShutdownRequested();
}
else
{
runtime.MarkStopRequested(markAsCompletedOnExit);
}
var process = runtime.Process;
if (process is null)
{
@@ -919,6 +1280,41 @@ public sealed partial class FfmpegService : IFfmpegService
}
}
private TimeSpan RecordRuntimeFailure(Guid liveRoomId)
{
var now = DateTimeOffset.UtcNow;
var state = _roomRuntimeFailureStates.AddOrUpdate(
liveRoomId,
_ => new RoomStartupFailureState { ConsecutiveFailures = 1, FirstFailureAt = now, LastFailureAt = now },
(_, existing) =>
{
existing.ConsecutiveFailures++;
existing.LastFailureAt = now;
return existing;
});
var backoffSeconds = RuntimeFailureBaseBackoff.TotalSeconds *
Math.Pow(2, Math.Min(state.ConsecutiveFailures - 1, 5));
var backoff = TimeSpan.FromSeconds(Math.Min(backoffSeconds, RuntimeFailureMaxBackoff.TotalSeconds));
_logger.LogWarning(
"Short runtime failure backoff for room {LiveRoomId}: {ConsecutiveFailures} consecutive failures, next poll delayed by {BackoffSeconds:F0}s",
liveRoomId,
state.ConsecutiveFailures,
backoff.TotalSeconds);
return backoff;
}
private void ResetRuntimeFailureBackoff(Guid liveRoomId)
{
if (_roomRuntimeFailureStates.TryRemove(liveRoomId, out var state) && state.ConsecutiveFailures > 1)
{
_logger.LogInformation(
"Short runtime failure backoff reset for room {LiveRoomId} after {ConsecutiveFailures} failures",
liveRoomId,
state.ConsecutiveFailures);
}
}
/// <summary>
/// Returns true if the failure notification for this room should be throttled
/// (i.e., at most one notification per <see cref="StartupFailureNotificationCooldown"/>).
@@ -0,0 +1,33 @@
using LiveRecorder.Application.Abstractions.Recording;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace LiveRecorder.Infrastructure.Services;
public sealed class FfmpegShutdownHostedService : IHostedService
{
private static readonly TimeSpan GracefulTimeout = TimeSpan.FromSeconds(20);
private static readonly TimeSpan ForceKillTimeout = TimeSpan.FromSeconds(10);
private readonly IFfmpegService _ffmpegService;
private readonly ILogger<FfmpegShutdownHostedService> _logger;
public FfmpegShutdownHostedService(
IFfmpegService ffmpegService,
ILogger<FfmpegShutdownHostedService> logger)
{
_ffmpegService = ffmpegService;
_logger = logger;
}
public Task StartAsync(CancellationToken cancellationToken) => Task.CompletedTask;
public async Task StopAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Stopping active recording and post-processing child processes.");
var stopped = await _ffmpegService.StopAllAndWaitAsync(
GracefulTimeout,
ForceKillTimeout,
cancellationToken);
_logger.LogInformation("Recording shutdown coordination completed for {SessionCount} sessions.", stopped);
}
}
@@ -1,12 +1,22 @@
using System.Diagnostics;
using System.Globalization;
using System.Text.Json;
using System.Text.RegularExpressions;
using LiveRecorder.Application.Abstractions.Recording;
using Microsoft.Extensions.Logging;
namespace LiveRecorder.Infrastructure.Services;
public sealed class FfmpegVideoMetadataService : IVideoMetadataService
{
private const string ThumbnailsSubDir = ".thumbnails";
private static readonly TimeSpan ProbeTimeout = TimeSpan.FromSeconds(60);
private readonly ILogger<FfmpegVideoMetadataService> _logger;
public FfmpegVideoMetadataService(ILogger<FfmpegVideoMetadataService> logger)
{
_logger = logger;
}
public async Task<VideoMetadata?> ExtractMetadataAsync(string filePath, CancellationToken cancellationToken = default)
{
@@ -17,12 +27,11 @@ public sealed class FfmpegVideoMetadataService : IVideoMetadataService
try
{
var process = new Process
using var process = new Process
{
StartInfo = new ProcessStartInfo
{
FileName = GetFfprobePath(),
Arguments = $"-v quiet -print_format json -show_format -show_streams \"{filePath}\"",
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
@@ -30,21 +39,55 @@ public sealed class FfmpegVideoMetadataService : IVideoMetadataService
}
};
process.Start();
var output = await process.StandardOutput.ReadToEndAsync(cancellationToken);
await process.WaitForExitAsync(cancellationToken);
if (process.ExitCode != 0 || string.IsNullOrWhiteSpace(output))
foreach (var argument in new[] { "-v", "error", "-print_format", "json", "-show_format", "-show_streams", filePath })
{
return null;
process.StartInfo.ArgumentList.Add(argument);
}
return ParseFfprobeOutput(output);
SanitizeFfprobeProcessEnvironment(process.StartInfo);
process.Start();
using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
timeoutCts.CancelAfter(ProbeTimeout);
var outputTask = process.StandardOutput.ReadToEndAsync(timeoutCts.Token);
var errorTask = process.StandardError.ReadToEndAsync(timeoutCts.Token);
await process.WaitForExitAsync(timeoutCts.Token);
var output = await outputTask;
var error = await errorTask;
if (process.ExitCode == 0 && !string.IsNullOrWhiteSpace(output))
{
var metadata = ParseFfprobeOutput(output);
if (metadata is not null)
{
return metadata;
}
}
else
{
_logger.LogWarning(
"ffprobe failed for {FilePath}: exitCode={ExitCode}; stderr={Error}",
filePath,
process.ExitCode,
string.IsNullOrWhiteSpace(error) ? "(empty)" : error.Trim());
}
}
catch
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
{
_logger.LogWarning("ffprobe timed out after {TimeoutSeconds}s for {FilePath}", ProbeTimeout.TotalSeconds, filePath);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "ffprobe could not be started or read media metadata for {FilePath}", filePath);
}
if (cancellationToken.IsCancellationRequested)
{
return null;
}
_logger.LogWarning("Falling back to ffmpeg header probing for {FilePath}", filePath);
return await ExtractMetadataWithFfmpegAsync(filePath, cancellationToken);
}
public async Task<string?> GenerateThumbnailAsync(string filePath, string outputDir, CancellationToken cancellationToken = default)
@@ -93,7 +136,10 @@ public sealed class FfmpegVideoMetadataService : IVideoMetadataService
};
process.Start();
var outputTask = process.StandardOutput.ReadToEndAsync(cancellationToken);
var errorTask = process.StandardError.ReadToEndAsync(cancellationToken);
await process.WaitForExitAsync(cancellationToken);
await Task.WhenAll(outputTask, errorTask);
if (process.ExitCode == 0 && File.Exists(thumbPath) && new FileInfo(thumbPath).Length > 0)
{
@@ -165,6 +211,155 @@ public sealed class FfmpegVideoMetadataService : IVideoMetadataService
}
}
private async Task<VideoMetadata?> ExtractMetadataWithFfmpegAsync(
string filePath,
CancellationToken cancellationToken)
{
try
{
using var process = new Process
{
StartInfo = new ProcessStartInfo
{
FileName = GetFfmpegPath(),
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
}
};
foreach (var argument in new[] { "-hide_banner", "-i", filePath, "-t", "0", "-f", "null", "-" })
{
process.StartInfo.ArgumentList.Add(argument);
}
// The recorder already proves that fnOS can launch this ffmpeg with
// the inherited application environment. Do not apply the ffprobe-
// specific library cleanup to this compatibility fallback.
process.Start();
using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
timeoutCts.CancelAfter(ProbeTimeout);
var outputTask = process.StandardOutput.ReadToEndAsync(timeoutCts.Token);
var errorTask = process.StandardError.ReadToEndAsync(timeoutCts.Token);
await process.WaitForExitAsync(timeoutCts.Token);
await outputTask;
var error = await errorTask;
var metadata = ParseFfmpegHeaderOutput(error);
if (process.ExitCode == 0 && metadata is not null)
{
_logger.LogInformation("ffmpeg header probing succeeded for {FilePath}", filePath);
return metadata;
}
_logger.LogWarning(
"ffmpeg header probing failed for {FilePath}: exitCode={ExitCode}; stderr={Error}",
filePath,
process.ExitCode,
string.IsNullOrWhiteSpace(error) ? "(empty)" : error.Trim());
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
{
_logger.LogWarning(
"ffmpeg header probing timed out after {TimeoutSeconds}s for {FilePath}",
ProbeTimeout.TotalSeconds,
filePath);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "ffmpeg header probing could not read media metadata for {FilePath}", filePath);
}
return null;
}
internal static VideoMetadata? ParseFfmpegHeaderOutput(string output)
{
if (string.IsNullOrWhiteSpace(output))
{
return null;
}
var durationMatch = Regex.Match(
output,
@"Duration:\s*(?<hours>\d+):(?<minutes>\d{2}):(?<seconds>\d{2}(?:\.\d+)?)",
RegexOptions.CultureInvariant);
if (!durationMatch.Success ||
!double.TryParse(durationMatch.Groups["hours"].Value, NumberStyles.None, CultureInfo.InvariantCulture, out var hours) ||
!double.TryParse(durationMatch.Groups["minutes"].Value, NumberStyles.None, CultureInfo.InvariantCulture, out var minutes) ||
!double.TryParse(durationMatch.Groups["seconds"].Value, NumberStyles.Float, CultureInfo.InvariantCulture, out var seconds))
{
return null;
}
var durationSeconds = hours * 3600 + minutes * 60 + seconds;
if (durationSeconds <= 0)
{
return null;
}
string? videoLine = null;
string? audioLine = null;
foreach (var line in output.Split('\n'))
{
if (line.Contains("Stream mapping:", StringComparison.Ordinal))
{
break;
}
if (videoLine is null && line.Contains("Video:", StringComparison.Ordinal))
{
videoLine = line;
}
else if (audioLine is null && line.Contains("Audio:", StringComparison.Ordinal))
{
audioLine = line;
}
}
var videoCodec = MatchValue(videoLine, @"Video:\s*(?<value>[^\s,(]+)");
var audioCodec = MatchValue(audioLine, @"Audio:\s*(?<value>[^\s,(]+)");
var dimensions = videoLine is null
? Match.Empty
: Regex.Match(videoLine, @"(?<!\d)(?<width>\d{2,5})x(?<height>\d{2,5})(?!\d)", RegexOptions.CultureInvariant);
var frameRate = MatchDouble(videoLine, @",\s*(?<value>\d+(?:\.\d+)?)\s+fps(?:,|\s)");
var bitRateKbps = MatchDouble(
output,
@"Duration:[^\r\n]*bitrate:\s*(?<value>\d+(?:\.\d+)?)\s*kb/s");
int? width = dimensions.Success && int.TryParse(dimensions.Groups["width"].Value, out var parsedWidth)
? parsedWidth
: null;
int? height = dimensions.Success && int.TryParse(dimensions.Groups["height"].Value, out var parsedHeight)
? parsedHeight
: null;
long? bitRate = bitRateKbps.HasValue
? (long)Math.Round(bitRateKbps.Value * 1000, MidpointRounding.AwayFromZero)
: null;
return new VideoMetadata(durationSeconds, width, height, videoCodec, audioCodec, frameRate, bitRate);
}
private static string? MatchValue(string? input, string pattern)
{
if (string.IsNullOrWhiteSpace(input))
{
return null;
}
var match = Regex.Match(input, pattern, RegexOptions.CultureInvariant);
return match.Success ? match.Groups["value"].Value : null;
}
private static double? MatchDouble(string? input, string pattern)
{
var value = MatchValue(input, pattern);
return double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed)
? parsed
: null;
}
private static double? ParseFrameRate(string fraction)
{
var parts = fraction.Split('/');
@@ -181,4 +376,12 @@ public sealed class FfmpegVideoMetadataService : IVideoMetadataService
private static string GetFfmpegPath() => "ffmpeg";
private static string GetFfprobePath() => "ffprobe";
internal static void SanitizeFfprobeProcessEnvironment(ProcessStartInfo startInfo)
{
// The fnOS package ships private Debian libraries for its bundled curl.
// Inheriting that LD_LIBRARY_PATH into the host's ffmpeg/ffprobe can make
// otherwise valid system binaries fail with incompatible shared libraries.
startInfo.Environment.Remove("LD_LIBRARY_PATH");
}
}
@@ -74,6 +74,13 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
var settingsService = scope.ServiceProvider.GetRequiredService<ISystemSettingsService>();
var settings = await settingsService.GetAsync(stoppingToken);
var ffmpegService = scope.ServiceProvider.GetRequiredService<IFfmpegService>();
var recoveredOrphanedSessions = await ffmpegService.RecoverOrphanedTerminalSessionTasksAsync(stoppingToken);
if (recoveredOrphanedSessions > 0)
{
_logger.LogWarning(
"Recovered {SessionCount} terminal recording sessions that still contained active segment tasks.",
recoveredOrphanedSessions);
}
// Always try to resume paused MP4 finalizations — even (especially) under the Red
// tier. Finalization is what flips a task to Completed, which fires the
// segment_completed script (upload + delete source) that frees disk space. Skipping
@@ -431,28 +438,37 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
return;
}
var hasRunningSession = await dbContext.RecordSessions.AnyAsync(
var activeSessionId = await dbContext.RecordSessions
.Where(
item => item.LiveRoomId == liveRoom.Id &&
(item.Status == RecordSessionStatus.Starting ||
item.Status == RecordSessionStatus.Running ||
item.Status == RecordSessionStatus.Stopping),
cancellationToken);
item.Status == RecordSessionStatus.Stopping))
.OrderByDescending(static item => item.CreatedAt)
.Select(static item => (Guid?)item.Id)
.FirstOrDefaultAsync(cancellationToken);
if (hasRunningSession)
if (activeSessionId.HasValue)
{
await UpdateAutoStartDecisionAsync(
var decisionChanged = await UpdateAutoStartDecisionIfChangedAsync(
dbContext,
liveRoom,
AutoStartDecisionCodes.SkippedActiveSession,
"Auto-start skipped because an active recording session already exists.",
detail: null,
$"activeSessionId={activeSessionId.Value}",
cancellationToken);
await logService.WriteAsync(
SystemLogLevel.Info,
"Scheduler",
"Auto-start skipped because an active recording session already exists.",
liveRoomId: liveRoom.Id,
cancellationToken: cancellationToken);
if (decisionChanged)
{
await logService.WriteAsync(
SystemLogLevel.Info,
"Scheduler",
"Auto-start skipped because an active recording session already exists.",
$"activeSessionId={activeSessionId.Value}",
liveRoomId: liveRoom.Id,
recordSessionId: activeSessionId.Value,
cancellationToken: cancellationToken);
}
return;
}
@@ -860,6 +876,40 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
await SaveChangesWithRetryAsync(dbContext, cancellationToken);
}
private static async Task<bool> UpdateAutoStartDecisionIfChangedAsync(
LiveRecorderDbContext dbContext,
Domain.Entities.LiveRoom liveRoom,
string code,
string summary,
string? detail,
CancellationToken cancellationToken)
{
var normalizedCode = Truncate(code, 64);
var normalizedSummary = Truncate(summary, 256);
var normalizedDetail = Truncate(detail, 2048);
if (!HasAutoStartDecisionChanged(liveRoom, normalizedCode, normalizedSummary, normalizedDetail))
{
return false;
}
liveRoom.SetLastAutoStartDecision(
normalizedCode,
normalizedSummary,
normalizedDetail,
DateTimeOffset.UtcNow);
await SaveChangesWithRetryAsync(dbContext, cancellationToken);
return true;
}
internal static bool HasAutoStartDecisionChanged(
Domain.Entities.LiveRoom liveRoom,
string? code,
string? summary,
string? detail) =>
!string.Equals(liveRoom.LastAutoStartDecisionCode, code, StringComparison.Ordinal) ||
!string.Equals(liveRoom.LastAutoStartDecisionSummary, summary, StringComparison.Ordinal) ||
!string.Equals(liveRoom.LastAutoStartDecisionDetail, detail, StringComparison.Ordinal);
private static async Task SaveChangesWithRetryAsync(LiveRecorderDbContext dbContext, CancellationToken cancellationToken)
{
for (var attempt = 1; attempt <= 5; attempt++)
@@ -38,6 +38,23 @@ public interface IOpenListClient
OpenListConnectionRequest connection,
string taskId,
CancellationToken cancellationToken = default);
Task<bool> TryCancelCopyTaskAsync(
OpenListConnectionRequest connection,
string taskId,
CancellationToken cancellationToken = default);
Task RenameAsync(
OpenListConnectionRequest connection,
string path,
string newName,
CancellationToken cancellationToken = default);
Task MoveAsync(
OpenListConnectionRequest connection,
string sourcePath,
string targetDirectory,
CancellationToken cancellationToken = default);
}
public sealed record OpenListObjectInfo(
@@ -400,6 +417,67 @@ public sealed class OpenListClient : IOpenListClient
return new OpenListTaskInfo(id, state, progress, status, error);
}
public async Task<bool> TryCancelCopyTaskAsync(
OpenListConnectionRequest connection,
string taskId,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(taskId))
{
return true;
}
var encodedTaskId = Uri.EscapeDataString(taskId.Trim());
var envelope = await SendAuthorizedAsync(
connection,
() => new HttpRequestMessage(HttpMethod.Post, $"/api/task/copy/cancel?tid={encodedTaskId}"),
cancellationToken);
return envelope.Code == 200 || envelope.Code == 404 || ContainsAny(envelope.Message, "not found", "finished");
}
public async Task RenameAsync(
OpenListConnectionRequest connection,
string path,
string newName,
CancellationToken cancellationToken = default)
{
path = NormalizePath(path);
if (string.IsNullOrWhiteSpace(newName) || newName.Contains('/') || newName.Contains('\\'))
{
throw new InvalidOperationException("OpenList 新文件名无效。");
}
var envelope = await SendAuthorizedAsync(
connection,
() => CreateJsonRequest(HttpMethod.Post, "/api/fs/rename", new { path, name = newName.Trim() }),
cancellationToken);
EnsureSuccess(envelope, $"OpenList rename '{path}'");
}
public async Task MoveAsync(
OpenListConnectionRequest connection,
string sourcePath,
string targetDirectory,
CancellationToken cancellationToken = default)
{
sourcePath = NormalizePath(sourcePath);
targetDirectory = NormalizePath(targetDirectory);
var envelope = await SendAuthorizedAsync(
connection,
() => CreateJsonRequest(
HttpMethod.Post,
"/api/fs/move",
new
{
src_dir = GetDirectoryName(sourcePath),
dst_dir = targetDirectory,
names = new[] { GetFileName(sourcePath) },
overwrite = false
}),
cancellationToken);
EnsureSuccess(envelope, $"OpenList move '{sourcePath}' to '{targetDirectory}'");
}
public static string NormalizeBaseUrl(string baseUrl)
{
if (!Uri.TryCreate(baseUrl?.Trim(), UriKind.Absolute, out var uri) ||
@@ -1,5 +1,6 @@
using System.Security.Cryptography;
using LiveRecorder.Application.Abstractions.Logging;
using LiveRecorder.Application.Abstractions.Recording;
using LiveRecorder.Application.Abstractions.Settings;
using LiveRecorder.Application.Models.RecordTasks;
using LiveRecorder.Application.Models.Settings;
@@ -16,8 +17,11 @@ namespace LiveRecorder.Infrastructure.Services;
public sealed class OpenListUploadQueueService
{
private const int MaxAttempts = 6;
private const int MaxAutomaticRecoveryBatchSize = 100;
private static readonly TimeSpan VerificationTimeout = TimeSpan.FromMinutes(2);
private static readonly TimeSpan ExternalTaskTimeout = TimeSpan.FromHours(24);
private static readonly TimeSpan ExternalTaskStallTimeout = TimeSpan.FromMinutes(60);
private static readonly TimeSpan CleanupRetryDelay = TimeSpan.FromHours(1);
private static readonly TimeSpan[] RetryDelays =
[
TimeSpan.FromMinutes(1),
@@ -31,17 +35,20 @@ public sealed class OpenListUploadQueueService
private readonly ISystemSettingsService _settingsService;
private readonly IOpenListClient _openListClient;
private readonly ISystemLogService _systemLogService;
private readonly IVideoMetadataService _videoMetadataService;
public OpenListUploadQueueService(
LiveRecorderDbContext dbContext,
ISystemSettingsService settingsService,
IOpenListClient openListClient,
ISystemLogService systemLogService)
ISystemLogService systemLogService,
IVideoMetadataService videoMetadataService)
{
_dbContext = dbContext;
_settingsService = settingsService;
_openListClient = openListClient;
_systemLogService = systemLogService;
_videoMetadataService = videoMetadataService;
}
public async Task<RecordArtifactUploadItemResultDto?> TryEnqueueAutomaticAsync(
@@ -59,6 +66,69 @@ public sealed class OpenListUploadQueueService
return await EnqueueInternalAsync(recordTaskId, settings, cancellationToken);
}
public async Task<int> RecoverPendingAutomaticUploadsAsync(
int take = MaxAutomaticRecoveryBatchSize,
CancellationToken cancellationToken = default)
{
var settings = await _settingsService.GetAsync(cancellationToken);
if (!settings.EnableFileUpload ||
!settings.EnableAutoUpload ||
settings.UploadTarget != UploadTargetType.OpenList)
{
return 0;
}
var completedBefore = DateTimeOffset.UtcNow.AddSeconds(-30);
var taskIds = await _dbContext.RecordTasks
.AsNoTracking()
.Where(item =>
(item.Status == RecordTaskStatus.Completed || item.Status == RecordTaskStatus.Stopped) &&
item.Result != null &&
item.Result.UploadStatus == RecordArtifactUploadStatus.NotUploaded &&
item.UploadJob == null &&
item.UpdatedAt <= completedBefore &&
!string.IsNullOrWhiteSpace(item.Result.FilePath))
.OrderBy(static item => item.UpdatedAt)
.Select(static item => item.Id)
.Take(Math.Clamp(take, 1, MaxAutomaticRecoveryBatchSize))
.ToArrayAsync(cancellationToken);
var recovered = 0;
foreach (var taskId in taskIds)
{
try
{
var result = await EnqueueInternalAsync(taskId, settings, cancellationToken);
if (result.Success && result.UploadStatus == RecordArtifactUploadStatus.Queued)
{
recovered++;
}
}
catch (Exception ex)
{
await _systemLogService.WriteAsync(
SystemLogLevel.Warning,
"Upload",
"跳过一个无法恢复的 OpenList 自动上传任务。",
$"recordTaskId={taskId}; error={ex.Message}",
recordTaskId: taskId,
cancellationToken: cancellationToken);
}
}
if (recovered > 0)
{
await _systemLogService.WriteAsync(
SystemLogLevel.Warning,
"Upload",
"Recovered completed recording segments that had not entered the automatic OpenList upload queue.",
$"count={recovered}",
cancellationToken: cancellationToken);
}
return recovered;
}
public async Task<RecordArtifactUploadItemResultDto> EnqueueAsync(
Guid recordTaskId,
CancellationToken cancellationToken = default)
@@ -145,6 +215,16 @@ public sealed class OpenListUploadQueueService
}
var result = job.RecordTask.Result;
var validationError = await GetUploadValidationErrorAsync(
job.RecordTask,
NormalizeAbsolutePath(result.FilePath),
cancellationToken);
if (!string.IsNullOrWhiteSpace(validationError))
{
await MarkFailedAsync(job, result, validationError, cancellationToken);
return true;
}
if (job.Status != RecordArtifactUploadStatus.Uploading)
{
job.BeginAttempt(now);
@@ -206,6 +286,12 @@ public sealed class OpenListUploadQueueService
return Failure(recordTaskId, "本地视频文件不存在,不能加入上传队列。", "openlist");
}
var validationError = await GetUploadValidationErrorAsync(recordTask, localVideoPath, cancellationToken);
if (!string.IsNullOrWhiteSpace(validationError))
{
return Failure(recordTaskId, validationError, "openlist");
}
var outputRoot = Path.GetFullPath(settings.OutputRoot, AppContext.BaseDirectory);
var videoRelativePath = GetSafeRelativePath(outputRoot, localVideoPath);
var sourceVideoPath = OpenListClient.CombinePath(settings.OpenListUpload.SourcePath, videoRelativePath);
@@ -285,6 +371,46 @@ public sealed class OpenListUploadQueueService
return QueuedResult(recordTaskId, result, job, "已加入 OpenList 上传队列。");
}
private async Task<string?> GetUploadValidationErrorAsync(
RecordTask recordTask,
string localVideoPath,
CancellationToken cancellationToken)
{
if (recordTask.Status is not (RecordTaskStatus.Completed or RecordTaskStatus.Stopped))
{
return "只有已完成或已停止且媒体有效的录制任务可以上传。";
}
if (recordTask.OutputFormat == RecordOutputFormat.Mp4 &&
!localVideoPath.EndsWith(".mp4", StringComparison.OrdinalIgnoreCase))
{
return "MP4 收尾未完成,仅保留了中间文件,已阻止自动上传。";
}
if (!File.Exists(localVideoPath) || new FileInfo(localVideoPath).Length <= 0)
{
return "本地视频文件不存在或为空,不能上传。";
}
var metadata = await _videoMetadataService.ExtractMetadataAsync(localVideoPath, cancellationToken);
if (metadata is null)
{
return "ffprobe 无法读取媒体信息,已阻止上传并保留本地文件。";
}
if (string.IsNullOrWhiteSpace(metadata.VideoCodec))
{
return "文件不包含视频流,已阻止上传并保留本地文件。";
}
if (!metadata.DurationSeconds.HasValue || metadata.DurationSeconds.Value < 5)
{
return $"实际媒体时长不足 5 秒({metadata.DurationSeconds.GetValueOrDefault():0.###} 秒),已阻止上传并保留本地文件。";
}
return null;
}
private async Task ProcessJobStepAsync(
RecordUploadJob job,
RecordResult result,
@@ -299,21 +425,34 @@ public sealed class OpenListUploadQueueService
var now = DateTimeOffset.UtcNow;
var targetPath = job.GetCurrentTargetPath();
var transferTargetPath = job.GetCurrentTransferTargetPath();
var expectedSize = job.GetCurrentSizeBytes();
var localPath = GetCurrentLocalPath(job, result);
if (job.VerificationStartedAt.HasValue)
{
var verification = await VerifyTargetAsync(connection, targetPath, localPath, expectedSize, cancellationToken);
var verification = await VerifyTargetAsync(connection, transferTargetPath, localPath, expectedSize, allowSizeOnlyMatch: true, cancellationToken);
if (verification == TargetVerification.Match)
{
if (!string.Equals(transferTargetPath, targetPath, StringComparison.Ordinal))
{
await PromoteStagedTransferAsync(job, connection, transferTargetPath, targetPath, now, cancellationToken);
return;
}
await CompleteCurrentArtifactAsync(job, result, cancellationToken);
return;
}
if (verification == TargetVerification.Conflict)
{
throw new OpenListUploadConflictException($"目标文件 '{targetPath}' 已存在但内容不一致。");
throw new OpenListUploadConflictException($"目标文件 '{transferTargetPath}' 已存在但内容不一致。");
}
if (!string.Equals(transferTargetPath, targetPath, StringComparison.Ordinal) &&
await TryRecoverInterruptedPromotionAsync(job, connection, transferTargetPath, targetPath, localPath, expectedSize, now, cancellationToken))
{
return;
}
if (now - job.VerificationStartedAt.Value < VerificationTimeout)
@@ -321,25 +460,49 @@ public sealed class OpenListUploadQueueService
return;
}
await ScheduleRetryOrFailAsync(job, result, $"OpenList 任务结束后两分钟内仍未发现目标文件 '{targetPath}'。", true, cancellationToken);
await ScheduleRetryOrFailAsync(job, result, $"OpenList 任务结束后两分钟内仍未发现目标文件 '{transferTargetPath}'。", true, cancellationToken);
return;
}
if (!string.IsNullOrWhiteSpace(job.ExternalTaskId))
{
var lastProgressAt = job.LastProgressAt ?? job.ExternalTaskStartedAt;
if (lastProgressAt.HasValue && now - lastProgressAt.Value > ExternalTaskStallTimeout)
{
var cancelled = await _openListClient.TryCancelCopyTaskAsync(connection, job.ExternalTaskId, cancellationToken);
if (cancelled)
{
await ScheduleRetryOrFailAsync(job, result, $"OpenList 复制任务连续 60 分钟没有进度,已取消并准备重试:{job.ExternalTaskId}", true, cancellationToken);
}
else
{
await MarkFailedAsync(job, result, $"OpenList 复制任务连续 60 分钟没有进度且无法取消,需要人工检查:{job.ExternalTaskId}", cancellationToken);
}
return;
}
if (job.ExternalTaskStartedAt.HasValue && now - job.ExternalTaskStartedAt.Value > ExternalTaskTimeout)
{
await ScheduleRetryOrFailAsync(job, result, $"OpenList 复制任务运行超过 24 小时:{job.ExternalTaskId}", true, cancellationToken);
_ = await _openListClient.TryCancelCopyTaskAsync(connection, job.ExternalTaskId, cancellationToken);
await ScheduleRetryOrFailAsync(job, result, $"OpenList 复制任务运行超过 24 小时,已停止跟踪并准备重试:{job.ExternalTaskId}", true, cancellationToken);
return;
}
var task = await _openListClient.TryGetCopyTaskAsync(connection, job.ExternalTaskId, cancellationToken);
if (task is null)
{
var missingTaskVerification = await VerifyTargetAsync(connection, targetPath, localPath, expectedSize, cancellationToken);
var missingTaskVerification = await VerifyTargetAsync(connection, transferTargetPath, localPath, expectedSize, allowSizeOnlyMatch: true, cancellationToken);
if (missingTaskVerification == TargetVerification.Match)
{
await CompleteCurrentArtifactAsync(job, result, cancellationToken);
if (!string.Equals(transferTargetPath, targetPath, StringComparison.Ordinal))
{
await PromoteStagedTransferAsync(job, connection, transferTargetPath, targetPath, now, cancellationToken);
}
else
{
await CompleteCurrentArtifactAsync(job, result, cancellationToken);
}
}
else if (missingTaskVerification == TargetVerification.Conflict)
{
@@ -380,7 +543,7 @@ public sealed class OpenListUploadQueueService
// first target probe so a new recording path can be uploaded normally.
await _openListClient.EnsureDirectoryAsync(connection, GetDirectoryName(targetPath), cancellationToken);
var existingTarget = await VerifyTargetAsync(connection, targetPath, localPath, expectedSize, cancellationToken);
var existingTarget = await VerifyTargetAsync(connection, targetPath, localPath, expectedSize, allowSizeOnlyMatch: false, cancellationToken);
if (existingTarget == TargetVerification.Match)
{
await CompleteCurrentArtifactAsync(job, result, cancellationToken);
@@ -389,7 +552,20 @@ public sealed class OpenListUploadQueueService
if (existingTarget == TargetVerification.Conflict)
{
throw new OpenListUploadConflictException($"目标文件 '{targetPath}' 已存在但内容不一致。");
if (HasTaskConflictSuffix(targetPath, job.RecordTaskId))
{
throw new OpenListUploadConflictException($"目标文件 '{targetPath}' 已存在但内容不一致。");
}
var conflictTargetPath = AppendTaskConflictSuffix(targetPath, job.RecordTaskId);
var stagingDirectory = OpenListClient.CombinePath(
GetDirectoryName(targetPath),
$".liverecorder-staging-{job.Id.ToString("N")[..8]}");
var stagedTargetPath = OpenListClient.CombinePath(stagingDirectory, GetFileName(job.GetCurrentSourcePath()));
await _openListClient.EnsureDirectoryAsync(connection, stagingDirectory, cancellationToken);
job.ResolveCurrentTargetConflict(conflictTargetPath, stagedTargetPath, now);
await _dbContext.SaveChangesAsync(cancellationToken);
return;
}
result.MarkUploadStarted("openlist", now);
@@ -407,7 +583,11 @@ public sealed class OpenListUploadQueueService
throw new InvalidOperationException($"OpenList 源文件大小不一致:期望 {expectedSize},实际 {sourceObject.Size},路径 {sourcePath}");
}
var copyResult = await _openListClient.CopyFileAsync(connection, sourcePath, targetPath, cancellationToken);
if (!string.Equals(GetDirectoryName(transferTargetPath), GetDirectoryName(targetPath), StringComparison.Ordinal))
{
await _openListClient.EnsureDirectoryAsync(connection, GetDirectoryName(transferTargetPath), cancellationToken);
}
var copyResult = await _openListClient.CopyFileAsync(connection, sourcePath, transferTargetPath, cancellationToken);
if (copyResult.TaskIds.Count == 0)
{
job.StartVerification(DateTimeOffset.UtcNow);
@@ -420,6 +600,74 @@ public sealed class OpenListUploadQueueService
await _dbContext.SaveChangesAsync(cancellationToken);
}
private async Task PromoteStagedTransferAsync(
RecordUploadJob job,
OpenListConnectionRequest connection,
string stagedPath,
string targetPath,
DateTimeOffset now,
CancellationToken cancellationToken)
{
var targetFileName = GetFileName(targetPath);
var renamedStagedPath = stagedPath;
if (!string.Equals(GetFileName(stagedPath), targetFileName, StringComparison.Ordinal))
{
await _openListClient.RenameAsync(connection, stagedPath, targetFileName, cancellationToken);
renamedStagedPath = OpenListClient.CombinePath(GetDirectoryName(stagedPath), targetFileName);
job.UpdateTransferTargetPath(renamedStagedPath, DateTimeOffset.UtcNow);
await _dbContext.SaveChangesAsync(cancellationToken);
}
await _openListClient.MoveAsync(connection, renamedStagedPath, GetDirectoryName(targetPath), cancellationToken);
job.CompleteTransferPromotion(now);
job.StartVerification(now);
await _dbContext.SaveChangesAsync(cancellationToken);
}
private async Task<bool> TryRecoverInterruptedPromotionAsync(
RecordUploadJob job,
OpenListConnectionRequest connection,
string stagedPath,
string targetPath,
string localPath,
long expectedSize,
DateTimeOffset now,
CancellationToken cancellationToken)
{
var finalVerification = await VerifyTargetAsync(
connection,
targetPath,
localPath,
expectedSize,
allowSizeOnlyMatch: true,
cancellationToken);
if (finalVerification == TargetVerification.Match)
{
job.CompleteTransferPromotion(now);
job.StartVerification(now);
await _dbContext.SaveChangesAsync(cancellationToken);
return true;
}
var renamedStagedPath = OpenListClient.CombinePath(GetDirectoryName(stagedPath), GetFileName(targetPath));
var renamedVerification = await VerifyTargetAsync(
connection,
renamedStagedPath,
localPath,
expectedSize,
allowSizeOnlyMatch: true,
cancellationToken);
if (renamedVerification != TargetVerification.Match)
{
return false;
}
job.UpdateTransferTargetPath(renamedStagedPath, now);
await _dbContext.SaveChangesAsync(cancellationToken);
await PromoteStagedTransferAsync(job, connection, renamedStagedPath, targetPath, now, cancellationToken);
return true;
}
private async Task CompleteCurrentArtifactAsync(
RecordUploadJob job,
RecordResult result,
@@ -461,6 +709,20 @@ public sealed class OpenListUploadQueueService
{
cleanupWarning = ex.Message;
}
if (!deletedLocalFiles && cleanupWarning is null)
{
cleanupWarning = "本地文件删除后仍然存在。";
}
}
if (cleanupWarning is not null)
{
var nextAttemptAt = now.Add(CleanupRetryDelay);
job.ScheduleRetry($"远端上传已完成,本地清理失败:{cleanupWarning}", nextAttemptAt, now, clearExternalTask: true);
result.MarkUploadWaitingRetry("openlist", job.ErrorMessage, now);
await _dbContext.SaveChangesAsync(cancellationToken);
return;
}
job.MarkSucceeded(now);
@@ -542,6 +804,7 @@ public sealed class OpenListUploadQueueService
string targetPath,
string localPath,
long expectedSize,
bool allowSizeOnlyMatch,
CancellationToken cancellationToken)
{
var remote = await _openListClient.TryGetObjectAsync(connection, targetPath, cancellationToken);
@@ -561,7 +824,7 @@ public sealed class OpenListUploadQueueService
pair.Key.Equals("md5", StringComparison.OrdinalIgnoreCase));
if (string.IsNullOrWhiteSpace(comparableHash.Key) || string.IsNullOrWhiteSpace(comparableHash.Value))
{
return TargetVerification.Match;
return allowSizeOnlyMatch ? TargetVerification.Match : TargetVerification.Conflict;
}
var localHash = await ComputeFileHashAsync(localPath, comparableHash.Key, cancellationToken);
@@ -679,6 +942,26 @@ public sealed class OpenListUploadQueueService
return index <= 0 ? "/" : normalized[..index];
}
private static string GetFileName(string path)
{
var normalized = OpenListClient.NormalizePath(path);
var index = normalized.LastIndexOf('/');
return index < 0 ? normalized : normalized[(index + 1)..];
}
private static string AppendTaskConflictSuffix(string path, Guid recordTaskId)
{
var directory = GetDirectoryName(path);
var fileName = GetFileName(path);
var extension = Path.GetExtension(fileName);
var stem = extension.Length == 0 ? fileName : fileName[..^extension.Length];
return OpenListClient.CombinePath(directory, $"{stem}_{recordTaskId.ToString("N")[..8]}{extension}");
}
private static bool HasTaskConflictSuffix(string path, Guid recordTaskId) =>
Path.GetFileNameWithoutExtension(GetFileName(path))
.EndsWith($"_{recordTaskId.ToString("N")[..8]}", StringComparison.OrdinalIgnoreCase);
private static RecordArtifactUploadItemResultDto Failure(Guid recordTaskId, string message, string provider) => new()
{
RecordTaskId = recordTaskId,
@@ -738,6 +1021,7 @@ public sealed class OpenListUploadQueueService
public sealed class OpenListUploadBackgroundService : BackgroundService
{
private static readonly TimeSpan IdleDelay = TimeSpan.FromSeconds(2);
private static readonly TimeSpan RecoveryInterval = TimeSpan.FromMinutes(1);
private readonly IServiceScopeFactory _scopeFactory;
private readonly ILogger<OpenListUploadBackgroundService> _logger;
@@ -751,21 +1035,21 @@ public sealed class OpenListUploadBackgroundService : BackgroundService
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var nextRecoveryAt = DateTimeOffset.MinValue;
while (!stoppingToken.IsCancellationRequested)
{
try
{
using var scope = _scopeFactory.CreateScope();
var queue = scope.ServiceProvider.GetRequiredService<OpenListUploadQueueService>();
var processed = await queue.ProcessNextAsync(stoppingToken);
if (!processed)
_ = await queue.ProcessNextAsync(stoppingToken);
if (DateTimeOffset.UtcNow >= nextRecoveryAt)
{
await Task.Delay(IdleDelay, stoppingToken);
}
else
{
await Task.Delay(IdleDelay, stoppingToken);
await queue.RecoverPendingAutomaticUploadsAsync(cancellationToken: stoppingToken);
nextRecoveryAt = DateTimeOffset.UtcNow.Add(RecoveryInterval);
}
await Task.Delay(IdleDelay, stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
@@ -48,6 +48,17 @@ public sealed class RecordSessionCleanupResolver
{
var cutoff = DateTimeOffset.UtcNow.AddDays(-Math.Max(1, filters.RetentionDays));
var nonEmptySessionIds = await ResolveConditionalSessionIdsAsync(filters, cutoff, cancellationToken);
if (filters.RequireUploadSuccess && nonEmptySessionIds.Count > 0)
{
nonEmptySessionIds = await _dbContext.RecordSessions
.AsNoTracking()
.Where(session => nonEmptySessionIds.Contains(session.Id) &&
_dbContext.RecordTasks
.Where(task => task.RecordSessionId == session.Id)
.All(task => task.Result != null && task.Result.UploadStatus == RecordArtifactUploadStatus.Succeeded))
.Select(static session => session.Id)
.ToArrayAsync(cancellationToken);
}
var emptySessionIds = await ResolveEmptySessionIdsAsync(cutoff, cancellationToken);
return nonEmptySessionIds
@@ -75,8 +86,14 @@ public sealed class RecordSessionCleanupResolver
IQueryable<RecordSession> query = _dbContext.RecordSessions
.AsNoTracking()
.Where(static item => item.Status != RecordSessionStatus.Starting &&
item.Status != RecordSessionStatus.Pending &&
item.Status != RecordSessionStatus.Running &&
item.Status != RecordSessionStatus.Stopping)
.Where(item => !_dbContext.RecordTasks.Any(task =>
task.RecordSessionId == item.Id && task.UploadJob != null &&
(task.UploadJob.Status == RecordArtifactUploadStatus.Queued ||
task.UploadJob.Status == RecordArtifactUploadStatus.Uploading ||
task.UploadJob.Status == RecordArtifactUploadStatus.WaitingRetry)))
.Where(item => _dbContext.RecordTasks.Any(task => task.RecordSessionId == item.Id));
if (createdBeforeUtc.HasValue)
@@ -133,6 +150,7 @@ public sealed class RecordSessionCleanupResolver
IQueryable<RecordSession> query = _dbContext.RecordSessions
.AsNoTracking()
.Where(static item => item.Status != RecordSessionStatus.Starting &&
item.Status != RecordSessionStatus.Pending &&
item.Status != RecordSessionStatus.Running &&
item.Status != RecordSessionStatus.Stopping)
.Where(item => !_dbContext.RecordTasks.Any(task => task.RecordSessionId == item.Id));
@@ -153,6 +171,7 @@ public sealed class RecordSessionCleanupResolver
var activeSessionIds = await _dbContext.RecordSessions
.AsNoTracking()
.Where(static item => item.Status == RecordSessionStatus.Starting ||
item.Status == RecordSessionStatus.Pending ||
item.Status == RecordSessionStatus.Running ||
item.Status == RecordSessionStatus.Stopping)
.Select(static item => item.Id)
@@ -46,13 +46,19 @@ public sealed class RecoveryService
Storage = new StorageGuardStatusDto
{
IsEnabled = storage.IsEnabled,
IsAvailable = storage.IsAvailable,
HasEnoughSpace = storage.HasEnoughSpace,
CheckedPath = storage.CheckedPath,
TotalBytes = storage.TotalBytes,
UsedBytes = storage.UsedBytes,
AvailableBytes = storage.AvailableBytes,
RequiredBytes = storage.RequiredBytes,
Message = storage.Message,
Tier = storage.Tier.ToString(),
UsagePercent = storage.UsagePercent
UsagePercent = storage.UsagePercent,
FreePercent = storage.FreePercent,
GreenThresholdPercent = storage.GreenThresholdPercent,
RedThresholdPercent = storage.RedThresholdPercent
},
LiveRooms = liveRooms,
Finalizations = finalizations
@@ -64,17 +70,17 @@ public sealed class RecoveryService
var room = await _dbContext.LiveRooms.FirstOrDefaultAsync(item => item.Id == liveRoomId, cancellationToken);
if (room is null)
{
return FailureResult("Live room was not found.");
return FailureResult("未找到该直播间。");
}
if (!room.IsEnabled)
{
return FailureResult("Live room is disabled and cannot be retried.");
return FailureResult("该直播间已禁用,无法重试开录。");
}
if (room.AvailabilityStatus != LiveRoomAvailabilityStatus.Live)
{
return FailureResult("Live room is not currently online.");
return FailureResult("该直播间当前未开播,无法重试开录。");
}
var hasActiveSession = await _dbContext.RecordSessions.AnyAsync(
@@ -88,11 +94,11 @@ public sealed class RecoveryService
{
room.SetLastAutoStartDecision(
AutoStartDecisionCodes.SkippedActiveSession,
"Auto-start skipped because an active recording session already exists.",
"已有活动录制会话,本次自动开录已跳过。",
null,
DateTimeOffset.UtcNow);
await _dbContext.SaveChangesAsync(cancellationToken);
return FailureResult("An active recording session already exists for this live room.");
return FailureResult("该直播间已经存在活动录制会话。");
}
try
@@ -114,14 +120,14 @@ public sealed class RecoveryService
Messages =
[
started
? $"Recording retry started for room {room.RoomId}."
: $"Recording retry did not start for room {room.RoomId}. Status={task.Status}; Error={task.ErrorMessage ?? "n/a"}"
? $"直播间 {room.RoomId} 已开始重试录制。"
: $"直播间 {room.RoomId} 未能开始录制。状态={task.Status};错误={task.ErrorMessage ?? ""}"
]
};
}
catch (Exception ex)
{
return FailureResult($"Recording retry failed for room {room.RoomId}: {ex.Message}");
return FailureResult($"直播间 {room.RoomId} 重试录制失败:{ex.Message}");
}
}
@@ -135,7 +141,7 @@ public sealed class RecoveryService
RequestedCount = 0,
SuccessCount = 0,
FailedCount = 0,
Messages = ["No live rooms currently require retry."]
Messages = ["当前没有需要重试开录的直播间。"]
};
}
@@ -159,7 +165,8 @@ public sealed class RecoveryService
public async Task<RecoveryActionResultDto> ResumeFinalizationAsync(Guid recordTaskId, CancellationToken cancellationToken = default)
{
var started = await _ffmpegService.StartManualFinalizeTaskAsync(recordTaskId, cancellationToken);
var recoveredOrphan = await _ffmpegService.TryRecoverOrphanedTerminalTaskAsync(recordTaskId, cancellationToken);
var started = recoveredOrphan || await _ffmpegService.StartManualFinalizeTaskAsync(recordTaskId, cancellationToken);
return new RecoveryActionResultDto
{
RequestedCount = 1,
@@ -168,8 +175,10 @@ public sealed class RecoveryService
Messages =
[
started
? $"MP4 finalization resumed for task {recordTaskId}."
: $"MP4 finalization could not be resumed for task {recordTaskId}."
? recoveredOrphan
? $"任务 {recordTaskId} 的遗留分片已恢复,并已进入后续上传流程。"
: $"任务 {recordTaskId} 已恢复 MP4 转码。"
: $"任务 {recordTaskId} 无法恢复 MP4 转码。"
]
};
}
@@ -184,7 +193,7 @@ public sealed class RecoveryService
RequestedCount = 0,
SuccessCount = 0,
FailedCount = 0,
Messages = ["No MP4 finalization tasks currently require recovery."]
Messages = ["当前没有需要恢复的 MP4 转码任务。"]
};
}
@@ -192,7 +201,8 @@ public sealed class RecoveryService
var messages = new List<string>();
foreach (var item in finalizations)
{
var started = await _ffmpegService.StartManualFinalizeTaskAsync(item.RecordTaskId, cancellationToken);
var recoveredOrphan = await _ffmpegService.TryRecoverOrphanedTerminalTaskAsync(item.RecordTaskId, cancellationToken);
var started = recoveredOrphan || await _ffmpegService.StartManualFinalizeTaskAsync(item.RecordTaskId, cancellationToken);
if (started)
{
successCount++;
@@ -200,8 +210,10 @@ public sealed class RecoveryService
messages.Add(
started
? $"MP4 finalization resumed for task {item.RecordTaskId}."
: $"MP4 finalization could not be resumed for task {item.RecordTaskId}.");
? recoveredOrphan
? $"任务 {item.RecordTaskId} 的遗留分片已恢复。"
: $"任务 {item.RecordTaskId} 已恢复 MP4 转码。"
: $"任务 {item.RecordTaskId} 无法恢复 MP4 转码。");
}
return new RecoveryActionResultDto
@@ -290,16 +302,6 @@ public sealed class RecoveryService
return false;
}
if (recordTask.Status == RecordTaskStatus.Processing)
{
return true;
}
if (recordTask.Status is RecordTaskStatus.Starting or RecordTaskStatus.Running or RecordTaskStatus.Stopping)
{
return false;
}
if (recordTask.RecordSession.Status is RecordSessionStatus.Starting or RecordSessionStatus.Running or RecordSessionStatus.Stopping)
{
return false;
@@ -310,8 +312,9 @@ public sealed class RecoveryService
return false;
}
var manifestSources = FfmpegService.ReadRecorderSegmentsManifest(recordTask.OutputFilePath);
var recorderOutputPath = ResolveManualFinalizeSourcePath(recordTask, recordTask.RecordSession);
if (!File.Exists(recorderOutputPath))
if (manifestSources.Count == 0 && !File.Exists(recorderOutputPath))
{
return false;
}
@@ -330,16 +333,21 @@ public sealed class RecoveryService
if (recordTask.Status == RecordTaskStatus.Processing)
{
return string.IsNullOrWhiteSpace(recordTask.ErrorMessage)
? "MP4 finalization is queued or paused and can be resumed."
? "MP4 转码正在排队或已暂停,可以继续恢复。"
: recordTask.ErrorMessage!;
}
if (recordTask.Status is RecordTaskStatus.Starting or RecordTaskStatus.Running or RecordTaskStatus.Stopping)
{
return "父会话已经结束,但该分片仍显示为录制中;可安全对账并恢复转码、上传。";
}
if (recordTask.Status == RecordTaskStatus.Completed)
{
return "The final MP4 output is missing, but the intermediate recording file is still available.";
return "最终 MP4 文件缺失,但中间录制文件仍然存在。";
}
return "Manual MP4 finalization can be retried from the intermediate recording file.";
return "可以使用保留的中间录制文件重新执行 MP4 转码。";
}
private static string ResolveManualFinalizeSourcePath(RecordTask recordTask, RecordSession recordSession)
@@ -15,77 +15,92 @@ public sealed class StorageGuardService : IStorageGuardService
}
public StorageGuardResult CheckCanStartOrResume(SystemSettingsDto settings, long additionalRequiredBytes = 0) =>
Check(settings, Math.Max(settings.PauseRecordingWhenFreeSpaceBelowMegabytes, settings.ResumeRecordingWhenFreeSpaceAboveMegabytes), additionalRequiredBytes);
Check(
settings,
Math.Max(settings.PauseRecordingWhenFreeSpaceBelowMegabytes, settings.ResumeRecordingWhenFreeSpaceAboveMegabytes),
additionalRequiredBytes);
public StorageGuardResult CheckShouldPause(SystemSettingsDto settings) =>
Check(settings, settings.PauseRecordingWhenFreeSpaceBelowMegabytes, additionalRequiredBytes: 0);
public StorageGuardResult CheckCanFinalize(SystemSettingsDto settings, long estimatedTemporaryBytes) =>
Check(settings, settings.PauseRecordingWhenFreeSpaceBelowMegabytes, Math.Max(0, estimatedTemporaryBytes));
private StorageGuardResult Check(SystemSettingsDto settings, int freeSpaceThresholdMegabytes, long additionalRequiredBytes)
{
if (!settings.EnableStorageGuard)
{
return new StorageGuardResult(false, true, ResolveOutputRoot(settings.OutputRoot), long.MaxValue, 0, "Storage guard is disabled.")
{
Tier = StorageTier.Green,
UsagePercent = 0
};
}
var checkedPath = ResolveOutputRoot(settings.OutputRoot);
var thresholdBytes = Math.Max(0, freeSpaceThresholdMegabytes) * Megabyte;
var requiredBytes = thresholdBytes + Math.Max(0, additionalRequiredBytes);
var isEnabled = settings.EnableStorageGuard;
var pauseThresholdBytes = isEnabled
? Math.Max(0, settings.PauseRecordingWhenFreeSpaceBelowMegabytes) * Megabyte
: 0;
var resumeThresholdBytes = isEnabled
? Math.Max(settings.PauseRecordingWhenFreeSpaceBelowMegabytes, settings.ResumeRecordingWhenFreeSpaceAboveMegabytes) * Megabyte
: 0;
var thresholdBytes = isEnabled ? Math.Max(0, freeSpaceThresholdMegabytes) * Megabyte : 0;
var requiredBytes = thresholdBytes + (isEnabled ? Math.Max(0, additionalRequiredBytes) : 0);
var greenThreshold = Math.Clamp(settings.StorageGreenThresholdPercent, 10, 90);
var redThreshold = Math.Clamp(settings.StorageRedThresholdPercent, 5, greenThreshold - 5);
var checkedPath = settings.OutputRoot?.Trim() ?? string.Empty;
try
{
checkedPath = ResolveOutputRoot(settings.OutputRoot ?? string.Empty);
var drive = new DriveInfo(Path.GetPathRoot(checkedPath) ?? checkedPath);
var availableBytes = drive.AvailableFreeSpace;
var totalBytes = drive.TotalSize;
var usedBytes = Math.Max(0, totalBytes - availableBytes);
var usagePercent = totalBytes > 0 ? (double)usedBytes / totalBytes * 100.0 : 0;
var freePercent = 100.0 - usagePercent;
// Determine tier using configurable thresholds
var greenThreshold = Math.Clamp(settings.StorageGreenThresholdPercent, 5, 90);
var redThreshold = Math.Clamp(settings.StorageRedThresholdPercent, 1, greenThreshold - 1);
var freePercent = totalBytes > 0 ? (double)availableBytes / totalBytes * 100.0 : 0;
StorageTier tier;
if (freePercent >= greenThreshold)
{
tier = StorageTier.Green;
}
else if (freePercent >= redThreshold)
{
tier = StorageTier.Yellow;
}
else
if (freePercent < redThreshold || availableBytes < pauseThresholdBytes)
{
tier = StorageTier.Red;
}
var hasEnoughSpace = availableBytes >= requiredBytes;
var message = tier switch
else if (freePercent >= greenThreshold && availableBytes >= resumeThresholdBytes)
{
StorageTier.Green => $"Storage is healthy. free={FormatBytes(availableBytes)} ({freePercent:F1}%), required={FormatBytes(requiredBytes)}, path={checkedPath}",
StorageTier.Yellow => $"Storage is low. free={FormatBytes(availableBytes)} ({freePercent:F1}%), required={FormatBytes(requiredBytes)}, path={checkedPath}. New recordings paused, existing recordings continue.",
StorageTier.Red => $"Storage is critically low. free={FormatBytes(availableBytes)} ({freePercent:F1}%), required={FormatBytes(requiredBytes)}, path={checkedPath}. All recordings paused, uploads continue.",
tier = StorageTier.Green;
}
else
{
tier = StorageTier.Yellow;
}
var hasEnoughSpace = !isEnabled || availableBytes >= requiredBytes;
var message = !isEnabled
? $"存储保护已关闭。可用={FormatBytes(availableBytes)}{freePercent:F1}%),路径={checkedPath}"
: tier switch
{
StorageTier.Green => $"存储空间正常。可用={FormatBytes(availableBytes)}{freePercent:F1}%),本次需要={FormatBytes(requiredBytes)},路径={checkedPath}",
StorageTier.Yellow => $"存储空间偏低。可用={FormatBytes(availableBytes)}{freePercent:F1}%),本次需要={FormatBytes(requiredBytes)},路径={checkedPath}。暂停新录制,已有录制继续。",
StorageTier.Red => $"存储空间严重不足。可用={FormatBytes(availableBytes)}{freePercent:F1}%),本次需要={FormatBytes(requiredBytes)},路径={checkedPath}。暂停录制;空间足够保留安全余量时仍允许 MP4 收尾,上传继续。",
_ => hasEnoughSpace
? $"Storage is available. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}"
: $"Storage is below threshold. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}"
? $"存储空间可用。可用={FormatBytes(availableBytes)},本次需要={FormatBytes(requiredBytes)},路径={checkedPath}"
: $"存储空间低于阈值。可用={FormatBytes(availableBytes)},本次需要={FormatBytes(requiredBytes)},路径={checkedPath}"
};
return new StorageGuardResult(true, hasEnoughSpace, checkedPath, availableBytes, requiredBytes, message)
return new StorageGuardResult(isEnabled, hasEnoughSpace, checkedPath, availableBytes, requiredBytes, message)
{
IsAvailable = totalBytes > 0,
Tier = tier,
UsagePercent = Math.Round(usagePercent, 1)
TotalBytes = totalBytes,
UsedBytes = usedBytes,
UsagePercent = Math.Round(Math.Clamp(usagePercent, 0, 100), 1),
FreePercent = Math.Round(Math.Clamp(freePercent, 0, 100), 1),
GreenThresholdPercent = greenThreshold,
RedThresholdPercent = redThreshold
};
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Storage guard failed to inspect output root {OutputRoot}", checkedPath);
return new StorageGuardResult(true, false, checkedPath, 0, requiredBytes, $"Unable to inspect storage path {checkedPath}: {ex.Message}")
return new StorageGuardResult(isEnabled, !isEnabled, checkedPath, 0, requiredBytes, $"无法检查存储路径 {checkedPath}{ex.Message}")
{
IsAvailable = false,
Tier = StorageTier.Red,
UsagePercent = 0
UsagePercent = 0,
FreePercent = 0,
GreenThresholdPercent = greenThreshold,
RedThresholdPercent = redThreshold
};
}
}
@@ -0,0 +1,81 @@
using LiveRecorder.Infrastructure.Persistence;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace LiveRecorder.Infrastructure.Services;
public sealed class SystemLogRetentionBackgroundService : BackgroundService
{
private const int BatchSize = 1000;
private static readonly TimeSpan Retention = TimeSpan.FromDays(90);
private static readonly TimeSpan Interval = TimeSpan.FromHours(24);
private readonly IServiceScopeFactory _scopeFactory;
private readonly ILogger<SystemLogRetentionBackgroundService> _logger;
public SystemLogRetentionBackgroundService(
IServiceScopeFactory scopeFactory,
ILogger<SystemLogRetentionBackgroundService> logger)
{
_scopeFactory = scopeFactory;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
try
{
var deleted = await DeleteExpiredLogsAsync(stoppingToken);
if (deleted > 0)
{
_logger.LogInformation("Deleted {LogCount} system log entries older than 90 days.", deleted);
}
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "System log retention cleanup failed.");
}
await Task.Delay(Interval, stoppingToken);
}
}
private async Task<int> DeleteExpiredLogsAsync(CancellationToken cancellationToken)
{
var cutoff = DateTimeOffset.UtcNow.Subtract(Retention);
var total = 0;
while (!cancellationToken.IsCancellationRequested)
{
using var scope = _scopeFactory.CreateScope();
var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
var ids = await dbContext.SystemLogEntries
.AsNoTracking()
.Where(log => log.CreatedAt < cutoff)
.OrderBy(static log => log.CreatedAt)
.Select(static log => log.Id)
.Take(BatchSize)
.ToArrayAsync(cancellationToken);
if (ids.Length == 0)
{
return total;
}
total += await dbContext.SystemLogEntries
.Where(log => ids.Contains(log.Id))
.ExecuteDeleteAsync(cancellationToken);
if (ids.Length < BatchSize)
{
return total;
}
}
return total;
}
}