feat: improve recording automation and task workflows

This commit is contained in:
2026-04-23 23:18:11 +08:00
parent 1c892259a9
commit 23ead56781
88 changed files with 9579 additions and 1581 deletions
@@ -0,0 +1,278 @@
using System.Diagnostics;
using LiveRecorder.Application.Abstractions.Logging;
using LiveRecorder.Application.Abstractions.Scripting;
using LiveRecorder.Application.Abstractions.Settings;
using LiveRecorder.Domain.Entities;
using LiveRecorder.Domain.Enums;
using Microsoft.Extensions.Logging;
namespace LiveRecorder.Infrastructure.Services;
public sealed class EventScriptService : IEventScriptService
{
private readonly ISystemSettingsService _settingsService;
private readonly ISystemLogService _systemLogService;
private readonly ILogger<EventScriptService> _logger;
public EventScriptService(
ISystemSettingsService settingsService,
ISystemLogService systemLogService,
ILogger<EventScriptService> logger)
{
_settingsService = settingsService;
_systemLogService = systemLogService;
_logger = logger;
}
public async Task RunLiveStartedAsync(LiveRoom liveRoom, DateTimeOffset occurredAt, CancellationToken cancellationToken = default)
{
var settings = await _settingsService.GetAsync(cancellationToken);
var environment = BuildLiveRoomEnvironment(liveRoom, occurredAt);
environment["LIVE_RECORDER_EVENT"] = "live_started";
await RunAsync(
settings.EnableEventScripts,
settings.LiveStartedScriptPath,
settings.EventScriptTimeoutSeconds,
"live_started",
environment,
liveRoom.Id,
recordSessionId: null,
recordTaskId: null,
cancellationToken);
}
public async Task RunLiveEndedAsync(LiveRoom liveRoom, DateTimeOffset occurredAt, CancellationToken cancellationToken = default)
{
var settings = await _settingsService.GetAsync(cancellationToken);
var environment = BuildLiveRoomEnvironment(liveRoom, occurredAt);
environment["LIVE_RECORDER_EVENT"] = "live_ended";
await RunAsync(
settings.EnableEventScripts,
settings.LiveEndedScriptPath,
settings.EventScriptTimeoutSeconds,
"live_ended",
environment,
liveRoom.Id,
recordSessionId: null,
recordTaskId: null,
cancellationToken);
}
public async Task RunSegmentCompletedAsync(
LiveRoom? liveRoom,
RecordSession recordSession,
RecordTask recordTask,
RecordResult? recordResult,
string segmentFilePath,
DateTimeOffset occurredAt,
CancellationToken cancellationToken = default)
{
var settings = await _settingsService.GetAsync(cancellationToken);
var environment = BuildLiveRoomEnvironment(liveRoom, occurredAt);
environment["LIVE_RECORDER_EVENT"] = "segment_completed";
environment["LIVE_RECORDER_RECORD_SESSION_ID"] = recordSession.Id.ToString();
environment["LIVE_RECORDER_RECORD_TASK_ID"] = recordTask.Id.ToString();
environment["LIVE_RECORDER_SEGMENT_INDEX"] = recordTask.SegmentIndex.ToString();
environment["LIVE_RECORDER_SEGMENT_FILE_PATH"] = NormalizePath(segmentFilePath);
environment["LIVE_RECORDER_DANMAKU_FILE_PATH"] = NormalizePath(recordResult?.DanmakuFilePath);
environment["LIVE_RECORDER_DURATION_SECONDS"] = recordResult?.DurationSeconds?.ToString("0.###") ?? recordTask.DurationSeconds?.ToString("0.###") ?? string.Empty;
environment["LIVE_RECORDER_FILE_SIZE_BYTES"] = recordResult?.FileSizeBytes?.ToString() ?? TryGetFileSize(segmentFilePath);
environment["LIVE_RECORDER_TASK_STATUS"] = recordTask.Status.ToString();
environment["LIVE_RECORDER_SESSION_STATUS"] = recordSession.Status.ToString();
await RunAsync(
settings.EnableEventScripts,
settings.SegmentCompletedScriptPath,
settings.EventScriptTimeoutSeconds,
"segment_completed",
environment,
liveRoom?.Id ?? recordSession.LiveRoomId,
recordSession.Id,
recordTask.Id,
cancellationToken);
}
private async Task RunAsync(
bool enabled,
string scriptPath,
int timeoutSeconds,
string eventName,
IReadOnlyDictionary<string, string> environment,
Guid? liveRoomId,
Guid? recordSessionId,
Guid? recordTaskId,
CancellationToken cancellationToken)
{
if (!enabled || string.IsNullOrWhiteSpace(scriptPath))
{
return;
}
var normalizedScriptPath = NormalizePath(scriptPath);
if (!File.Exists(normalizedScriptPath))
{
await _systemLogService.WriteAsync(
SystemLogLevel.Warning,
"Script",
$"Event script was not found for {eventName}.",
normalizedScriptPath,
liveRoomId,
recordSessionId,
recordTaskId,
cancellationToken);
return;
}
var startInfo = CreateStartInfo(normalizedScriptPath);
foreach (var pair in environment)
{
startInfo.Environment[pair.Key] = pair.Value;
}
using var process = new Process
{
StartInfo = startInfo,
EnableRaisingEvents = true
};
try
{
process.Start();
using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
timeoutCts.CancelAfter(TimeSpan.FromSeconds(Math.Clamp(timeoutSeconds, 1, 3600)));
await process.WaitForExitAsync(timeoutCts.Token);
await _systemLogService.WriteAsync(
process.ExitCode == 0 ? SystemLogLevel.Info : SystemLogLevel.Warning,
"Script",
process.ExitCode == 0
? $"Event script completed for {eventName}."
: $"Event script exited with code {process.ExitCode} for {eventName}.",
normalizedScriptPath,
liveRoomId,
recordSessionId,
recordTaskId,
cancellationToken);
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
{
TryKill(process);
await _systemLogService.WriteAsync(
SystemLogLevel.Warning,
"Script",
$"Event script timed out for {eventName}.",
normalizedScriptPath,
liveRoomId,
recordSessionId,
recordTaskId,
CancellationToken.None);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Event script failed for {EventName}", eventName);
await _systemLogService.WriteAsync(
SystemLogLevel.Warning,
"Script",
$"Event script failed for {eventName}.",
ex.ToString(),
liveRoomId,
recordSessionId,
recordTaskId,
cancellationToken);
}
}
private static Dictionary<string, string> BuildLiveRoomEnvironment(LiveRoom? liveRoom, DateTimeOffset occurredAt)
{
return new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["LIVE_RECORDER_EVENT"] = string.Empty,
["LIVE_RECORDER_PLATFORM"] = liveRoom?.Platform.ToString() ?? string.Empty,
["LIVE_RECORDER_LIVE_ROOM_ID"] = liveRoom?.Id.ToString() ?? string.Empty,
["LIVE_RECORDER_ROOM_ID"] = liveRoom?.RoomId ?? string.Empty,
["LIVE_RECORDER_TITLE"] = liveRoom?.Title ?? string.Empty,
["LIVE_RECORDER_ANCHOR"] = liveRoom?.AnchorName ?? string.Empty,
["LIVE_RECORDER_SOURCE_URL"] = liveRoom?.SourceUrl ?? liveRoom?.NormalizedUrl ?? string.Empty,
["LIVE_RECORDER_OCCURRED_AT_UTC"] = occurredAt.ToString("O")
};
}
private static ProcessStartInfo CreateStartInfo(string scriptPath)
{
var extension = Path.GetExtension(scriptPath);
var startInfo = new ProcessStartInfo
{
UseShellExecute = false,
RedirectStandardError = false,
RedirectStandardOutput = false,
CreateNoWindow = true,
WorkingDirectory = Path.GetDirectoryName(scriptPath) ?? AppContext.BaseDirectory
};
if (OperatingSystem.IsWindows() && extension.Equals(".ps1", StringComparison.OrdinalIgnoreCase))
{
startInfo.FileName = "powershell";
startInfo.ArgumentList.Add("-NoProfile");
startInfo.ArgumentList.Add("-ExecutionPolicy");
startInfo.ArgumentList.Add("Bypass");
startInfo.ArgumentList.Add("-File");
startInfo.ArgumentList.Add(scriptPath);
return startInfo;
}
if (OperatingSystem.IsWindows() && (extension.Equals(".bat", StringComparison.OrdinalIgnoreCase) || extension.Equals(".cmd", StringComparison.OrdinalIgnoreCase)))
{
startInfo.FileName = "cmd";
startInfo.ArgumentList.Add("/c");
startInfo.ArgumentList.Add(scriptPath);
return startInfo;
}
if (!OperatingSystem.IsWindows() && extension.Equals(".sh", StringComparison.OrdinalIgnoreCase))
{
startInfo.FileName = "/bin/sh";
startInfo.ArgumentList.Add(scriptPath);
return startInfo;
}
startInfo.FileName = scriptPath;
return startInfo;
}
private static string NormalizePath(string? path)
{
if (string.IsNullOrWhiteSpace(path))
{
return string.Empty;
}
return Path.IsPathRooted(path)
? path
: Path.GetFullPath(path, AppContext.BaseDirectory);
}
private static string TryGetFileSize(string? path)
{
var normalizedPath = NormalizePath(path);
if (string.IsNullOrWhiteSpace(normalizedPath) || !File.Exists(normalizedPath))
{
return string.Empty;
}
return new FileInfo(normalizedPath).Length.ToString();
}
private static void TryKill(Process process)
{
try
{
if (!process.HasExited)
{
process.Kill(true);
}
}
catch
{
}
}
}
File diff suppressed because it is too large Load Diff
@@ -1,18 +1,32 @@
using System.Diagnostics;
using System.Diagnostics;
using System.Globalization;
using System.Text;
using LiveRecorder.Application.Abstractions.Platforms;
using LiveRecorder.Domain.Entities;
using LiveRecorder.Domain.Enums;
using LiveRecorder.Infrastructure.Persistence;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
namespace LiveRecorder.Infrastructure.Services;
public sealed partial class FfmpegService
{
private static async Task<(string OutputPath, string? ErrorMessage)> TryFinalizeMp4Async(
private const string LowStoragePauseErrorPrefix = "MP4 finalization paused because storage is below threshold.";
private static readonly TimeSpan Mp4FinalizeInactivityTimeout = TimeSpan.FromMinutes(10);
private static readonly TimeSpan Mp4FinalizePollInterval = TimeSpan.FromSeconds(1);
private async Task<(string OutputPath, string? ErrorMessage)> TryFinalizeMp4Async(
string ffmpegPath,
int maxConcurrentTranscodeTasks,
int mp4FinalizeTimeoutMinutes,
Guid recordSessionId,
Guid recordTaskId,
string sourcePath,
string targetPath)
string targetPath,
double? expectedDurationSeconds,
CancellationToken cancellationToken)
{
if (!File.Exists(sourcePath))
{
@@ -28,24 +42,286 @@ public sealed partial class FfmpegService
File.Delete(tempPath);
}
var remuxProcess = new Process
async Task<string?> GetLowStoragePauseMessageAsync()
{
StartInfo = new ProcessStartInfo
using var storageScope = _serviceScopeFactory.CreateScope();
var settingsService = storageScope.ServiceProvider.GetRequiredService<LiveRecorder.Application.Abstractions.Settings.ISystemSettingsService>();
var settings = await settingsService.GetAsync(cancellationToken);
var sourceSizeBytes = new FileInfo(sourcePath).Length;
var storageCheck = _storageGuardService.CheckCanStartOrResume(settings, sourceSizeBytes);
return storageCheck.HasEnoughSpace
? null
: $"{LowStoragePauseErrorPrefix} {storageCheck.Message}";
}
var lowStoragePauseMessage = await GetLowStoragePauseMessageAsync();
if (!string.IsNullOrWhiteSpace(lowStoragePauseMessage))
{
SetPostProcessState(recordSessionId, recordTaskId, "Paused: Low storage", null, lowStoragePauseMessage);
return (sourcePath, lowStoragePauseMessage);
}
SetPostProcessState(
recordSessionId,
recordTaskId,
"Queued",
null,
$"Waiting for an available ffmpeg transcode slot (max {Math.Clamp(maxConcurrentTranscodeTasks, 1, 16)}).");
using var transcodeSlot = await AcquireTranscodeSlotAsync(maxConcurrentTranscodeTasks, cancellationToken);
lowStoragePauseMessage = await GetLowStoragePauseMessageAsync();
if (!string.IsNullOrWhiteSpace(lowStoragePauseMessage))
{
SetPostProcessState(recordSessionId, recordTaskId, "Paused: Low storage", null, lowStoragePauseMessage);
return (sourcePath, lowStoragePauseMessage);
}
var stderrLines = new Queue<string>();
var progressSync = new object();
double? processedSeconds = null;
var lastReportedWholePercent = -1;
var attemptStartedAt = DateTimeOffset.UtcNow;
var activeStage = "Finalizing MP4";
var maxDuration = TimeSpan.FromMinutes(Math.Clamp(mp4FinalizeTimeoutMinutes, 1, 1440));
long lastActivityTicks = attemptStartedAt.UtcTicks;
void TouchActivity()
{
System.Threading.Interlocked.Exchange(ref lastActivityTicks, DateTimeOffset.UtcNow.UtcTicks);
}
void ReportProgress(double? seconds, string? stageOverride = null)
{
lock (progressSync)
{
FileName = ffmpegPath,
Arguments = $"-hide_banner -y -i {Quote(sourcePath)} -c copy -movflags +faststart {Quote(tempPath)}",
UseShellExecute = false,
RedirectStandardError = true,
RedirectStandardOutput = true,
CreateNoWindow = true
TouchActivity();
if (seconds.HasValue)
{
processedSeconds = seconds;
}
double? progressPercent = null;
string? detail = null;
if (expectedDurationSeconds.HasValue && expectedDurationSeconds.Value > 0 && processedSeconds.HasValue)
{
progressPercent = Math.Clamp(processedSeconds.Value / expectedDurationSeconds.Value * 100d, 0d, 99d);
detail = $"Processed {processedSeconds.Value:F1}s / {expectedDurationSeconds.Value:F1}s";
}
else if (processedSeconds.HasValue)
{
detail = $"Processed {processedSeconds.Value:F1}s";
}
var wholePercent = progressPercent.HasValue ? (int)Math.Floor(progressPercent.Value) : -1;
if (stageOverride is null && wholePercent == lastReportedWholePercent)
{
return;
}
lastReportedWholePercent = wholePercent;
SetPostProcessState(
recordSessionId,
recordTaskId,
stageOverride ?? activeStage,
progressPercent,
detail ?? $"Optimizing MP4 index for {Path.GetFileName(targetPath)}");
}
};
}
remuxProcess.Start();
await remuxProcess.WaitForExitAsync();
if (remuxProcess.ExitCode == 0 && File.Exists(tempPath))
string? GetErrorDetail()
{
lock (stderrLines)
{
return stderrLines.Count == 0 ? null : string.Join(" | ", stderrLines);
}
}
void ClearErrorDetail()
{
lock (stderrLines)
{
stderrLines.Clear();
}
}
async Task<string?> RunFinalizeAttemptAsync(
Mp4FinalizeStrategy strategy,
string stage,
string detail)
{
if (File.Exists(tempPath))
{
File.Delete(tempPath);
}
ClearErrorDetail();
activeStage = stage;
processedSeconds = null;
lastReportedWholePercent = -1;
attemptStartedAt = DateTimeOffset.UtcNow;
System.Threading.Interlocked.Exchange(ref lastActivityTicks, attemptStartedAt.UtcTicks);
SetPostProcessState(recordSessionId, recordTaskId, stage, 0, detail);
using var finalizeProcess = new Process
{
StartInfo = new ProcessStartInfo
{
FileName = ffmpegPath,
UseShellExecute = false,
RedirectStandardError = true,
RedirectStandardOutput = true,
CreateNoWindow = true
}
};
foreach (var argument in BuildMp4FinalizeArgumentList(sourcePath, tempPath, strategy))
{
finalizeProcess.StartInfo.ArgumentList.Add(argument);
}
finalizeProcess.OutputDataReceived += (_, args) =>
{
if (string.IsNullOrWhiteSpace(args.Data))
{
return;
}
TouchActivity();
if (TryParseFfmpegProgressSeconds(args.Data, out var seconds))
{
ReportProgress(seconds);
return;
}
if (args.Data.StartsWith("progress=", StringComparison.OrdinalIgnoreCase))
{
ReportProgress(processedSeconds);
}
};
finalizeProcess.ErrorDataReceived += (_, args) =>
{
if (string.IsNullOrWhiteSpace(args.Data))
{
return;
}
TouchActivity();
_logger.LogDebug("ffmpeg-postprocess[{RecordTaskId}] {Line}", recordTaskId, args.Data);
lock (stderrLines)
{
stderrLines.Enqueue(args.Data.Trim());
while (stderrLines.Count > 10)
{
stderrLines.Dequeue();
}
}
};
try
{
finalizeProcess.Start();
finalizeProcess.BeginOutputReadLine();
finalizeProcess.BeginErrorReadLine();
while (!finalizeProcess.HasExited)
{
cancellationToken.ThrowIfCancellationRequested();
var now = DateTimeOffset.UtcNow;
var lastActivityAt = new DateTimeOffset(System.Threading.Interlocked.Read(ref lastActivityTicks), TimeSpan.Zero);
if (now - attemptStartedAt > maxDuration)
{
throw new TimeoutException($"MP4 finalization exceeded the maximum allowed duration of {maxDuration.TotalMinutes:F0} minutes.");
}
if (now - lastActivityAt > Mp4FinalizeInactivityTimeout)
{
throw new TimeoutException($"MP4 finalization did not report progress for more than {Mp4FinalizeInactivityTimeout.TotalMinutes:F0} minutes.");
}
await Task.Delay(Mp4FinalizePollInterval, cancellationToken);
}
await finalizeProcess.WaitForExitAsync(cancellationToken);
}
catch (TimeoutException ex)
{
try
{
if (!finalizeProcess.HasExited)
{
finalizeProcess.Kill(true);
}
}
catch (Exception killEx)
{
_logger.LogWarning(killEx, "Timed-out MP4 finalization process could not be killed for task {RecordTaskId}", recordTaskId);
}
return ex.Message;
}
catch (Exception ex)
{
try
{
if (!finalizeProcess.HasExited)
{
finalizeProcess.Kill(true);
}
}
catch (Exception killEx)
{
_logger.LogWarning(killEx, "Failed MP4 finalization process could not be killed for task {RecordTaskId}", recordTaskId);
}
return ex.Message;
}
if (finalizeProcess.ExitCode == 0 && File.Exists(tempPath))
{
return null;
}
if (File.Exists(tempPath))
{
File.Delete(tempPath);
}
return GetErrorDetail() ?? $"ffmpeg exited with code {finalizeProcess.ExitCode}.";
}
var finalizationError = await RunFinalizeAttemptAsync(
Mp4FinalizeStrategy.StreamCopy,
"Finalizing MP4",
$"Optimizing MP4 index for {Path.GetFileName(targetPath)}");
if (IsNoSpaceLeftError(finalizationError))
{
finalizationError = $"{LowStoragePauseErrorPrefix} {finalizationError}";
}
if (!string.IsNullOrWhiteSpace(finalizationError) && IsRepairableMp4FinalizeError(finalizationError))
{
var repairError = await RunFinalizeAttemptAsync(
Mp4FinalizeStrategy.RepairTranscode,
"Repairing MP4",
$"Repairing stream metadata for {Path.GetFileName(targetPath)}");
finalizationError = string.IsNullOrWhiteSpace(repairError)
? null
: IsNoSpaceLeftError(repairError)
? $"{LowStoragePauseErrorPrefix} {repairError}"
: $"{finalizationError} | fallback repair failed: {repairError}";
}
if (string.IsNullOrWhiteSpace(finalizationError) && File.Exists(tempPath))
{
ReportProgress(expectedDurationSeconds, "Writing MP4 index");
if (File.Exists(targetPath))
{
File.Replace(tempPath, targetPath, null, ignoreMetadataErrors: true);
@@ -60,6 +336,7 @@ public sealed partial class FfmpegService
File.Delete(sourcePath);
}
SetPostProcessState(recordSessionId, recordTaskId, "Completed", 100, "MP4 seek index is ready");
return (targetPath, null);
}
@@ -69,7 +346,285 @@ public sealed partial class FfmpegService
}
var fallbackPath = File.Exists(targetPath) ? targetPath : sourcePath;
return (fallbackPath, "The MP4 file could not be finalized into a seekable output.");
string? errorDetail;
lock (stderrLines)
{
errorDetail = stderrLines.Count == 0 ? null : string.Join(" | ", stderrLines);
}
return (
fallbackPath,
string.IsNullOrWhiteSpace(finalizationError ?? errorDetail)
? "The MP4 file could not be finalized into a seekable output."
: $"The MP4 file could not be finalized into a seekable output. {finalizationError ?? errorDetail}");
}
private async Task<(string OutputPath, string? ErrorMessage)> TryFinalizeTaskOutputAsync(
string ffmpegPath,
int maxConcurrentTranscodeTasks,
int mp4FinalizeTimeoutMinutes,
RecordSession recordSession,
RecordTask recordTask,
double? expectedDurationSeconds,
CancellationToken cancellationToken = default)
=> await TryFinalizeTaskOutputAsync(
ffmpegPath,
maxConcurrentTranscodeTasks,
mp4FinalizeTimeoutMinutes,
recordSession,
recordTask,
expectedDurationSeconds,
recorderSegmentPaths: null,
cancellationToken);
private async Task<(string OutputPath, string? ErrorMessage)> TryFinalizeTaskOutputAsync(
string ffmpegPath,
int maxConcurrentTranscodeTasks,
int mp4FinalizeTimeoutMinutes,
RecordSession recordSession,
RecordTask recordTask,
double? expectedDurationSeconds,
IReadOnlyList<string>? recorderSegmentPaths,
CancellationToken cancellationToken = default)
{
var fallbackOutputPath = recordTask.OutputFilePath ?? string.Empty;
if (recordSession.OutputFormat != RecordOutputFormat.Mp4)
{
return (fallbackOutputPath, null);
}
if (recordSession.SaveMode == RecordSaveMode.SingleFile)
{
if (string.IsNullOrWhiteSpace(recordSession.OutputPathPattern))
{
return (fallbackOutputPath, null);
}
var finalOutputPath = NormalizeAbsolutePath(recordSession.OutputPathPattern);
var recorderOutputPath = GetRecorderOutputPath(finalOutputPath, recordSession.OutputFormat, recordSession.SaveMode);
if (!File.Exists(recorderOutputPath))
{
return (File.Exists(finalOutputPath) ? finalOutputPath : fallbackOutputPath, null);
}
return await TryFinalizeMp4Async(
ffmpegPath,
maxConcurrentTranscodeTasks,
mp4FinalizeTimeoutMinutes,
recordSession.Id,
recordTask.Id,
recorderOutputPath,
finalOutputPath,
expectedDurationSeconds,
cancellationToken);
}
if (string.IsNullOrWhiteSpace(recordTask.OutputFilePath))
{
return (fallbackOutputPath, null);
}
var finalSegmentOutputPath = NormalizeAbsolutePath(recordTask.OutputFilePath);
var normalizedRecorderSegmentPaths = recorderSegmentPaths?
.Where(static item => !string.IsNullOrWhiteSpace(item))
.Select(NormalizeAbsolutePath)
.Distinct(StringComparer.OrdinalIgnoreCase)
.Where(File.Exists)
.ToArray();
if (normalizedRecorderSegmentPaths is null || normalizedRecorderSegmentPaths.Length == 0)
{
normalizedRecorderSegmentPaths =
[
NormalizeAbsolutePath(
GetRecorderOutputPath(finalSegmentOutputPath, recordSession.OutputFormat, recordSession.SaveMode))
];
}
normalizedRecorderSegmentPaths = normalizedRecorderSegmentPaths
.Where(File.Exists)
.ToArray();
if (normalizedRecorderSegmentPaths.Length == 0)
{
return (File.Exists(finalSegmentOutputPath) ? finalSegmentOutputPath : recordTask.OutputFilePath, null);
}
string? materializedSourcePath = null;
var cleanupMaterializedSource = false;
var preserveMaterializedSource = false;
try
{
materializedSourcePath = await MaterializeRecorderSegmentSourceAsync(
finalSegmentOutputPath,
normalizedRecorderSegmentPaths,
cancellationToken);
if (string.IsNullOrWhiteSpace(materializedSourcePath))
{
return (File.Exists(finalSegmentOutputPath) ? finalSegmentOutputPath : recordTask.OutputFilePath, null);
}
cleanupMaterializedSource =
!string.Equals(materializedSourcePath, normalizedRecorderSegmentPaths[0], StringComparison.OrdinalIgnoreCase);
var finalizationResult = await TryFinalizeMp4Async(
ffmpegPath,
maxConcurrentTranscodeTasks,
mp4FinalizeTimeoutMinutes,
recordSession.Id,
recordTask.Id,
materializedSourcePath,
finalSegmentOutputPath,
expectedDurationSeconds,
cancellationToken);
preserveMaterializedSource = IsLowStoragePauseError(finalizationResult.ErrorMessage);
return finalizationResult;
}
finally
{
if (cleanupMaterializedSource &&
!preserveMaterializedSource &&
!string.IsNullOrWhiteSpace(materializedSourcePath) &&
File.Exists(materializedSourcePath))
{
File.Delete(materializedSourcePath);
}
}
}
private static async Task<string?> MaterializeRecorderSegmentSourceAsync(
string finalOutputPath,
IReadOnlyList<string> recorderSegmentPaths,
CancellationToken cancellationToken)
{
if (recorderSegmentPaths.Count == 0)
{
return null;
}
if (recorderSegmentPaths.Count == 1)
{
return recorderSegmentPaths[0];
}
var combinedPath = Path.Combine(
Path.GetDirectoryName(finalOutputPath)!,
$"{Path.GetFileNameWithoutExtension(finalOutputPath)}.concat.ts");
if (File.Exists(combinedPath))
{
File.Delete(combinedPath);
}
await using var outputStream = new FileStream(
combinedPath,
FileMode.CreateNew,
FileAccess.Write,
FileShare.None,
bufferSize: 1024 * 128,
useAsync: true);
foreach (var path in recorderSegmentPaths)
{
await using var inputStream = new FileStream(
path,
FileMode.Open,
FileAccess.Read,
FileShare.ReadWrite,
bufferSize: 1024 * 128,
useAsync: true);
await inputStream.CopyToAsync(outputStream, 1024 * 128, cancellationToken);
}
await outputStream.FlushAsync(cancellationToken);
return combinedPath;
}
private static IReadOnlyList<string> BuildMp4FinalizeArgumentList(
string sourcePath,
string targetPath,
Mp4FinalizeStrategy strategy)
{
var arguments = new List<string>
{
"-hide_banner",
"-y",
"-nostats",
"-progress",
"pipe:1",
"-analyzeduration",
"100M",
"-probesize",
"100M",
"-fflags",
"+genpts+igndts+discardcorrupt",
"-err_detect",
"ignore_err",
"-i",
sourcePath,
"-map",
"0:v:0",
"-map",
"0:a:0?",
"-dn",
"-sn"
};
if (strategy == Mp4FinalizeStrategy.RepairTranscode)
{
arguments.AddRange(
[
"-c:v", "libx264",
"-preset", "veryfast",
"-crf", "23",
"-c:a", "aac",
"-b:a", "128k"
]);
}
else
{
arguments.AddRange(["-c", "copy"]);
}
arguments.AddRange(
[
"-movflags",
"+faststart",
"-avoid_negative_ts",
"make_zero",
targetPath
]);
return arguments;
}
private static bool IsRepairableMp4FinalizeError(string errorDetail)
{
if (IsLowStoragePauseError(errorDetail))
{
return false;
}
return errorDetail.Contains("dimensions not set", StringComparison.OrdinalIgnoreCase) ||
errorDetail.Contains("Could not write header", StringComparison.OrdinalIgnoreCase) ||
errorDetail.Contains("incorrect codec parameters", StringComparison.OrdinalIgnoreCase) ||
errorDetail.Contains("codec parameters", StringComparison.OrdinalIgnoreCase) ||
errorDetail.Contains("Invalid data found when processing input", StringComparison.OrdinalIgnoreCase) ||
errorDetail.Contains("Error initializing output stream", StringComparison.OrdinalIgnoreCase);
}
private static bool IsLowStoragePauseError(string? errorDetail) =>
!string.IsNullOrWhiteSpace(errorDetail) &&
errorDetail.Contains(LowStoragePauseErrorPrefix, StringComparison.OrdinalIgnoreCase);
private static bool IsNoSpaceLeftError(string? errorDetail) =>
!string.IsNullOrWhiteSpace(errorDetail) &&
(errorDetail.Contains("No space left on device", StringComparison.OrdinalIgnoreCase) ||
errorDetail.Contains("database or disk is full", StringComparison.OrdinalIgnoreCase));
private enum Mp4FinalizeStrategy
{
StreamCopy,
RepairTranscode
}
private static IReadOnlyList<string> BuildArgumentList(
@@ -83,6 +638,8 @@ public sealed partial class FfmpegService
int readWriteTimeoutMilliseconds,
int segmentDurationMinutes,
StreamInputHeaders? inputHeaders,
string? selectedProtocol,
string? selectedVideoCodec,
FfmpegInputOptionProfile inputOptionProfile)
{
var arguments = new List<string> { "-hide_banner", "-y" };
@@ -107,7 +664,9 @@ public sealed partial class FfmpegService
}
}
if (enableReconnect && inputOptionProfile == FfmpegInputOptionProfile.Baseline)
if (enableReconnect &&
inputOptionProfile == FfmpegInputOptionProfile.Baseline &&
ShouldEnableReconnect(streamUrl, selectedProtocol))
{
arguments.AddRange(
[
@@ -121,19 +680,30 @@ public sealed partial class FfmpegService
arguments.AddRange(["-rw_timeout", readWriteTimeoutMilliseconds.ToString(), "-fflags", "+discardcorrupt+genpts", "-i", streamUrl]);
arguments.AddRange(BuildCodecArguments(recordingTemplate));
var writesTransportStream = useIntermediateTransportStream || outputFormat == RecordOutputFormat.Ts;
var bitstreamFilter = ResolveTransportStreamBitstreamFilter(recordingTemplate, writesTransportStream, selectedVideoCodec);
if (!string.IsNullOrWhiteSpace(bitstreamFilter))
{
arguments.AddRange(["-bsf:v", bitstreamFilter]);
}
if (saveMode == RecordSaveMode.Segmented)
{
var segmentFormat = writesTransportStream
? "mpegts"
: "mp4";
arguments.AddRange(
[
"-f", "segment",
"-segment_start_number", "1",
"-segment_time", Math.Max(60, segmentDurationMinutes * 60).ToString(),
"-break_non_keyframes", "0",
"-reset_timestamps", "1",
"-strftime", "0",
"-segment_format", outputFormat == RecordOutputFormat.Ts ? "mpegts" : "mp4"
"-segment_format", segmentFormat
]);
if (outputFormat == RecordOutputFormat.Mp4)
if (!useIntermediateTransportStream && outputFormat == RecordOutputFormat.Mp4)
{
arguments.AddRange(["-segment_format_options", $"movflags={BuildSegmentedMp4MovFlags()}"]);
}
@@ -173,6 +743,35 @@ public sealed partial class FfmpegService
]
};
private static string? ResolveTransportStreamBitstreamFilter(
RecordingTemplateType recordingTemplate,
bool writesTransportStream,
string? selectedVideoCodec)
{
if (!writesTransportStream || !UsesStreamCopy(recordingTemplate) || string.IsNullOrWhiteSpace(selectedVideoCodec))
{
return null;
}
var normalizedCodec = selectedVideoCodec.Trim();
if (normalizedCodec.Contains("h264", StringComparison.OrdinalIgnoreCase) ||
normalizedCodec.Contains("avc", StringComparison.OrdinalIgnoreCase))
{
return "h264_mp4toannexb";
}
if (normalizedCodec.Contains("hevc", StringComparison.OrdinalIgnoreCase) ||
normalizedCodec.Contains("h265", StringComparison.OrdinalIgnoreCase))
{
return "hevc_mp4toannexb";
}
return null;
}
private static bool UsesStreamCopy(RecordingTemplateType recordingTemplate) =>
recordingTemplate is RecordingTemplateType.StreamCopy or RecordingTemplateType.ArchiveTs;
private static long? CalculateFileSize(string? outputPath)
{
if (string.IsNullOrWhiteSpace(outputPath))
@@ -215,7 +814,7 @@ public sealed partial class FfmpegService
return false;
}
private static void UpsertRecordResult(
private static async Task UpsertRecordResultAsync(
RecordTask recordTask,
LiveRecorderDbContext dbContext,
string? effectiveOutputPath,
@@ -223,71 +822,122 @@ public sealed partial class FfmpegService
double? durationSeconds,
string? danmakuFilePath,
int danmakuMessageCount,
DateTimeOffset endedAt)
DateTimeOffset endedAt,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(effectiveOutputPath))
{
effectiveOutputPath = recordTask.OutputFilePath ?? string.Empty;
}
if (recordTask.Result is null)
{
dbContext.RecordResults.Add(new RecordResult(
recordTask.Id,
effectiveOutputPath,
fileSize,
durationSeconds,
danmakuFilePath,
danmakuMessageCount,
recordTask.Status,
recordTask.ErrorMessage,
endedAt));
return;
}
var resultId = Guid.NewGuid();
var normalizedDanmakuCount = Math.Max(0, danmakuMessageCount);
var finalStatus = (int)recordTask.Status;
recordTask.Result.Update(
effectiveOutputPath,
fileSize,
durationSeconds,
danmakuFilePath,
danmakuMessageCount,
recordTask.Status,
recordTask.ErrorMessage);
// Multiple background paths can reconcile the same segment after ffmpeg exits.
// Use SQLite's atomic upsert instead of EF Add-or-Update to avoid RecordTaskId
// unique constraint races between scoped DbContext instances.
await dbContext.Database.ExecuteSqlInterpolatedAsync($"""
INSERT INTO RecordResults
(Id, RecordTaskId, FilePath, FileSizeBytes, DurationSeconds, DanmakuFilePath, DanmakuMessageCount, FinalStatus, ErrorMessage, CreatedAt)
VALUES
({resultId}, {recordTask.Id}, {effectiveOutputPath}, {fileSize}, {durationSeconds}, {danmakuFilePath}, {normalizedDanmakuCount}, {finalStatus}, {recordTask.ErrorMessage}, {endedAt})
ON CONFLICT(RecordTaskId) DO UPDATE SET
FilePath = excluded.FilePath,
FileSizeBytes = excluded.FileSizeBytes,
DurationSeconds = excluded.DurationSeconds,
DanmakuFilePath = excluded.DanmakuFilePath,
DanmakuMessageCount = excluded.DanmakuMessageCount,
FinalStatus = excluded.FinalStatus,
ErrorMessage = excluded.ErrorMessage;
""", cancellationToken);
}
private static Task<RecordResult?> LoadRecordResultAsync(
LiveRecorderDbContext dbContext,
Guid recordTaskId,
CancellationToken cancellationToken = default) =>
dbContext.RecordResults
.AsNoTracking()
.FirstOrDefaultAsync(item => item.RecordTaskId == recordTaskId, cancellationToken);
private static string BuildSingleFileMp4MovFlags() =>
"+faststart+frag_keyframe+empty_moov+default_base_moof";
private static string BuildSegmentedMp4MovFlags() =>
"+faststart+frag_keyframe+empty_moov+default_base_moof";
"+faststart";
private static string GetRecorderOutputPath(
string finalOutputPath,
RecordOutputFormat outputFormat,
RecordSaveMode saveMode)
{
if (saveMode == RecordSaveMode.SingleFile && outputFormat == RecordOutputFormat.Mp4)
if (outputFormat != RecordOutputFormat.Mp4)
{
return finalOutputPath;
}
if (saveMode == RecordSaveMode.SingleFile)
{
return Path.Combine(
Path.GetDirectoryName(finalOutputPath)!,
$"{Path.GetFileNameWithoutExtension(finalOutputPath)}.recording.ts");
}
return finalOutputPath;
return Path.ChangeExtension(finalOutputPath, ".ts");
}
private static bool ShouldUseIntermediateTransportStream(
string outputPath,
RecordOutputFormat outputFormat,
RecordSaveMode saveMode) =>
saveMode == RecordSaveMode.SingleFile &&
outputFormat == RecordOutputFormat.Mp4 &&
(saveMode == RecordSaveMode.SingleFile || saveMode == RecordSaveMode.Segmented) &&
outputPath.EndsWith(".ts", StringComparison.OrdinalIgnoreCase);
private static string ResolveSegmentOutputPath(
string outputPathPattern,
RecordSaveMode saveMode,
int segmentIndex)
{
if (saveMode != RecordSaveMode.Segmented)
{
return outputPathPattern;
}
return outputPathPattern.Replace("%05d", $"{Math.Max(1, segmentIndex):D5}", StringComparison.OrdinalIgnoreCase);
}
private static string ResolveRecorderSegmentOutputPath(
string outputPathPattern,
RecordOutputFormat outputFormat,
RecordSaveMode saveMode,
int segmentIndex) =>
NormalizeAbsolutePath(GetRecorderOutputPath(
ResolveSegmentOutputPath(outputPathPattern, saveMode, segmentIndex),
outputFormat,
saveMode));
private static bool IsHttpInput(string streamUrl) =>
streamUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ||
streamUrl.StartsWith("https://", StringComparison.OrdinalIgnoreCase);
private static bool ShouldEnableReconnect(string streamUrl, string? selectedProtocol)
{
if (!string.IsNullOrWhiteSpace(selectedProtocol) &&
selectedProtocol.Equals("hls", StringComparison.OrdinalIgnoreCase))
{
return false;
}
return !streamUrl.Contains(".m3u8", StringComparison.OrdinalIgnoreCase);
}
private static string NormalizeAbsolutePath(string path) =>
Path.IsPathRooted(path)
? path
: Path.GetFullPath(path, AppContext.BaseDirectory);
private static string? BuildCustomHeaderArgument(StreamInputHeaders? inputHeaders)
{
if (inputHeaders is null)
@@ -376,13 +1026,95 @@ public sealed partial class FfmpegService
return count;
}
private static async Task WaitForFileToStabilizeAsync(string? path, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(path))
{
return;
}
var absolutePath = NormalizeAbsolutePath(path);
if (!File.Exists(absolutePath))
{
return;
}
using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
timeoutCts.CancelAfter(TimeSpan.FromSeconds(4));
long? previousLength = null;
DateTime previousWriteTimeUtc = default;
var stableChecks = 0;
try
{
while (true)
{
timeoutCts.Token.ThrowIfCancellationRequested();
var fileInfo = new FileInfo(absolutePath);
if (!fileInfo.Exists)
{
return;
}
if (previousLength == fileInfo.Length && previousWriteTimeUtc == fileInfo.LastWriteTimeUtc)
{
stableChecks++;
if (stableChecks >= 2)
{
return;
}
}
else
{
previousLength = fileInfo.Length;
previousWriteTimeUtc = fileInfo.LastWriteTimeUtc;
stableChecks = 0;
}
await Task.Delay(250, timeoutCts.Token);
}
}
catch (OperationCanceledException) when (timeoutCts.IsCancellationRequested)
{
}
}
private static bool IsActiveTaskStatus(RecordTaskStatus status) =>
status is RecordTaskStatus.Starting or RecordTaskStatus.Running or RecordTaskStatus.Stopping;
private static bool IsActiveSessionStatus(RecordSessionStatus status) =>
status is RecordSessionStatus.Starting or RecordSessionStatus.Running or RecordSessionStatus.Stopping;
private static string Quote(string value) => $"\"{value.Replace("\"", "\\\"", StringComparison.Ordinal)}\"";
private static bool TryParseFfmpegProgressSeconds(string line, out double seconds)
{
if (line.StartsWith("out_time=", StringComparison.OrdinalIgnoreCase))
{
if (TimeSpan.TryParse(
line["out_time=".Length..],
CultureInfo.InvariantCulture,
out var timeSpan))
{
seconds = Math.Max(0, timeSpan.TotalSeconds);
return true;
}
}
if (line.StartsWith("out_time_ms=", StringComparison.OrdinalIgnoreCase) ||
line.StartsWith("out_time_us=", StringComparison.OrdinalIgnoreCase))
{
var raw = line[(line.IndexOf('=') + 1)..];
if (long.TryParse(raw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var value))
{
seconds = Math.Max(0, value / 1_000_000d);
return true;
}
}
seconds = 0;
return false;
}
private enum FfmpegInputOptionProfile
{
@@ -6,6 +6,8 @@ using LiveRecorder.Application.Abstractions.Notifications;
using LiveRecorder.Application.Abstractions.Platforms;
using LiveRecorder.Application.Abstractions.Recording;
using LiveRecorder.Application.Abstractions.Settings;
using LiveRecorder.Application.Abstractions.Storage;
using LiveRecorder.Application.Services;
using LiveRecorder.Domain.Entities;
using LiveRecorder.Domain.Enums;
using LiveRecorder.Infrastructure.Persistence;
@@ -22,19 +24,46 @@ public sealed partial class FfmpegService : IFfmpegService
RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private readonly ConcurrentDictionary<Guid, SessionProcessRuntime> _processes = new();
private readonly ConcurrentDictionary<Guid, PostProcessRuntimeEntry> _postProcessStates = new();
private readonly object _transcodeConcurrencyLock = new();
private readonly Queue<TaskCompletionSource<IDisposable>> _transcodeWaiters = new();
private int _activeTranscodeTasks;
private int _maxConcurrentTranscodeTasks = 1;
private readonly IServiceScopeFactory _serviceScopeFactory;
private readonly IStorageGuardService _storageGuardService;
private readonly ILogger<FfmpegService> _logger;
public FfmpegService(
IServiceScopeFactory serviceScopeFactory,
IStorageGuardService storageGuardService,
ILogger<FfmpegService> logger)
{
_serviceScopeFactory = serviceScopeFactory;
_storageGuardService = storageGuardService;
_logger = logger;
}
public bool IsRunning(Guid recordSessionId) => _processes.ContainsKey(recordSessionId);
public IReadOnlyDictionary<Guid, RecordTaskRuntimeState> GetTaskRuntimeStates(IReadOnlyCollection<Guid> recordTaskIds)
{
if (recordTaskIds.Count == 0)
{
return new Dictionary<Guid, RecordTaskRuntimeState>();
}
var snapshot = new Dictionary<Guid, RecordTaskRuntimeState>();
foreach (var taskId in recordTaskIds)
{
if (_postProcessStates.TryGetValue(taskId, out var state))
{
snapshot[taskId] = state.State;
}
}
return snapshot;
}
public Task CompleteAsync(Guid recordSessionId, CancellationToken cancellationToken = default) =>
RequestStopAsync(recordSessionId, markAsCompletedOnExit: true, cancellationToken);
@@ -42,11 +71,13 @@ public sealed partial class FfmpegService : IFfmpegService
RecordSession recordSession,
RecordTask initialTask,
StreamUrlResult streamUrlResult,
RecordingExecutionSettings recordingSettings,
CancellationToken cancellationToken = default) =>
StartInternalAsync(
recordSession,
initialTask,
streamUrlResult,
recordingSettings,
inputOptionProfile: FfmpegInputOptionProfile.Baseline,
hasRetriedWithCompatibilityProfile: false,
hasRetriedWithRefreshedStream: false,
@@ -57,6 +88,7 @@ public sealed partial class FfmpegService : IFfmpegService
RecordSession recordSession,
RecordTask initialTask,
StreamUrlResult streamUrlResult,
RecordingExecutionSettings recordingSettings,
FfmpegInputOptionProfile inputOptionProfile,
bool hasRetriedWithCompatibilityProfile,
bool hasRetriedWithRefreshedStream,
@@ -66,6 +98,7 @@ public sealed partial class FfmpegService : IFfmpegService
ArgumentNullException.ThrowIfNull(recordSession);
ArgumentNullException.ThrowIfNull(initialTask);
ArgumentNullException.ThrowIfNull(streamUrlResult);
ArgumentNullException.ThrowIfNull(recordingSettings);
if (string.IsNullOrWhiteSpace(streamUrlResult.SelectedUrl) || string.IsNullOrWhiteSpace(recordSession.OutputPathPattern))
{
@@ -106,10 +139,12 @@ public sealed partial class FfmpegService : IFfmpegService
recordSession.SaveMode,
initialTask.Id,
Math.Max(1, initialTask.SegmentIndex),
initialTask.OutputFilePath ?? outputPathPattern,
ResolveRecorderSegmentOutputPath(outputPathPattern, recordSession.OutputFormat, recordSession.SaveMode, Math.Max(1, initialTask.SegmentIndex)),
streamUrlResult.SelectedQuality,
streamUrlResult.SelectedProtocol,
streamUrlResult.SelectedVideoCodec,
streamUrlResult.InputHeaders,
recordingSettings,
inputOptionProfile,
hasRetriedWithCompatibilityProfile,
hasRetriedWithRefreshedStream,
@@ -120,12 +155,14 @@ public sealed partial class FfmpegService : IFfmpegService
recorderOutputPath,
recordSession.OutputFormat,
recordSession.SaveMode,
settings.RecordingTemplate,
settings.EnableAutoReconnect,
settings.ReconnectDelayMaxSeconds,
settings.ReadWriteTimeoutMilliseconds,
settings.SegmentDurationMinutes,
recordingSettings.RecordingTemplate,
recordingSettings.EnableAutoReconnect,
recordingSettings.ReconnectDelayMaxSeconds,
recordingSettings.ReadWriteTimeoutMilliseconds,
recordingSettings.SegmentDurationMinutes,
streamUrlResult.InputHeaders,
streamUrlResult.SelectedProtocol,
streamUrlResult.SelectedVideoCodec,
inputOptionProfile))
{
process.StartInfo.ArgumentList.Add(argument);
@@ -191,6 +228,7 @@ public sealed partial class FfmpegService : IFfmpegService
{
if (!process.HasExited)
{
runtime.MarkForceKilled();
process.Kill(true);
}
}
@@ -204,7 +242,7 @@ public sealed partial class FfmpegService : IFfmpegService
public async Task<bool> TryReconcileInactiveSessionAsync(Guid recordSessionId, CancellationToken cancellationToken = default)
{
if (IsRunning(recordSessionId))
if (IsRunning(recordSessionId) || IsSessionUnderPostProcessing(recordSessionId))
{
return false;
}
@@ -232,37 +270,39 @@ public sealed partial class FfmpegService : IFfmpegService
.ToList();
var anyUsableOutput = false;
string? finalizationError = null;
string? sessionFinalizationError = null;
foreach (var task in tasks.Where(item => IsActiveTaskStatus(item.Status)))
{
var effectiveOutputPath = task.OutputFilePath ?? string.Empty;
if (recordSession.SaveMode == RecordSaveMode.SingleFile &&
recordSession.OutputFormat == RecordOutputFormat.Mp4 &&
!string.IsNullOrWhiteSpace(recordSession.OutputPathPattern))
var durationSeconds = task.StartedAt.HasValue
? (double?)Math.Max(0, (endedAt - task.StartedAt.Value).TotalSeconds)
: null;
var finalizationResult = await TryFinalizeTaskOutputAsync(
settings.FfmpegPath,
settings.MaxConcurrentFfmpegTranscodeTasks,
settings.Mp4FinalizeTimeoutMinutes,
recordSession,
task,
durationSeconds,
cancellationToken);
var effectiveOutputPath = finalizationResult.OutputPath;
var taskFinalizationError = finalizationResult.ErrorMessage;
if (!IsLowStoragePauseError(taskFinalizationError))
{
var finalOutputPath = Path.IsPathRooted(recordSession.OutputPathPattern)
? recordSession.OutputPathPattern
: Path.GetFullPath(recordSession.OutputPathPattern, AppContext.BaseDirectory);
var recorderOutputPath = GetRecorderOutputPath(finalOutputPath, recordSession.OutputFormat, recordSession.SaveMode);
if (File.Exists(recorderOutputPath))
{
var finalizationResult = await TryFinalizeMp4Async(settings.FfmpegPath, recorderOutputPath, finalOutputPath);
effectiveOutputPath = finalizationResult.OutputPath;
finalizationError ??= finalizationResult.ErrorMessage;
}
sessionFinalizationError ??= taskFinalizationError;
}
var fileSize = CalculateFileSize(effectiveOutputPath);
var danmakuPath = task.Result?.DanmakuFilePath ?? GuessDanmakuPath(task.OutputFilePath);
var danmakuCount = CountDanmakuMessages(danmakuPath);
var durationSeconds = task.StartedAt.HasValue
? (double?)Math.Max(0, (endedAt - task.StartedAt.Value).TotalSeconds)
: null;
if (!string.IsNullOrWhiteSpace(finalizationError))
if (IsLowStoragePauseError(taskFinalizationError))
{
task.MarkFailed(finalizationError, endedAt);
task.MarkProcessing(taskFinalizationError, endedAt);
}
else if (!string.IsNullOrWhiteSpace(taskFinalizationError))
{
task.MarkFailed(taskFinalizationError, endedAt);
}
else if (HasUsableOutput(effectiveOutputPath, fileSize))
{
@@ -274,13 +314,14 @@ public sealed partial class FfmpegService : IFfmpegService
task.MarkStopped(endedAt, durationSeconds, "Recording process was no longer running when the session was reconciled.");
}
UpsertRecordResult(task, dbContext, effectiveOutputPath, fileSize, durationSeconds, danmakuPath, danmakuCount, endedAt);
await UpsertRecordResultAsync(task, dbContext, effectiveOutputPath, fileSize, durationSeconds, danmakuPath, danmakuCount, endedAt, cancellationToken);
ClearPostProcessState(task.Id);
}
recordSession.SyncSegmentCount(tasks.Count, endedAt);
if (!string.IsNullOrWhiteSpace(finalizationError))
if (!string.IsNullOrWhiteSpace(sessionFinalizationError))
{
recordSession.MarkFailed(finalizationError, endedAt);
recordSession.MarkFailed(sessionFinalizationError, endedAt);
}
else if (anyUsableOutput)
{
@@ -295,6 +336,176 @@ public sealed partial class FfmpegService : IFfmpegService
return true;
}
public async Task<bool> StartManualFinalizeTaskAsync(Guid recordTaskId, CancellationToken cancellationToken = default)
{
if (_postProcessStates.ContainsKey(recordTaskId))
{
return false;
}
using var scope = _serviceScopeFactory.CreateScope();
var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
var recordTask = await dbContext.RecordTasks
.Include(item => item.RecordSession)
.Include(item => item.Result)
.FirstOrDefaultAsync(item => item.Id == recordTaskId, cancellationToken);
if (recordTask?.RecordSession is null)
{
return false;
}
if (recordTask.OutputFormat != RecordOutputFormat.Mp4 ||
IsActiveTaskStatus(recordTask.Status) ||
IsRunning(recordTask.RecordSessionId))
{
return false;
}
if (string.IsNullOrWhiteSpace(recordTask.OutputFilePath))
{
return false;
}
var recorderOutputPath = ResolveManualFinalizeSourcePath(recordTask, recordTask.RecordSession);
if (!File.Exists(recorderOutputPath))
{
return false;
}
SetPostProcessState(
recordTask.RecordSessionId,
recordTask.Id,
"Queued",
null,
$"Manual MP4 finalization queued for {Path.GetFileName(recordTask.OutputFilePath)}");
_ = Task.Run(
async () => await RunManualFinalizeTaskAsync(recordTask.Id),
CancellationToken.None);
return true;
}
public async Task<int> ResumePausedFinalizationsAsync(CancellationToken cancellationToken = default)
{
using var scope = _serviceScopeFactory.CreateScope();
var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
var settingsService = scope.ServiceProvider.GetRequiredService<ISystemSettingsService>();
var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
var settings = await settingsService.GetAsync(cancellationToken);
var storageCheck = _storageGuardService.CheckCanStartOrResume(settings);
if (!storageCheck.HasEnoughSpace)
{
await logService.WriteAsync(
SystemLogLevel.Warning,
"Storage",
"Paused MP4 finalization remains blocked because storage is below threshold.",
storageCheck.Message,
cancellationToken: cancellationToken);
return 0;
}
var candidates = await dbContext.RecordTasks
.Include(item => item.RecordSession)
.Include(item => item.Result)
.Where(item => item.OutputFormat == RecordOutputFormat.Mp4 &&
(item.Status == RecordTaskStatus.Processing ||
item.Status == RecordTaskStatus.Completed))
.OrderBy(static item => item.UpdatedAt)
.Take(100)
.ToListAsync(cancellationToken);
var queuedTaskIds = new List<Guid>();
var repairedInterruptedTasks = 0;
var now = DateTimeOffset.UtcNow;
foreach (var candidate in candidates)
{
if (candidate.Status == RecordTaskStatus.Processing)
{
queuedTaskIds.Add(candidate.Id);
continue;
}
if (!NeedsInterruptedMp4Finalization(candidate))
{
continue;
}
candidate.MarkProcessing("MP4 finalization was interrupted before completion. Re-queued after restart.", now);
queuedTaskIds.Add(candidate.Id);
repairedInterruptedTasks++;
}
if (repairedInterruptedTasks > 0)
{
await dbContext.SaveChangesAsync(cancellationToken);
}
var started = 0;
foreach (var taskId in queuedTaskIds.Take(20))
{
if (await StartManualFinalizeTaskAsync(taskId, cancellationToken))
{
started++;
}
}
if (started > 0)
{
await logService.WriteAsync(
SystemLogLevel.Info,
"Storage",
"Storage is available. Resumed paused MP4 finalization tasks.",
$"count={started}; repairedInterrupted={repairedInterruptedTasks}; {storageCheck.Message}",
cancellationToken: cancellationToken);
}
return started;
}
private static bool NeedsInterruptedMp4Finalization(RecordTask recordTask)
{
if (recordTask.Status != RecordTaskStatus.Completed ||
recordTask.RecordSession is null ||
recordTask.OutputFormat != RecordOutputFormat.Mp4 ||
recordTask.RecordSession.OutputFormat != RecordOutputFormat.Mp4 ||
string.IsNullOrWhiteSpace(recordTask.OutputFilePath))
{
return false;
}
var finalOutputPath = NormalizeAbsolutePath(recordTask.OutputFilePath);
if (HasUsableOutput(finalOutputPath, CalculateFileSize(finalOutputPath)))
{
return false;
}
var recorderOutputPath = ResolveManualFinalizeSourcePath(recordTask, recordTask.RecordSession);
return File.Exists(recorderOutputPath);
}
private static string ResolveManualFinalizeSourcePath(RecordTask recordTask, RecordSession recordSession)
{
var resultPath = recordTask.Result?.FilePath;
if (!string.IsNullOrWhiteSpace(resultPath) &&
resultPath.EndsWith(".ts", StringComparison.OrdinalIgnoreCase))
{
var normalizedResultPath = NormalizeAbsolutePath(resultPath);
if (File.Exists(normalizedResultPath))
{
return normalizedResultPath;
}
}
return NormalizeAbsolutePath(
GetRecorderOutputPath(
recordTask.OutputFilePath ?? resultPath ?? string.Empty,
recordSession.OutputFormat,
recordSession.SaveMode));
}
private async Task PersistStartupProfileAsync(SessionProcessRuntime runtime, CancellationToken cancellationToken)
{
using var scope = _serviceScopeFactory.CreateScope();
@@ -303,7 +514,7 @@ public sealed partial class FfmpegService : IFfmpegService
SystemLogLevel.Info,
"FFmpeg",
$"ffmpeg input profile={runtime.InputOptionProfile}.",
detail: $"stream={runtime.SelectedProtocol}:{runtime.SelectedQuality}; compatibilityRetry={runtime.HasRetriedWithCompatibilityProfile}; refreshRetry={runtime.HasRetriedWithRefreshedStream}",
detail: $"stream={runtime.SelectedProtocol}:{runtime.SelectedQuality}; videoCodec={runtime.SelectedVideoCodec ?? "unknown"}; compatibilityRetry={runtime.HasRetriedWithCompatibilityProfile}; refreshRetry={runtime.HasRetriedWithRefreshedStream}",
liveRoomId: runtime.LiveRoomId,
recordSessionId: runtime.RecordSessionId,
recordTaskId: runtime.CurrentTaskId,
@@ -351,29 +562,107 @@ public sealed partial class FfmpegService : IFfmpegService
await process.StandardInput.WriteLineAsync("q");
await process.StandardInput.FlushAsync();
using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
timeoutCts.CancelAfter(TimeSpan.FromSeconds(12));
try
{
await process.WaitForExitAsync(timeoutCts.Token);
}
catch (OperationCanceledException)
{
if (!process.HasExited)
{
process.Kill(true);
}
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Stop ffmpeg process failed for session {RecordSessionId}", recordSessionId);
}
}
if (!process.HasExited)
private void SetPostProcessState(
Guid recordSessionId,
Guid recordTaskId,
string stage,
double? progressPercent,
string? detail = null)
{
var normalizedProgress = progressPercent.HasValue
? Math.Clamp(progressPercent.Value, 0d, 100d)
: (double?)null;
_postProcessStates[recordTaskId] = new PostProcessRuntimeEntry(
recordSessionId,
new RecordTaskRuntimeState(
RecordTaskStatus.Processing,
stage,
normalizedProgress,
detail));
}
private void ClearPostProcessState(Guid recordTaskId) =>
_postProcessStates.TryRemove(recordTaskId, out _);
private bool IsSessionUnderPostProcessing(Guid recordSessionId) =>
_postProcessStates.Values.Any(entry => entry.RecordSessionId == recordSessionId);
private async Task<IDisposable> AcquireTranscodeSlotAsync(int maxConcurrentTasks, CancellationToken cancellationToken)
{
var normalizedMaxConcurrentTasks = Math.Clamp(maxConcurrentTasks, 1, 16);
TaskCompletionSource<IDisposable> waiter;
lock (_transcodeConcurrencyLock)
{
_maxConcurrentTranscodeTasks = normalizedMaxConcurrentTasks;
if (_transcodeWaiters.Count == 0 && _activeTranscodeTasks < _maxConcurrentTranscodeTasks)
{
process.Kill(true);
_activeTranscodeTasks++;
return new TranscodeSlotLease(this);
}
waiter = new TaskCompletionSource<IDisposable>(TaskCreationOptions.RunContinuationsAsynchronously);
_transcodeWaiters.Enqueue(waiter);
}
using var cancellationRegistration = cancellationToken.Register(
static state => ((TaskCompletionSource<IDisposable>)state!).TrySetCanceled(),
waiter);
return await waiter.Task.ConfigureAwait(false);
}
private void ReleaseTranscodeSlot()
{
lock (_transcodeConcurrencyLock)
{
if (_activeTranscodeTasks > 0)
{
_activeTranscodeTasks--;
}
while (_activeTranscodeTasks < _maxConcurrentTranscodeTasks && _transcodeWaiters.Count > 0)
{
var waiter = _transcodeWaiters.Dequeue();
if (waiter.Task.IsCompleted)
{
continue;
}
_activeTranscodeTasks++;
if (waiter.TrySetResult(new TranscodeSlotLease(this)))
{
return;
}
_activeTranscodeTasks--;
}
}
}
private sealed record PostProcessRuntimeEntry(Guid RecordSessionId, RecordTaskRuntimeState State);
private sealed class TranscodeSlotLease : IDisposable
{
private readonly FfmpegService _owner;
private int _disposed;
public TranscodeSlotLease(FfmpegService owner)
{
_owner = owner;
}
public void Dispose()
{
if (Interlocked.Exchange(ref _disposed, 1) == 0)
{
_owner.ReleaseTranscodeSlot();
}
}
}
@@ -1,12 +1,16 @@
using System.Collections.Concurrent;
using LiveRecorder.Application.Abstractions.Logging;
using LiveRecorder.Application.Abstractions.Notifications;
using LiveRecorder.Application.Abstractions.Platforms;
using LiveRecorder.Application.Abstractions.Recording;
using LiveRecorder.Application.Abstractions.Settings;
using LiveRecorder.Application.Abstractions.Storage;
using LiveRecorder.Application.Models.Settings;
using LiveRecorder.Application.Models.RecordTasks;
using LiveRecorder.Application.Services;
using LiveRecorder.Domain.Enums;
using LiveRecorder.Infrastructure.Persistence;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
@@ -16,8 +20,13 @@ namespace LiveRecorder.Infrastructure.Services;
public sealed class LiveRoomPollingBackgroundService : BackgroundService
{
private static readonly TimeSpan OfflineGracefulStopTimeout = TimeSpan.FromSeconds(20);
private static readonly TimeSpan OfflineForcedStopTimeout = TimeSpan.FromSeconds(8);
private static readonly TimeSpan ExceptionEmailCooldown = TimeSpan.FromHours(6);
private readonly IServiceScopeFactory _serviceScopeFactory;
private readonly ILogger<LiveRoomPollingBackgroundService> _logger;
private readonly ConcurrentDictionary<string, DateTimeOffset> _exceptionEmailSentAt = new(StringComparer.OrdinalIgnoreCase);
public LiveRoomPollingBackgroundService(
IServiceScopeFactory serviceScopeFactory,
@@ -38,9 +47,15 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
using var scope = _serviceScopeFactory.CreateScope();
var settingsService = scope.ServiceProvider.GetRequiredService<ISystemSettingsService>();
var settings = await settingsService.GetAsync(stoppingToken);
var emailNotificationService = scope.ServiceProvider.GetRequiredService<IEmailNotificationService>();
delaySeconds = settings.PollingIntervalSeconds;
var ffmpegService = scope.ServiceProvider.GetRequiredService<IFfmpegService>();
var storageGuardService = scope.ServiceProvider.GetRequiredService<IStorageGuardService>();
if (storageGuardService.CheckCanStartOrResume(settings).HasEnoughSpace)
{
await ffmpegService.ResumePausedFinalizationsAsync(stoppingToken);
}
if (!settings.EnableBackgroundPolling)
{
await DelayAsync(delaySeconds, stoppingToken);
@@ -48,104 +63,23 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
}
var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
var adapterFactory = scope.ServiceProvider.GetRequiredService<ILivePlatformAdapterFactory>();
var ffmpegService = scope.ServiceProvider.GetRequiredService<IFfmpegService>();
var recordService = scope.ServiceProvider.GetRequiredService<RecordService>();
var liveRoomStatusService = scope.ServiceProvider.GetRequiredService<LiveRoomStatusService>();
var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
var liveRooms = (await dbContext.LiveRooms.ToListAsync(stoppingToken))
.Where(static item => item.IsEnabled)
var liveRoomIds = (await dbContext.LiveRooms
.AsNoTracking()
.Where(static item => item.IsEnabled)
.Select(static item => new { item.Id, item.UpdatedAt })
.ToListAsync(stoppingToken))
.OrderBy(static item => item.UpdatedAt)
.Select(static item => item.Id)
.ToList();
foreach (var liveRoom in liveRooms)
foreach (var liveRoomId in liveRoomIds)
{
if (stoppingToken.IsCancellationRequested)
{
break;
}
try
{
var adapter = adapterFactory.GetByPlatform(liveRoom.Platform);
var liveStatus = await adapter.GetLiveStatusAsync(liveRoom.RoomId, stoppingToken);
var now = DateTimeOffset.UtcNow;
await liveRoomStatusService.ApplySnapshotAsync(liveRoom, liveStatus, now, stoppingToken);
await dbContext.SaveChangesAsync(stoppingToken);
if (!liveStatus.IsLive)
{
await CompleteActiveSessionsForOfflineRoomAsync(
dbContext,
ffmpegService,
logService,
liveRoom.Id,
stoppingToken);
continue;
}
if (!settings.AutoStartRecordingOnLive)
{
continue;
}
var hasRunningSession = await dbContext.RecordSessions.AnyAsync(
item => item.LiveRoomId == liveRoom.Id &&
(item.Status == RecordSessionStatus.Starting ||
item.Status == RecordSessionStatus.Running ||
item.Status == RecordSessionStatus.Stopping),
stoppingToken);
if (hasRunningSession)
{
continue;
}
_logger.LogInformation("Auto-start recording for live room {RoomId}", liveRoom.RoomId);
await logService.WriteAsync(
SystemLogLevel.Info,
"Scheduler",
"Live detected by background poller. Auto-starting recording task.",
liveRoomId: liveRoom.Id,
cancellationToken: stoppingToken);
await recordService.StartAsync(
new StartRecordTaskRequest
{
LiveRoomId = liveRoom.Id,
PreferredQuality = settings.DefaultQuality,
OutputFormat = settings.DefaultOutputFormat
},
stoppingToken);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Polling room {RoomId} failed", liveRoom.RoomId);
var isTransient = IsTransientPollingException(ex, stoppingToken);
await logService.WriteAsync(
isTransient ? SystemLogLevel.Warning : SystemLogLevel.Error,
"Scheduler",
isTransient
? "Transient background polling failure. The room will be retried on the next cycle."
: "Background polling failed for a live room.",
ex.ToString(),
liveRoomId: liveRoom.Id,
cancellationToken: stoppingToken);
if (!isTransient)
{
await emailNotificationService.SendExceptionAsync(
"Scheduler",
"Background polling failed for a live room.",
ex.ToString(),
liveRoom,
cancellationToken: stoppingToken);
}
}
await PollLiveRoomAsync(liveRoomId, settings, stoppingToken);
}
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
@@ -160,11 +94,14 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
{
using var notificationScope = _serviceScopeFactory.CreateScope();
var emailNotificationService = notificationScope.ServiceProvider.GetRequiredService<IEmailNotificationService>();
await emailNotificationService.SendExceptionAsync(
"Scheduler",
"Background live room polling failed.",
ex.ToString(),
cancellationToken: stoppingToken);
if (ShouldSendExceptionEmail(BuildExceptionEmailKey("background-loop", ex)))
{
await emailNotificationService.SendExceptionAsync(
"Scheduler",
"Background live room polling failed.",
ex.ToString(),
cancellationToken: stoppingToken);
}
}
catch (Exception notificationEx)
{
@@ -179,6 +116,128 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
private static Task DelayAsync(int delaySeconds, CancellationToken cancellationToken) =>
Task.Delay(TimeSpan.FromSeconds(Math.Clamp(delaySeconds, 10, 3600)), cancellationToken);
private async Task PollLiveRoomAsync(Guid liveRoomId, SystemSettingsDto settings, CancellationToken cancellationToken)
{
using var scope = _serviceScopeFactory.CreateScope();
var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
var adapterFactory = scope.ServiceProvider.GetRequiredService<ILivePlatformAdapterFactory>();
var ffmpegService = scope.ServiceProvider.GetRequiredService<IFfmpegService>();
var recordService = scope.ServiceProvider.GetRequiredService<RecordService>();
var liveRoomStatusService = scope.ServiceProvider.GetRequiredService<LiveRoomStatusService>();
var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
var emailNotificationService = scope.ServiceProvider.GetRequiredService<IEmailNotificationService>();
var storageGuardService = scope.ServiceProvider.GetRequiredService<IStorageGuardService>();
var liveRoom = await dbContext.LiveRooms.FirstOrDefaultAsync(item => item.Id == liveRoomId, cancellationToken);
if (liveRoom is null || !liveRoom.IsEnabled)
{
return;
}
try
{
var adapter = adapterFactory.GetByPlatform(liveRoom.Platform);
var liveStatus = await adapter.GetLiveStatusAsync(liveRoom.RoomId, cancellationToken);
var now = DateTimeOffset.UtcNow;
await liveRoomStatusService.ApplySnapshotAsync(liveRoom, liveStatus, now, cancellationToken);
await SaveChangesWithRetryAsync(dbContext, cancellationToken);
if (!liveStatus.IsLive)
{
await CompleteActiveSessionsForOfflineRoomAsync(
dbContext,
ffmpegService,
logService,
liveRoom.Id,
cancellationToken);
return;
}
var pauseCheck = storageGuardService.CheckShouldPause(settings);
if (!pauseCheck.HasEnoughSpace)
{
await PauseActiveSessionsForLowStorageAsync(
dbContext,
ffmpegService,
logService,
liveRoom.Id,
pauseCheck.Message,
cancellationToken);
}
if (!settings.AutoStartRecordingOnLive)
{
return;
}
var startCheck = storageGuardService.CheckCanStartOrResume(settings);
if (!startCheck.HasEnoughSpace)
{
await logService.WriteAsync(
SystemLogLevel.Warning,
"Storage",
"Auto-start recording skipped because storage is below resume threshold.",
startCheck.Message,
liveRoomId: liveRoom.Id,
cancellationToken: cancellationToken);
return;
}
var hasRunningSession = await dbContext.RecordSessions.AnyAsync(
item => item.LiveRoomId == liveRoom.Id &&
(item.Status == RecordSessionStatus.Starting ||
item.Status == RecordSessionStatus.Running ||
item.Status == RecordSessionStatus.Stopping),
cancellationToken);
if (hasRunningSession)
{
return;
}
_logger.LogInformation("Auto-start recording for live room {RoomId}", liveRoom.RoomId);
await logService.WriteAsync(
SystemLogLevel.Info,
"Scheduler",
"Live detected by background poller. Auto-starting recording task.",
liveRoomId: liveRoom.Id,
cancellationToken: cancellationToken);
await recordService.StartAsync(
new StartRecordTaskRequest
{
LiveRoomId = liveRoom.Id
},
cancellationToken);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Polling room {RoomId} failed", liveRoom.RoomId);
var isTransient = IsTransientPollingException(ex, cancellationToken);
await logService.WriteAsync(
isTransient ? SystemLogLevel.Warning : SystemLogLevel.Error,
"Scheduler",
isTransient
? "Transient background polling failure. The room will be retried on the next cycle."
: "Background polling failed for a live room.",
ex.ToString(),
liveRoomId: liveRoom.Id,
cancellationToken: cancellationToken);
if (!isTransient && ShouldSendExceptionEmail(BuildExceptionEmailKey("live-room-poll", ex, liveRoom.Id)))
{
await emailNotificationService.SendExceptionAsync(
"Scheduler",
"Background polling failed for a live room.",
ex.ToString(),
liveRoom,
cancellationToken: cancellationToken);
}
}
}
private static async Task CompleteActiveSessionsForOfflineRoomAsync(
LiveRecorderDbContext dbContext,
IFfmpegService ffmpegService,
@@ -201,7 +260,40 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
{
if (ffmpegService.IsRunning(activeSession.Id))
{
await ffmpegService.CompleteAsync(activeSession.Id, cancellationToken);
var stopped = await ffmpegService.StopAndWaitAsync(
activeSession.Id,
markAsCompletedOnExit: true,
OfflineGracefulStopTimeout,
cancellationToken);
if (!stopped)
{
await logService.WriteAsync(
SystemLogLevel.Warning,
"Scheduler",
"Live room is offline, but the recorder did not stop gracefully in time. Force killing the ffmpeg process.",
liveRoomId: liveRoomId,
recordSessionId: activeSession.Id,
cancellationToken: cancellationToken);
var killed = await ffmpegService.KillAndWaitAsync(
activeSession.Id,
OfflineForcedStopTimeout,
cancellationToken);
if (!killed)
{
await logService.WriteAsync(
SystemLogLevel.Error,
"Scheduler",
"Live room is offline, but the active recording session is still shutting down.",
liveRoomId: liveRoomId,
recordSessionId: activeSession.Id,
cancellationToken: cancellationToken);
continue;
}
}
await ffmpegService.TryReconcileInactiveSessionAsync(activeSession.Id, cancellationToken);
await logService.WriteAsync(
SystemLogLevel.Info,
"Scheduler",
@@ -225,6 +317,57 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
}
}
private static async Task PauseActiveSessionsForLowStorageAsync(
LiveRecorderDbContext dbContext,
IFfmpegService ffmpegService,
ISystemLogService logService,
Guid liveRoomId,
string detail,
CancellationToken cancellationToken)
{
var activeSessions = await dbContext.RecordSessions
.Where(item => item.LiveRoomId == liveRoomId &&
(item.Status == RecordSessionStatus.Starting ||
item.Status == RecordSessionStatus.Running ||
item.Status == RecordSessionStatus.Stopping))
.ToListAsync(cancellationToken);
foreach (var activeSession in activeSessions.OrderBy(static item => item.CreatedAt))
{
await logService.WriteAsync(
SystemLogLevel.Warning,
"Storage",
"Storage is below threshold. Pausing active recording session.",
detail,
liveRoomId: liveRoomId,
recordSessionId: activeSession.Id,
cancellationToken: cancellationToken);
if (ffmpegService.IsRunning(activeSession.Id))
{
var stopped = await ffmpegService.StopAndWaitAsync(
activeSession.Id,
markAsCompletedOnExit: false,
OfflineGracefulStopTimeout,
cancellationToken);
if (!stopped)
{
await logService.WriteAsync(
SystemLogLevel.Warning,
"Storage",
"Recorder did not stop gracefully after low storage pause. Force killing the ffmpeg process.",
detail,
liveRoomId,
activeSession.Id,
cancellationToken: cancellationToken);
await ffmpegService.KillAndWaitAsync(activeSession.Id, OfflineForcedStopTimeout, cancellationToken);
}
}
await ffmpegService.TryReconcileInactiveSessionAsync(activeSession.Id, cancellationToken);
}
}
private static bool IsTransientPollingException(Exception exception, CancellationToken cancellationToken)
{
if (exception is OperationCanceledException && cancellationToken.IsCancellationRequested)
@@ -237,7 +380,99 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
return true;
}
if (exception is DbUpdateException dbUpdateException &&
IsSqliteLockException(dbUpdateException))
{
return true;
}
if (exception is SqliteException sqliteException &&
IsSqliteLockException(sqliteException))
{
return true;
}
return exception.InnerException is not null &&
IsTransientPollingException(exception.InnerException, cancellationToken);
}
private bool ShouldSendExceptionEmail(string key)
{
var now = DateTimeOffset.UtcNow;
while (true)
{
if (_exceptionEmailSentAt.TryGetValue(key, out var lastSentAt))
{
if (now - lastSentAt < ExceptionEmailCooldown)
{
_logger.LogWarning(
"Suppressed repeated scheduler exception email. Key={Key}; CooldownMinutes={CooldownMinutes}",
key,
ExceptionEmailCooldown.TotalMinutes);
return false;
}
if (_exceptionEmailSentAt.TryUpdate(key, now, lastSentAt))
{
return true;
}
continue;
}
if (_exceptionEmailSentAt.TryAdd(key, now))
{
return true;
}
}
}
private static string BuildExceptionEmailKey(string scope, Exception exception, Guid? liveRoomId = null)
{
if (IsSqliteStorageFullException(exception))
{
return $"{scope}:sqlite-storage-full";
}
var root = exception.GetBaseException();
var message = root.Message.Length > 160 ? root.Message[..160] : root.Message;
return $"{scope}:{liveRoomId?.ToString() ?? "global"}:{root.GetType().FullName}:{message}";
}
private static bool IsSqliteStorageFullException(Exception exception)
{
if (exception is SqliteException sqliteException &&
(sqliteException.SqliteErrorCode == 13 ||
sqliteException.Message.Contains("database or disk is full", StringComparison.OrdinalIgnoreCase)))
{
return true;
}
return exception.InnerException is not null && IsSqliteStorageFullException(exception.InnerException);
}
private static async Task SaveChangesWithRetryAsync(LiveRecorderDbContext dbContext, CancellationToken cancellationToken)
{
for (var attempt = 1; attempt <= 5; attempt++)
{
try
{
await dbContext.SaveChangesAsync(cancellationToken);
return;
}
catch (DbUpdateException ex) when (attempt < 5 && IsSqliteLockException(ex))
{
await Task.Delay(TimeSpan.FromMilliseconds(300 * Math.Pow(2, attempt - 1)), cancellationToken);
}
}
}
private static bool IsSqliteLockException(DbUpdateException exception) =>
exception.InnerException is SqliteException sqliteException &&
IsSqliteLockException(sqliteException);
private static bool IsSqliteLockException(SqliteException exception) =>
exception.SqliteErrorCode is 5 or 6 ||
exception.Message.Contains("database is locked", StringComparison.OrdinalIgnoreCase) ||
exception.Message.Contains("database table is locked", StringComparison.OrdinalIgnoreCase);
}
@@ -29,9 +29,9 @@ public sealed class RecordMediaService : IRecordMediaService
var recordTask = await _recordTaskRepository.GetByIdAsync(recordTaskId, cancellationToken)
?? throw new KeyNotFoundException("Recording task was not found.");
if (recordTask.Status != RecordTaskStatus.Completed)
if (recordTask.Status is not (RecordTaskStatus.Completed or RecordTaskStatus.Stopped))
{
throw new InvalidOperationException("Preview is only available for completed tasks.");
throw new InvalidOperationException("Preview is only available for finalized MP4 tasks.");
}
if (recordTask.OutputFormat != RecordOutputFormat.Mp4)
@@ -0,0 +1,80 @@
using LiveRecorder.Application.Abstractions.Storage;
using LiveRecorder.Application.Models.Settings;
using Microsoft.Extensions.Logging;
namespace LiveRecorder.Infrastructure.Services;
public sealed class StorageGuardService : IStorageGuardService
{
private const long Megabyte = 1024L * 1024L;
private readonly ILogger<StorageGuardService> _logger;
public StorageGuardService(ILogger<StorageGuardService> logger)
{
_logger = logger;
}
public StorageGuardResult CheckCanStartOrResume(SystemSettingsDto settings, long additionalRequiredBytes = 0) =>
Check(settings, Math.Max(settings.PauseRecordingWhenFreeSpaceBelowMegabytes, settings.ResumeRecordingWhenFreeSpaceAboveMegabytes), additionalRequiredBytes);
public StorageGuardResult CheckShouldPause(SystemSettingsDto settings) =>
Check(settings, settings.PauseRecordingWhenFreeSpaceBelowMegabytes, additionalRequiredBytes: 0);
private StorageGuardResult Check(SystemSettingsDto settings, int freeSpaceThresholdMegabytes, long additionalRequiredBytes)
{
if (!settings.EnableStorageGuard)
{
return new StorageGuardResult(false, true, ResolveOutputRoot(settings.OutputRoot), long.MaxValue, 0, "Storage guard is disabled.");
}
var checkedPath = ResolveOutputRoot(settings.OutputRoot);
var thresholdBytes = Math.Max(0, freeSpaceThresholdMegabytes) * Megabyte;
var requiredBytes = thresholdBytes + Math.Max(0, additionalRequiredBytes);
try
{
var drive = new DriveInfo(Path.GetPathRoot(checkedPath) ?? checkedPath);
var availableBytes = drive.AvailableFreeSpace;
var hasEnoughSpace = availableBytes >= requiredBytes;
var message = hasEnoughSpace
? $"Storage is available. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}"
: $"Storage is below threshold. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}";
return new StorageGuardResult(true, hasEnoughSpace, checkedPath, availableBytes, requiredBytes, message);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Storage guard failed to inspect output root {OutputRoot}", checkedPath);
return new StorageGuardResult(true, false, checkedPath, 0, requiredBytes, $"Unable to inspect storage path {checkedPath}: {ex.Message}");
}
}
private static string ResolveOutputRoot(string outputRoot)
{
var root = string.IsNullOrWhiteSpace(outputRoot) ? "records" : outputRoot.Trim();
return Path.IsPathRooted(root)
? root
: Path.GetFullPath(root, AppContext.BaseDirectory);
}
private static string FormatBytes(long bytes)
{
if (bytes == long.MaxValue)
{
return "unlimited";
}
string[] units = ["B", "KB", "MB", "GB", "TB"];
var value = Math.Max(0, bytes);
var unitIndex = 0;
var display = (double)value;
while (display >= 1024 && unitIndex < units.Length - 1)
{
display /= 1024;
unitIndex++;
}
return $"{display:0.##} {units[unitIndex]}";
}
}