feat: improve recording automation and task workflows
This commit is contained in:
@@ -0,0 +1,278 @@
|
||||
using System.Diagnostics;
|
||||
using LiveRecorder.Application.Abstractions.Logging;
|
||||
using LiveRecorder.Application.Abstractions.Scripting;
|
||||
using LiveRecorder.Application.Abstractions.Settings;
|
||||
using LiveRecorder.Domain.Entities;
|
||||
using LiveRecorder.Domain.Enums;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace LiveRecorder.Infrastructure.Services;
|
||||
|
||||
public sealed class EventScriptService : IEventScriptService
|
||||
{
|
||||
private readonly ISystemSettingsService _settingsService;
|
||||
private readonly ISystemLogService _systemLogService;
|
||||
private readonly ILogger<EventScriptService> _logger;
|
||||
|
||||
public EventScriptService(
|
||||
ISystemSettingsService settingsService,
|
||||
ISystemLogService systemLogService,
|
||||
ILogger<EventScriptService> logger)
|
||||
{
|
||||
_settingsService = settingsService;
|
||||
_systemLogService = systemLogService;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task RunLiveStartedAsync(LiveRoom liveRoom, DateTimeOffset occurredAt, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var settings = await _settingsService.GetAsync(cancellationToken);
|
||||
var environment = BuildLiveRoomEnvironment(liveRoom, occurredAt);
|
||||
environment["LIVE_RECORDER_EVENT"] = "live_started";
|
||||
await RunAsync(
|
||||
settings.EnableEventScripts,
|
||||
settings.LiveStartedScriptPath,
|
||||
settings.EventScriptTimeoutSeconds,
|
||||
"live_started",
|
||||
environment,
|
||||
liveRoom.Id,
|
||||
recordSessionId: null,
|
||||
recordTaskId: null,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public async Task RunLiveEndedAsync(LiveRoom liveRoom, DateTimeOffset occurredAt, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var settings = await _settingsService.GetAsync(cancellationToken);
|
||||
var environment = BuildLiveRoomEnvironment(liveRoom, occurredAt);
|
||||
environment["LIVE_RECORDER_EVENT"] = "live_ended";
|
||||
await RunAsync(
|
||||
settings.EnableEventScripts,
|
||||
settings.LiveEndedScriptPath,
|
||||
settings.EventScriptTimeoutSeconds,
|
||||
"live_ended",
|
||||
environment,
|
||||
liveRoom.Id,
|
||||
recordSessionId: null,
|
||||
recordTaskId: null,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public async Task RunSegmentCompletedAsync(
|
||||
LiveRoom? liveRoom,
|
||||
RecordSession recordSession,
|
||||
RecordTask recordTask,
|
||||
RecordResult? recordResult,
|
||||
string segmentFilePath,
|
||||
DateTimeOffset occurredAt,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var settings = await _settingsService.GetAsync(cancellationToken);
|
||||
var environment = BuildLiveRoomEnvironment(liveRoom, occurredAt);
|
||||
environment["LIVE_RECORDER_EVENT"] = "segment_completed";
|
||||
environment["LIVE_RECORDER_RECORD_SESSION_ID"] = recordSession.Id.ToString();
|
||||
environment["LIVE_RECORDER_RECORD_TASK_ID"] = recordTask.Id.ToString();
|
||||
environment["LIVE_RECORDER_SEGMENT_INDEX"] = recordTask.SegmentIndex.ToString();
|
||||
environment["LIVE_RECORDER_SEGMENT_FILE_PATH"] = NormalizePath(segmentFilePath);
|
||||
environment["LIVE_RECORDER_DANMAKU_FILE_PATH"] = NormalizePath(recordResult?.DanmakuFilePath);
|
||||
environment["LIVE_RECORDER_DURATION_SECONDS"] = recordResult?.DurationSeconds?.ToString("0.###") ?? recordTask.DurationSeconds?.ToString("0.###") ?? string.Empty;
|
||||
environment["LIVE_RECORDER_FILE_SIZE_BYTES"] = recordResult?.FileSizeBytes?.ToString() ?? TryGetFileSize(segmentFilePath);
|
||||
environment["LIVE_RECORDER_TASK_STATUS"] = recordTask.Status.ToString();
|
||||
environment["LIVE_RECORDER_SESSION_STATUS"] = recordSession.Status.ToString();
|
||||
|
||||
await RunAsync(
|
||||
settings.EnableEventScripts,
|
||||
settings.SegmentCompletedScriptPath,
|
||||
settings.EventScriptTimeoutSeconds,
|
||||
"segment_completed",
|
||||
environment,
|
||||
liveRoom?.Id ?? recordSession.LiveRoomId,
|
||||
recordSession.Id,
|
||||
recordTask.Id,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private async Task RunAsync(
|
||||
bool enabled,
|
||||
string scriptPath,
|
||||
int timeoutSeconds,
|
||||
string eventName,
|
||||
IReadOnlyDictionary<string, string> environment,
|
||||
Guid? liveRoomId,
|
||||
Guid? recordSessionId,
|
||||
Guid? recordTaskId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!enabled || string.IsNullOrWhiteSpace(scriptPath))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var normalizedScriptPath = NormalizePath(scriptPath);
|
||||
if (!File.Exists(normalizedScriptPath))
|
||||
{
|
||||
await _systemLogService.WriteAsync(
|
||||
SystemLogLevel.Warning,
|
||||
"Script",
|
||||
$"Event script was not found for {eventName}.",
|
||||
normalizedScriptPath,
|
||||
liveRoomId,
|
||||
recordSessionId,
|
||||
recordTaskId,
|
||||
cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
var startInfo = CreateStartInfo(normalizedScriptPath);
|
||||
foreach (var pair in environment)
|
||||
{
|
||||
startInfo.Environment[pair.Key] = pair.Value;
|
||||
}
|
||||
|
||||
using var process = new Process
|
||||
{
|
||||
StartInfo = startInfo,
|
||||
EnableRaisingEvents = true
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
process.Start();
|
||||
using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
timeoutCts.CancelAfter(TimeSpan.FromSeconds(Math.Clamp(timeoutSeconds, 1, 3600)));
|
||||
await process.WaitForExitAsync(timeoutCts.Token);
|
||||
|
||||
await _systemLogService.WriteAsync(
|
||||
process.ExitCode == 0 ? SystemLogLevel.Info : SystemLogLevel.Warning,
|
||||
"Script",
|
||||
process.ExitCode == 0
|
||||
? $"Event script completed for {eventName}."
|
||||
: $"Event script exited with code {process.ExitCode} for {eventName}.",
|
||||
normalizedScriptPath,
|
||||
liveRoomId,
|
||||
recordSessionId,
|
||||
recordTaskId,
|
||||
cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
TryKill(process);
|
||||
await _systemLogService.WriteAsync(
|
||||
SystemLogLevel.Warning,
|
||||
"Script",
|
||||
$"Event script timed out for {eventName}.",
|
||||
normalizedScriptPath,
|
||||
liveRoomId,
|
||||
recordSessionId,
|
||||
recordTaskId,
|
||||
CancellationToken.None);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Event script failed for {EventName}", eventName);
|
||||
await _systemLogService.WriteAsync(
|
||||
SystemLogLevel.Warning,
|
||||
"Script",
|
||||
$"Event script failed for {eventName}.",
|
||||
ex.ToString(),
|
||||
liveRoomId,
|
||||
recordSessionId,
|
||||
recordTaskId,
|
||||
cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
private static Dictionary<string, string> BuildLiveRoomEnvironment(LiveRoom? liveRoom, DateTimeOffset occurredAt)
|
||||
{
|
||||
return new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["LIVE_RECORDER_EVENT"] = string.Empty,
|
||||
["LIVE_RECORDER_PLATFORM"] = liveRoom?.Platform.ToString() ?? string.Empty,
|
||||
["LIVE_RECORDER_LIVE_ROOM_ID"] = liveRoom?.Id.ToString() ?? string.Empty,
|
||||
["LIVE_RECORDER_ROOM_ID"] = liveRoom?.RoomId ?? string.Empty,
|
||||
["LIVE_RECORDER_TITLE"] = liveRoom?.Title ?? string.Empty,
|
||||
["LIVE_RECORDER_ANCHOR"] = liveRoom?.AnchorName ?? string.Empty,
|
||||
["LIVE_RECORDER_SOURCE_URL"] = liveRoom?.SourceUrl ?? liveRoom?.NormalizedUrl ?? string.Empty,
|
||||
["LIVE_RECORDER_OCCURRED_AT_UTC"] = occurredAt.ToString("O")
|
||||
};
|
||||
}
|
||||
|
||||
private static ProcessStartInfo CreateStartInfo(string scriptPath)
|
||||
{
|
||||
var extension = Path.GetExtension(scriptPath);
|
||||
var startInfo = new ProcessStartInfo
|
||||
{
|
||||
UseShellExecute = false,
|
||||
RedirectStandardError = false,
|
||||
RedirectStandardOutput = false,
|
||||
CreateNoWindow = true,
|
||||
WorkingDirectory = Path.GetDirectoryName(scriptPath) ?? AppContext.BaseDirectory
|
||||
};
|
||||
|
||||
if (OperatingSystem.IsWindows() && extension.Equals(".ps1", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
startInfo.FileName = "powershell";
|
||||
startInfo.ArgumentList.Add("-NoProfile");
|
||||
startInfo.ArgumentList.Add("-ExecutionPolicy");
|
||||
startInfo.ArgumentList.Add("Bypass");
|
||||
startInfo.ArgumentList.Add("-File");
|
||||
startInfo.ArgumentList.Add(scriptPath);
|
||||
return startInfo;
|
||||
}
|
||||
|
||||
if (OperatingSystem.IsWindows() && (extension.Equals(".bat", StringComparison.OrdinalIgnoreCase) || extension.Equals(".cmd", StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
startInfo.FileName = "cmd";
|
||||
startInfo.ArgumentList.Add("/c");
|
||||
startInfo.ArgumentList.Add(scriptPath);
|
||||
return startInfo;
|
||||
}
|
||||
|
||||
if (!OperatingSystem.IsWindows() && extension.Equals(".sh", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
startInfo.FileName = "/bin/sh";
|
||||
startInfo.ArgumentList.Add(scriptPath);
|
||||
return startInfo;
|
||||
}
|
||||
|
||||
startInfo.FileName = scriptPath;
|
||||
return startInfo;
|
||||
}
|
||||
|
||||
private static string NormalizePath(string? path)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(path))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
return Path.IsPathRooted(path)
|
||||
? path
|
||||
: Path.GetFullPath(path, AppContext.BaseDirectory);
|
||||
}
|
||||
|
||||
private static string TryGetFileSize(string? path)
|
||||
{
|
||||
var normalizedPath = NormalizePath(path);
|
||||
if (string.IsNullOrWhiteSpace(normalizedPath) || !File.Exists(normalizedPath))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
return new FileInfo(normalizedPath).Length.ToString();
|
||||
}
|
||||
|
||||
private static void TryKill(Process process)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!process.HasExited)
|
||||
{
|
||||
process.Kill(true);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,18 +1,32 @@
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using LiveRecorder.Application.Abstractions.Platforms;
|
||||
using LiveRecorder.Domain.Entities;
|
||||
using LiveRecorder.Domain.Enums;
|
||||
using LiveRecorder.Infrastructure.Persistence;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace LiveRecorder.Infrastructure.Services;
|
||||
|
||||
public sealed partial class FfmpegService
|
||||
{
|
||||
private static async Task<(string OutputPath, string? ErrorMessage)> TryFinalizeMp4Async(
|
||||
private const string LowStoragePauseErrorPrefix = "MP4 finalization paused because storage is below threshold.";
|
||||
private static readonly TimeSpan Mp4FinalizeInactivityTimeout = TimeSpan.FromMinutes(10);
|
||||
private static readonly TimeSpan Mp4FinalizePollInterval = TimeSpan.FromSeconds(1);
|
||||
|
||||
private async Task<(string OutputPath, string? ErrorMessage)> TryFinalizeMp4Async(
|
||||
string ffmpegPath,
|
||||
int maxConcurrentTranscodeTasks,
|
||||
int mp4FinalizeTimeoutMinutes,
|
||||
Guid recordSessionId,
|
||||
Guid recordTaskId,
|
||||
string sourcePath,
|
||||
string targetPath)
|
||||
string targetPath,
|
||||
double? expectedDurationSeconds,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!File.Exists(sourcePath))
|
||||
{
|
||||
@@ -28,24 +42,286 @@ public sealed partial class FfmpegService
|
||||
File.Delete(tempPath);
|
||||
}
|
||||
|
||||
var remuxProcess = new Process
|
||||
async Task<string?> GetLowStoragePauseMessageAsync()
|
||||
{
|
||||
StartInfo = new ProcessStartInfo
|
||||
using var storageScope = _serviceScopeFactory.CreateScope();
|
||||
var settingsService = storageScope.ServiceProvider.GetRequiredService<LiveRecorder.Application.Abstractions.Settings.ISystemSettingsService>();
|
||||
var settings = await settingsService.GetAsync(cancellationToken);
|
||||
var sourceSizeBytes = new FileInfo(sourcePath).Length;
|
||||
var storageCheck = _storageGuardService.CheckCanStartOrResume(settings, sourceSizeBytes);
|
||||
return storageCheck.HasEnoughSpace
|
||||
? null
|
||||
: $"{LowStoragePauseErrorPrefix} {storageCheck.Message}";
|
||||
}
|
||||
|
||||
var lowStoragePauseMessage = await GetLowStoragePauseMessageAsync();
|
||||
if (!string.IsNullOrWhiteSpace(lowStoragePauseMessage))
|
||||
{
|
||||
SetPostProcessState(recordSessionId, recordTaskId, "Paused: Low storage", null, lowStoragePauseMessage);
|
||||
return (sourcePath, lowStoragePauseMessage);
|
||||
}
|
||||
|
||||
SetPostProcessState(
|
||||
recordSessionId,
|
||||
recordTaskId,
|
||||
"Queued",
|
||||
null,
|
||||
$"Waiting for an available ffmpeg transcode slot (max {Math.Clamp(maxConcurrentTranscodeTasks, 1, 16)}).");
|
||||
|
||||
using var transcodeSlot = await AcquireTranscodeSlotAsync(maxConcurrentTranscodeTasks, cancellationToken);
|
||||
|
||||
lowStoragePauseMessage = await GetLowStoragePauseMessageAsync();
|
||||
if (!string.IsNullOrWhiteSpace(lowStoragePauseMessage))
|
||||
{
|
||||
SetPostProcessState(recordSessionId, recordTaskId, "Paused: Low storage", null, lowStoragePauseMessage);
|
||||
return (sourcePath, lowStoragePauseMessage);
|
||||
}
|
||||
|
||||
var stderrLines = new Queue<string>();
|
||||
var progressSync = new object();
|
||||
double? processedSeconds = null;
|
||||
var lastReportedWholePercent = -1;
|
||||
var attemptStartedAt = DateTimeOffset.UtcNow;
|
||||
var activeStage = "Finalizing MP4";
|
||||
var maxDuration = TimeSpan.FromMinutes(Math.Clamp(mp4FinalizeTimeoutMinutes, 1, 1440));
|
||||
long lastActivityTicks = attemptStartedAt.UtcTicks;
|
||||
|
||||
void TouchActivity()
|
||||
{
|
||||
System.Threading.Interlocked.Exchange(ref lastActivityTicks, DateTimeOffset.UtcNow.UtcTicks);
|
||||
}
|
||||
|
||||
void ReportProgress(double? seconds, string? stageOverride = null)
|
||||
{
|
||||
lock (progressSync)
|
||||
{
|
||||
FileName = ffmpegPath,
|
||||
Arguments = $"-hide_banner -y -i {Quote(sourcePath)} -c copy -movflags +faststart {Quote(tempPath)}",
|
||||
UseShellExecute = false,
|
||||
RedirectStandardError = true,
|
||||
RedirectStandardOutput = true,
|
||||
CreateNoWindow = true
|
||||
TouchActivity();
|
||||
if (seconds.HasValue)
|
||||
{
|
||||
processedSeconds = seconds;
|
||||
}
|
||||
|
||||
double? progressPercent = null;
|
||||
string? detail = null;
|
||||
|
||||
if (expectedDurationSeconds.HasValue && expectedDurationSeconds.Value > 0 && processedSeconds.HasValue)
|
||||
{
|
||||
progressPercent = Math.Clamp(processedSeconds.Value / expectedDurationSeconds.Value * 100d, 0d, 99d);
|
||||
detail = $"Processed {processedSeconds.Value:F1}s / {expectedDurationSeconds.Value:F1}s";
|
||||
}
|
||||
else if (processedSeconds.HasValue)
|
||||
{
|
||||
detail = $"Processed {processedSeconds.Value:F1}s";
|
||||
}
|
||||
|
||||
var wholePercent = progressPercent.HasValue ? (int)Math.Floor(progressPercent.Value) : -1;
|
||||
if (stageOverride is null && wholePercent == lastReportedWholePercent)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
lastReportedWholePercent = wholePercent;
|
||||
SetPostProcessState(
|
||||
recordSessionId,
|
||||
recordTaskId,
|
||||
stageOverride ?? activeStage,
|
||||
progressPercent,
|
||||
detail ?? $"Optimizing MP4 index for {Path.GetFileName(targetPath)}");
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
remuxProcess.Start();
|
||||
await remuxProcess.WaitForExitAsync();
|
||||
|
||||
if (remuxProcess.ExitCode == 0 && File.Exists(tempPath))
|
||||
string? GetErrorDetail()
|
||||
{
|
||||
lock (stderrLines)
|
||||
{
|
||||
return stderrLines.Count == 0 ? null : string.Join(" | ", stderrLines);
|
||||
}
|
||||
}
|
||||
|
||||
void ClearErrorDetail()
|
||||
{
|
||||
lock (stderrLines)
|
||||
{
|
||||
stderrLines.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
async Task<string?> RunFinalizeAttemptAsync(
|
||||
Mp4FinalizeStrategy strategy,
|
||||
string stage,
|
||||
string detail)
|
||||
{
|
||||
if (File.Exists(tempPath))
|
||||
{
|
||||
File.Delete(tempPath);
|
||||
}
|
||||
|
||||
ClearErrorDetail();
|
||||
activeStage = stage;
|
||||
processedSeconds = null;
|
||||
lastReportedWholePercent = -1;
|
||||
attemptStartedAt = DateTimeOffset.UtcNow;
|
||||
System.Threading.Interlocked.Exchange(ref lastActivityTicks, attemptStartedAt.UtcTicks);
|
||||
|
||||
SetPostProcessState(recordSessionId, recordTaskId, stage, 0, detail);
|
||||
|
||||
using var finalizeProcess = new Process
|
||||
{
|
||||
StartInfo = new ProcessStartInfo
|
||||
{
|
||||
FileName = ffmpegPath,
|
||||
UseShellExecute = false,
|
||||
RedirectStandardError = true,
|
||||
RedirectStandardOutput = true,
|
||||
CreateNoWindow = true
|
||||
}
|
||||
};
|
||||
|
||||
foreach (var argument in BuildMp4FinalizeArgumentList(sourcePath, tempPath, strategy))
|
||||
{
|
||||
finalizeProcess.StartInfo.ArgumentList.Add(argument);
|
||||
}
|
||||
|
||||
finalizeProcess.OutputDataReceived += (_, args) =>
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(args.Data))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
TouchActivity();
|
||||
|
||||
if (TryParseFfmpegProgressSeconds(args.Data, out var seconds))
|
||||
{
|
||||
ReportProgress(seconds);
|
||||
return;
|
||||
}
|
||||
|
||||
if (args.Data.StartsWith("progress=", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
ReportProgress(processedSeconds);
|
||||
}
|
||||
};
|
||||
|
||||
finalizeProcess.ErrorDataReceived += (_, args) =>
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(args.Data))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
TouchActivity();
|
||||
_logger.LogDebug("ffmpeg-postprocess[{RecordTaskId}] {Line}", recordTaskId, args.Data);
|
||||
lock (stderrLines)
|
||||
{
|
||||
stderrLines.Enqueue(args.Data.Trim());
|
||||
while (stderrLines.Count > 10)
|
||||
{
|
||||
stderrLines.Dequeue();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
finalizeProcess.Start();
|
||||
finalizeProcess.BeginOutputReadLine();
|
||||
finalizeProcess.BeginErrorReadLine();
|
||||
|
||||
while (!finalizeProcess.HasExited)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var lastActivityAt = new DateTimeOffset(System.Threading.Interlocked.Read(ref lastActivityTicks), TimeSpan.Zero);
|
||||
if (now - attemptStartedAt > maxDuration)
|
||||
{
|
||||
throw new TimeoutException($"MP4 finalization exceeded the maximum allowed duration of {maxDuration.TotalMinutes:F0} minutes.");
|
||||
}
|
||||
|
||||
if (now - lastActivityAt > Mp4FinalizeInactivityTimeout)
|
||||
{
|
||||
throw new TimeoutException($"MP4 finalization did not report progress for more than {Mp4FinalizeInactivityTimeout.TotalMinutes:F0} minutes.");
|
||||
}
|
||||
|
||||
await Task.Delay(Mp4FinalizePollInterval, cancellationToken);
|
||||
}
|
||||
|
||||
await finalizeProcess.WaitForExitAsync(cancellationToken);
|
||||
}
|
||||
catch (TimeoutException ex)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!finalizeProcess.HasExited)
|
||||
{
|
||||
finalizeProcess.Kill(true);
|
||||
}
|
||||
}
|
||||
catch (Exception killEx)
|
||||
{
|
||||
_logger.LogWarning(killEx, "Timed-out MP4 finalization process could not be killed for task {RecordTaskId}", recordTaskId);
|
||||
}
|
||||
|
||||
return ex.Message;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!finalizeProcess.HasExited)
|
||||
{
|
||||
finalizeProcess.Kill(true);
|
||||
}
|
||||
}
|
||||
catch (Exception killEx)
|
||||
{
|
||||
_logger.LogWarning(killEx, "Failed MP4 finalization process could not be killed for task {RecordTaskId}", recordTaskId);
|
||||
}
|
||||
|
||||
return ex.Message;
|
||||
}
|
||||
|
||||
if (finalizeProcess.ExitCode == 0 && File.Exists(tempPath))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (File.Exists(tempPath))
|
||||
{
|
||||
File.Delete(tempPath);
|
||||
}
|
||||
|
||||
return GetErrorDetail() ?? $"ffmpeg exited with code {finalizeProcess.ExitCode}.";
|
||||
}
|
||||
|
||||
var finalizationError = await RunFinalizeAttemptAsync(
|
||||
Mp4FinalizeStrategy.StreamCopy,
|
||||
"Finalizing MP4",
|
||||
$"Optimizing MP4 index for {Path.GetFileName(targetPath)}");
|
||||
|
||||
if (IsNoSpaceLeftError(finalizationError))
|
||||
{
|
||||
finalizationError = $"{LowStoragePauseErrorPrefix} {finalizationError}";
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(finalizationError) && IsRepairableMp4FinalizeError(finalizationError))
|
||||
{
|
||||
var repairError = await RunFinalizeAttemptAsync(
|
||||
Mp4FinalizeStrategy.RepairTranscode,
|
||||
"Repairing MP4",
|
||||
$"Repairing stream metadata for {Path.GetFileName(targetPath)}");
|
||||
|
||||
finalizationError = string.IsNullOrWhiteSpace(repairError)
|
||||
? null
|
||||
: IsNoSpaceLeftError(repairError)
|
||||
? $"{LowStoragePauseErrorPrefix} {repairError}"
|
||||
: $"{finalizationError} | fallback repair failed: {repairError}";
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(finalizationError) && File.Exists(tempPath))
|
||||
{
|
||||
ReportProgress(expectedDurationSeconds, "Writing MP4 index");
|
||||
if (File.Exists(targetPath))
|
||||
{
|
||||
File.Replace(tempPath, targetPath, null, ignoreMetadataErrors: true);
|
||||
@@ -60,6 +336,7 @@ public sealed partial class FfmpegService
|
||||
File.Delete(sourcePath);
|
||||
}
|
||||
|
||||
SetPostProcessState(recordSessionId, recordTaskId, "Completed", 100, "MP4 seek index is ready");
|
||||
return (targetPath, null);
|
||||
}
|
||||
|
||||
@@ -69,7 +346,285 @@ public sealed partial class FfmpegService
|
||||
}
|
||||
|
||||
var fallbackPath = File.Exists(targetPath) ? targetPath : sourcePath;
|
||||
return (fallbackPath, "The MP4 file could not be finalized into a seekable output.");
|
||||
string? errorDetail;
|
||||
lock (stderrLines)
|
||||
{
|
||||
errorDetail = stderrLines.Count == 0 ? null : string.Join(" | ", stderrLines);
|
||||
}
|
||||
|
||||
return (
|
||||
fallbackPath,
|
||||
string.IsNullOrWhiteSpace(finalizationError ?? errorDetail)
|
||||
? "The MP4 file could not be finalized into a seekable output."
|
||||
: $"The MP4 file could not be finalized into a seekable output. {finalizationError ?? errorDetail}");
|
||||
}
|
||||
|
||||
private async Task<(string OutputPath, string? ErrorMessage)> TryFinalizeTaskOutputAsync(
|
||||
string ffmpegPath,
|
||||
int maxConcurrentTranscodeTasks,
|
||||
int mp4FinalizeTimeoutMinutes,
|
||||
RecordSession recordSession,
|
||||
RecordTask recordTask,
|
||||
double? expectedDurationSeconds,
|
||||
CancellationToken cancellationToken = default)
|
||||
=> await TryFinalizeTaskOutputAsync(
|
||||
ffmpegPath,
|
||||
maxConcurrentTranscodeTasks,
|
||||
mp4FinalizeTimeoutMinutes,
|
||||
recordSession,
|
||||
recordTask,
|
||||
expectedDurationSeconds,
|
||||
recorderSegmentPaths: null,
|
||||
cancellationToken);
|
||||
|
||||
private async Task<(string OutputPath, string? ErrorMessage)> TryFinalizeTaskOutputAsync(
|
||||
string ffmpegPath,
|
||||
int maxConcurrentTranscodeTasks,
|
||||
int mp4FinalizeTimeoutMinutes,
|
||||
RecordSession recordSession,
|
||||
RecordTask recordTask,
|
||||
double? expectedDurationSeconds,
|
||||
IReadOnlyList<string>? recorderSegmentPaths,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var fallbackOutputPath = recordTask.OutputFilePath ?? string.Empty;
|
||||
if (recordSession.OutputFormat != RecordOutputFormat.Mp4)
|
||||
{
|
||||
return (fallbackOutputPath, null);
|
||||
}
|
||||
|
||||
if (recordSession.SaveMode == RecordSaveMode.SingleFile)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(recordSession.OutputPathPattern))
|
||||
{
|
||||
return (fallbackOutputPath, null);
|
||||
}
|
||||
|
||||
var finalOutputPath = NormalizeAbsolutePath(recordSession.OutputPathPattern);
|
||||
var recorderOutputPath = GetRecorderOutputPath(finalOutputPath, recordSession.OutputFormat, recordSession.SaveMode);
|
||||
if (!File.Exists(recorderOutputPath))
|
||||
{
|
||||
return (File.Exists(finalOutputPath) ? finalOutputPath : fallbackOutputPath, null);
|
||||
}
|
||||
|
||||
return await TryFinalizeMp4Async(
|
||||
ffmpegPath,
|
||||
maxConcurrentTranscodeTasks,
|
||||
mp4FinalizeTimeoutMinutes,
|
||||
recordSession.Id,
|
||||
recordTask.Id,
|
||||
recorderOutputPath,
|
||||
finalOutputPath,
|
||||
expectedDurationSeconds,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(recordTask.OutputFilePath))
|
||||
{
|
||||
return (fallbackOutputPath, null);
|
||||
}
|
||||
|
||||
var finalSegmentOutputPath = NormalizeAbsolutePath(recordTask.OutputFilePath);
|
||||
var normalizedRecorderSegmentPaths = recorderSegmentPaths?
|
||||
.Where(static item => !string.IsNullOrWhiteSpace(item))
|
||||
.Select(NormalizeAbsolutePath)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Where(File.Exists)
|
||||
.ToArray();
|
||||
|
||||
if (normalizedRecorderSegmentPaths is null || normalizedRecorderSegmentPaths.Length == 0)
|
||||
{
|
||||
normalizedRecorderSegmentPaths =
|
||||
[
|
||||
NormalizeAbsolutePath(
|
||||
GetRecorderOutputPath(finalSegmentOutputPath, recordSession.OutputFormat, recordSession.SaveMode))
|
||||
];
|
||||
}
|
||||
|
||||
normalizedRecorderSegmentPaths = normalizedRecorderSegmentPaths
|
||||
.Where(File.Exists)
|
||||
.ToArray();
|
||||
if (normalizedRecorderSegmentPaths.Length == 0)
|
||||
{
|
||||
return (File.Exists(finalSegmentOutputPath) ? finalSegmentOutputPath : recordTask.OutputFilePath, null);
|
||||
}
|
||||
|
||||
string? materializedSourcePath = null;
|
||||
var cleanupMaterializedSource = false;
|
||||
var preserveMaterializedSource = false;
|
||||
try
|
||||
{
|
||||
materializedSourcePath = await MaterializeRecorderSegmentSourceAsync(
|
||||
finalSegmentOutputPath,
|
||||
normalizedRecorderSegmentPaths,
|
||||
cancellationToken);
|
||||
if (string.IsNullOrWhiteSpace(materializedSourcePath))
|
||||
{
|
||||
return (File.Exists(finalSegmentOutputPath) ? finalSegmentOutputPath : recordTask.OutputFilePath, null);
|
||||
}
|
||||
|
||||
cleanupMaterializedSource =
|
||||
!string.Equals(materializedSourcePath, normalizedRecorderSegmentPaths[0], StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
var finalizationResult = await TryFinalizeMp4Async(
|
||||
ffmpegPath,
|
||||
maxConcurrentTranscodeTasks,
|
||||
mp4FinalizeTimeoutMinutes,
|
||||
recordSession.Id,
|
||||
recordTask.Id,
|
||||
materializedSourcePath,
|
||||
finalSegmentOutputPath,
|
||||
expectedDurationSeconds,
|
||||
cancellationToken);
|
||||
preserveMaterializedSource = IsLowStoragePauseError(finalizationResult.ErrorMessage);
|
||||
return finalizationResult;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (cleanupMaterializedSource &&
|
||||
!preserveMaterializedSource &&
|
||||
!string.IsNullOrWhiteSpace(materializedSourcePath) &&
|
||||
File.Exists(materializedSourcePath))
|
||||
{
|
||||
File.Delete(materializedSourcePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<string?> MaterializeRecorderSegmentSourceAsync(
|
||||
string finalOutputPath,
|
||||
IReadOnlyList<string> recorderSegmentPaths,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (recorderSegmentPaths.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (recorderSegmentPaths.Count == 1)
|
||||
{
|
||||
return recorderSegmentPaths[0];
|
||||
}
|
||||
|
||||
var combinedPath = Path.Combine(
|
||||
Path.GetDirectoryName(finalOutputPath)!,
|
||||
$"{Path.GetFileNameWithoutExtension(finalOutputPath)}.concat.ts");
|
||||
|
||||
if (File.Exists(combinedPath))
|
||||
{
|
||||
File.Delete(combinedPath);
|
||||
}
|
||||
|
||||
await using var outputStream = new FileStream(
|
||||
combinedPath,
|
||||
FileMode.CreateNew,
|
||||
FileAccess.Write,
|
||||
FileShare.None,
|
||||
bufferSize: 1024 * 128,
|
||||
useAsync: true);
|
||||
|
||||
foreach (var path in recorderSegmentPaths)
|
||||
{
|
||||
await using var inputStream = new FileStream(
|
||||
path,
|
||||
FileMode.Open,
|
||||
FileAccess.Read,
|
||||
FileShare.ReadWrite,
|
||||
bufferSize: 1024 * 128,
|
||||
useAsync: true);
|
||||
await inputStream.CopyToAsync(outputStream, 1024 * 128, cancellationToken);
|
||||
}
|
||||
|
||||
await outputStream.FlushAsync(cancellationToken);
|
||||
return combinedPath;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> BuildMp4FinalizeArgumentList(
|
||||
string sourcePath,
|
||||
string targetPath,
|
||||
Mp4FinalizeStrategy strategy)
|
||||
{
|
||||
var arguments = new List<string>
|
||||
{
|
||||
"-hide_banner",
|
||||
"-y",
|
||||
"-nostats",
|
||||
"-progress",
|
||||
"pipe:1",
|
||||
"-analyzeduration",
|
||||
"100M",
|
||||
"-probesize",
|
||||
"100M",
|
||||
"-fflags",
|
||||
"+genpts+igndts+discardcorrupt",
|
||||
"-err_detect",
|
||||
"ignore_err",
|
||||
"-i",
|
||||
sourcePath,
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-map",
|
||||
"0:a:0?",
|
||||
"-dn",
|
||||
"-sn"
|
||||
};
|
||||
|
||||
if (strategy == Mp4FinalizeStrategy.RepairTranscode)
|
||||
{
|
||||
arguments.AddRange(
|
||||
[
|
||||
"-c:v", "libx264",
|
||||
"-preset", "veryfast",
|
||||
"-crf", "23",
|
||||
"-c:a", "aac",
|
||||
"-b:a", "128k"
|
||||
]);
|
||||
}
|
||||
else
|
||||
{
|
||||
arguments.AddRange(["-c", "copy"]);
|
||||
}
|
||||
|
||||
arguments.AddRange(
|
||||
[
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
"-avoid_negative_ts",
|
||||
"make_zero",
|
||||
targetPath
|
||||
]);
|
||||
|
||||
return arguments;
|
||||
}
|
||||
|
||||
private static bool IsRepairableMp4FinalizeError(string errorDetail)
|
||||
{
|
||||
if (IsLowStoragePauseError(errorDetail))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return errorDetail.Contains("dimensions not set", StringComparison.OrdinalIgnoreCase) ||
|
||||
errorDetail.Contains("Could not write header", StringComparison.OrdinalIgnoreCase) ||
|
||||
errorDetail.Contains("incorrect codec parameters", StringComparison.OrdinalIgnoreCase) ||
|
||||
errorDetail.Contains("codec parameters", StringComparison.OrdinalIgnoreCase) ||
|
||||
errorDetail.Contains("Invalid data found when processing input", StringComparison.OrdinalIgnoreCase) ||
|
||||
errorDetail.Contains("Error initializing output stream", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static bool IsLowStoragePauseError(string? errorDetail) =>
|
||||
!string.IsNullOrWhiteSpace(errorDetail) &&
|
||||
errorDetail.Contains(LowStoragePauseErrorPrefix, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static bool IsNoSpaceLeftError(string? errorDetail) =>
|
||||
!string.IsNullOrWhiteSpace(errorDetail) &&
|
||||
(errorDetail.Contains("No space left on device", StringComparison.OrdinalIgnoreCase) ||
|
||||
errorDetail.Contains("database or disk is full", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
private enum Mp4FinalizeStrategy
|
||||
{
|
||||
StreamCopy,
|
||||
RepairTranscode
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> BuildArgumentList(
|
||||
@@ -83,6 +638,8 @@ public sealed partial class FfmpegService
|
||||
int readWriteTimeoutMilliseconds,
|
||||
int segmentDurationMinutes,
|
||||
StreamInputHeaders? inputHeaders,
|
||||
string? selectedProtocol,
|
||||
string? selectedVideoCodec,
|
||||
FfmpegInputOptionProfile inputOptionProfile)
|
||||
{
|
||||
var arguments = new List<string> { "-hide_banner", "-y" };
|
||||
@@ -107,7 +664,9 @@ public sealed partial class FfmpegService
|
||||
}
|
||||
}
|
||||
|
||||
if (enableReconnect && inputOptionProfile == FfmpegInputOptionProfile.Baseline)
|
||||
if (enableReconnect &&
|
||||
inputOptionProfile == FfmpegInputOptionProfile.Baseline &&
|
||||
ShouldEnableReconnect(streamUrl, selectedProtocol))
|
||||
{
|
||||
arguments.AddRange(
|
||||
[
|
||||
@@ -121,19 +680,30 @@ public sealed partial class FfmpegService
|
||||
arguments.AddRange(["-rw_timeout", readWriteTimeoutMilliseconds.ToString(), "-fflags", "+discardcorrupt+genpts", "-i", streamUrl]);
|
||||
arguments.AddRange(BuildCodecArguments(recordingTemplate));
|
||||
|
||||
var writesTransportStream = useIntermediateTransportStream || outputFormat == RecordOutputFormat.Ts;
|
||||
var bitstreamFilter = ResolveTransportStreamBitstreamFilter(recordingTemplate, writesTransportStream, selectedVideoCodec);
|
||||
if (!string.IsNullOrWhiteSpace(bitstreamFilter))
|
||||
{
|
||||
arguments.AddRange(["-bsf:v", bitstreamFilter]);
|
||||
}
|
||||
|
||||
if (saveMode == RecordSaveMode.Segmented)
|
||||
{
|
||||
var segmentFormat = writesTransportStream
|
||||
? "mpegts"
|
||||
: "mp4";
|
||||
arguments.AddRange(
|
||||
[
|
||||
"-f", "segment",
|
||||
"-segment_start_number", "1",
|
||||
"-segment_time", Math.Max(60, segmentDurationMinutes * 60).ToString(),
|
||||
"-break_non_keyframes", "0",
|
||||
"-reset_timestamps", "1",
|
||||
"-strftime", "0",
|
||||
"-segment_format", outputFormat == RecordOutputFormat.Ts ? "mpegts" : "mp4"
|
||||
"-segment_format", segmentFormat
|
||||
]);
|
||||
|
||||
if (outputFormat == RecordOutputFormat.Mp4)
|
||||
if (!useIntermediateTransportStream && outputFormat == RecordOutputFormat.Mp4)
|
||||
{
|
||||
arguments.AddRange(["-segment_format_options", $"movflags={BuildSegmentedMp4MovFlags()}"]);
|
||||
}
|
||||
@@ -173,6 +743,35 @@ public sealed partial class FfmpegService
|
||||
]
|
||||
};
|
||||
|
||||
private static string? ResolveTransportStreamBitstreamFilter(
|
||||
RecordingTemplateType recordingTemplate,
|
||||
bool writesTransportStream,
|
||||
string? selectedVideoCodec)
|
||||
{
|
||||
if (!writesTransportStream || !UsesStreamCopy(recordingTemplate) || string.IsNullOrWhiteSpace(selectedVideoCodec))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var normalizedCodec = selectedVideoCodec.Trim();
|
||||
if (normalizedCodec.Contains("h264", StringComparison.OrdinalIgnoreCase) ||
|
||||
normalizedCodec.Contains("avc", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return "h264_mp4toannexb";
|
||||
}
|
||||
|
||||
if (normalizedCodec.Contains("hevc", StringComparison.OrdinalIgnoreCase) ||
|
||||
normalizedCodec.Contains("h265", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return "hevc_mp4toannexb";
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool UsesStreamCopy(RecordingTemplateType recordingTemplate) =>
|
||||
recordingTemplate is RecordingTemplateType.StreamCopy or RecordingTemplateType.ArchiveTs;
|
||||
|
||||
private static long? CalculateFileSize(string? outputPath)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(outputPath))
|
||||
@@ -215,7 +814,7 @@ public sealed partial class FfmpegService
|
||||
return false;
|
||||
}
|
||||
|
||||
private static void UpsertRecordResult(
|
||||
private static async Task UpsertRecordResultAsync(
|
||||
RecordTask recordTask,
|
||||
LiveRecorderDbContext dbContext,
|
||||
string? effectiveOutputPath,
|
||||
@@ -223,71 +822,122 @@ public sealed partial class FfmpegService
|
||||
double? durationSeconds,
|
||||
string? danmakuFilePath,
|
||||
int danmakuMessageCount,
|
||||
DateTimeOffset endedAt)
|
||||
DateTimeOffset endedAt,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(effectiveOutputPath))
|
||||
{
|
||||
effectiveOutputPath = recordTask.OutputFilePath ?? string.Empty;
|
||||
}
|
||||
|
||||
if (recordTask.Result is null)
|
||||
{
|
||||
dbContext.RecordResults.Add(new RecordResult(
|
||||
recordTask.Id,
|
||||
effectiveOutputPath,
|
||||
fileSize,
|
||||
durationSeconds,
|
||||
danmakuFilePath,
|
||||
danmakuMessageCount,
|
||||
recordTask.Status,
|
||||
recordTask.ErrorMessage,
|
||||
endedAt));
|
||||
return;
|
||||
}
|
||||
var resultId = Guid.NewGuid();
|
||||
var normalizedDanmakuCount = Math.Max(0, danmakuMessageCount);
|
||||
var finalStatus = (int)recordTask.Status;
|
||||
|
||||
recordTask.Result.Update(
|
||||
effectiveOutputPath,
|
||||
fileSize,
|
||||
durationSeconds,
|
||||
danmakuFilePath,
|
||||
danmakuMessageCount,
|
||||
recordTask.Status,
|
||||
recordTask.ErrorMessage);
|
||||
// Multiple background paths can reconcile the same segment after ffmpeg exits.
|
||||
// Use SQLite's atomic upsert instead of EF Add-or-Update to avoid RecordTaskId
|
||||
// unique constraint races between scoped DbContext instances.
|
||||
await dbContext.Database.ExecuteSqlInterpolatedAsync($"""
|
||||
INSERT INTO RecordResults
|
||||
(Id, RecordTaskId, FilePath, FileSizeBytes, DurationSeconds, DanmakuFilePath, DanmakuMessageCount, FinalStatus, ErrorMessage, CreatedAt)
|
||||
VALUES
|
||||
({resultId}, {recordTask.Id}, {effectiveOutputPath}, {fileSize}, {durationSeconds}, {danmakuFilePath}, {normalizedDanmakuCount}, {finalStatus}, {recordTask.ErrorMessage}, {endedAt})
|
||||
ON CONFLICT(RecordTaskId) DO UPDATE SET
|
||||
FilePath = excluded.FilePath,
|
||||
FileSizeBytes = excluded.FileSizeBytes,
|
||||
DurationSeconds = excluded.DurationSeconds,
|
||||
DanmakuFilePath = excluded.DanmakuFilePath,
|
||||
DanmakuMessageCount = excluded.DanmakuMessageCount,
|
||||
FinalStatus = excluded.FinalStatus,
|
||||
ErrorMessage = excluded.ErrorMessage;
|
||||
""", cancellationToken);
|
||||
}
|
||||
|
||||
private static Task<RecordResult?> LoadRecordResultAsync(
|
||||
LiveRecorderDbContext dbContext,
|
||||
Guid recordTaskId,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
dbContext.RecordResults
|
||||
.AsNoTracking()
|
||||
.FirstOrDefaultAsync(item => item.RecordTaskId == recordTaskId, cancellationToken);
|
||||
|
||||
private static string BuildSingleFileMp4MovFlags() =>
|
||||
"+faststart+frag_keyframe+empty_moov+default_base_moof";
|
||||
|
||||
private static string BuildSegmentedMp4MovFlags() =>
|
||||
"+faststart+frag_keyframe+empty_moov+default_base_moof";
|
||||
"+faststart";
|
||||
|
||||
private static string GetRecorderOutputPath(
|
||||
string finalOutputPath,
|
||||
RecordOutputFormat outputFormat,
|
||||
RecordSaveMode saveMode)
|
||||
{
|
||||
if (saveMode == RecordSaveMode.SingleFile && outputFormat == RecordOutputFormat.Mp4)
|
||||
if (outputFormat != RecordOutputFormat.Mp4)
|
||||
{
|
||||
return finalOutputPath;
|
||||
}
|
||||
|
||||
if (saveMode == RecordSaveMode.SingleFile)
|
||||
{
|
||||
return Path.Combine(
|
||||
Path.GetDirectoryName(finalOutputPath)!,
|
||||
$"{Path.GetFileNameWithoutExtension(finalOutputPath)}.recording.ts");
|
||||
}
|
||||
|
||||
return finalOutputPath;
|
||||
return Path.ChangeExtension(finalOutputPath, ".ts");
|
||||
}
|
||||
|
||||
private static bool ShouldUseIntermediateTransportStream(
|
||||
string outputPath,
|
||||
RecordOutputFormat outputFormat,
|
||||
RecordSaveMode saveMode) =>
|
||||
saveMode == RecordSaveMode.SingleFile &&
|
||||
outputFormat == RecordOutputFormat.Mp4 &&
|
||||
(saveMode == RecordSaveMode.SingleFile || saveMode == RecordSaveMode.Segmented) &&
|
||||
outputPath.EndsWith(".ts", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static string ResolveSegmentOutputPath(
|
||||
string outputPathPattern,
|
||||
RecordSaveMode saveMode,
|
||||
int segmentIndex)
|
||||
{
|
||||
if (saveMode != RecordSaveMode.Segmented)
|
||||
{
|
||||
return outputPathPattern;
|
||||
}
|
||||
|
||||
return outputPathPattern.Replace("%05d", $"{Math.Max(1, segmentIndex):D5}", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static string ResolveRecorderSegmentOutputPath(
|
||||
string outputPathPattern,
|
||||
RecordOutputFormat outputFormat,
|
||||
RecordSaveMode saveMode,
|
||||
int segmentIndex) =>
|
||||
NormalizeAbsolutePath(GetRecorderOutputPath(
|
||||
ResolveSegmentOutputPath(outputPathPattern, saveMode, segmentIndex),
|
||||
outputFormat,
|
||||
saveMode));
|
||||
|
||||
private static bool IsHttpInput(string streamUrl) =>
|
||||
streamUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ||
|
||||
streamUrl.StartsWith("https://", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static bool ShouldEnableReconnect(string streamUrl, string? selectedProtocol)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(selectedProtocol) &&
|
||||
selectedProtocol.Equals("hls", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return !streamUrl.Contains(".m3u8", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static string NormalizeAbsolutePath(string path) =>
|
||||
Path.IsPathRooted(path)
|
||||
? path
|
||||
: Path.GetFullPath(path, AppContext.BaseDirectory);
|
||||
|
||||
private static string? BuildCustomHeaderArgument(StreamInputHeaders? inputHeaders)
|
||||
{
|
||||
if (inputHeaders is null)
|
||||
@@ -376,13 +1026,95 @@ public sealed partial class FfmpegService
|
||||
return count;
|
||||
}
|
||||
|
||||
private static async Task WaitForFileToStabilizeAsync(string? path, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(path))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var absolutePath = NormalizeAbsolutePath(path);
|
||||
if (!File.Exists(absolutePath))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
timeoutCts.CancelAfter(TimeSpan.FromSeconds(4));
|
||||
|
||||
long? previousLength = null;
|
||||
DateTime previousWriteTimeUtc = default;
|
||||
var stableChecks = 0;
|
||||
|
||||
try
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
timeoutCts.Token.ThrowIfCancellationRequested();
|
||||
|
||||
var fileInfo = new FileInfo(absolutePath);
|
||||
if (!fileInfo.Exists)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (previousLength == fileInfo.Length && previousWriteTimeUtc == fileInfo.LastWriteTimeUtc)
|
||||
{
|
||||
stableChecks++;
|
||||
if (stableChecks >= 2)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
previousLength = fileInfo.Length;
|
||||
previousWriteTimeUtc = fileInfo.LastWriteTimeUtc;
|
||||
stableChecks = 0;
|
||||
}
|
||||
|
||||
await Task.Delay(250, timeoutCts.Token);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (timeoutCts.IsCancellationRequested)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsActiveTaskStatus(RecordTaskStatus status) =>
|
||||
status is RecordTaskStatus.Starting or RecordTaskStatus.Running or RecordTaskStatus.Stopping;
|
||||
|
||||
private static bool IsActiveSessionStatus(RecordSessionStatus status) =>
|
||||
status is RecordSessionStatus.Starting or RecordSessionStatus.Running or RecordSessionStatus.Stopping;
|
||||
|
||||
private static string Quote(string value) => $"\"{value.Replace("\"", "\\\"", StringComparison.Ordinal)}\"";
|
||||
private static bool TryParseFfmpegProgressSeconds(string line, out double seconds)
|
||||
{
|
||||
if (line.StartsWith("out_time=", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (TimeSpan.TryParse(
|
||||
line["out_time=".Length..],
|
||||
CultureInfo.InvariantCulture,
|
||||
out var timeSpan))
|
||||
{
|
||||
seconds = Math.Max(0, timeSpan.TotalSeconds);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if (line.StartsWith("out_time_ms=", StringComparison.OrdinalIgnoreCase) ||
|
||||
line.StartsWith("out_time_us=", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
var raw = line[(line.IndexOf('=') + 1)..];
|
||||
if (long.TryParse(raw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var value))
|
||||
{
|
||||
seconds = Math.Max(0, value / 1_000_000d);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
seconds = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
private enum FfmpegInputOptionProfile
|
||||
{
|
||||
|
||||
@@ -6,6 +6,8 @@ using LiveRecorder.Application.Abstractions.Notifications;
|
||||
using LiveRecorder.Application.Abstractions.Platforms;
|
||||
using LiveRecorder.Application.Abstractions.Recording;
|
||||
using LiveRecorder.Application.Abstractions.Settings;
|
||||
using LiveRecorder.Application.Abstractions.Storage;
|
||||
using LiveRecorder.Application.Services;
|
||||
using LiveRecorder.Domain.Entities;
|
||||
using LiveRecorder.Domain.Enums;
|
||||
using LiveRecorder.Infrastructure.Persistence;
|
||||
@@ -22,19 +24,46 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
|
||||
private readonly ConcurrentDictionary<Guid, SessionProcessRuntime> _processes = new();
|
||||
private readonly ConcurrentDictionary<Guid, PostProcessRuntimeEntry> _postProcessStates = new();
|
||||
private readonly object _transcodeConcurrencyLock = new();
|
||||
private readonly Queue<TaskCompletionSource<IDisposable>> _transcodeWaiters = new();
|
||||
private int _activeTranscodeTasks;
|
||||
private int _maxConcurrentTranscodeTasks = 1;
|
||||
private readonly IServiceScopeFactory _serviceScopeFactory;
|
||||
private readonly IStorageGuardService _storageGuardService;
|
||||
private readonly ILogger<FfmpegService> _logger;
|
||||
|
||||
public FfmpegService(
|
||||
IServiceScopeFactory serviceScopeFactory,
|
||||
IStorageGuardService storageGuardService,
|
||||
ILogger<FfmpegService> logger)
|
||||
{
|
||||
_serviceScopeFactory = serviceScopeFactory;
|
||||
_storageGuardService = storageGuardService;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public bool IsRunning(Guid recordSessionId) => _processes.ContainsKey(recordSessionId);
|
||||
|
||||
public IReadOnlyDictionary<Guid, RecordTaskRuntimeState> GetTaskRuntimeStates(IReadOnlyCollection<Guid> recordTaskIds)
|
||||
{
|
||||
if (recordTaskIds.Count == 0)
|
||||
{
|
||||
return new Dictionary<Guid, RecordTaskRuntimeState>();
|
||||
}
|
||||
|
||||
var snapshot = new Dictionary<Guid, RecordTaskRuntimeState>();
|
||||
foreach (var taskId in recordTaskIds)
|
||||
{
|
||||
if (_postProcessStates.TryGetValue(taskId, out var state))
|
||||
{
|
||||
snapshot[taskId] = state.State;
|
||||
}
|
||||
}
|
||||
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
public Task CompleteAsync(Guid recordSessionId, CancellationToken cancellationToken = default) =>
|
||||
RequestStopAsync(recordSessionId, markAsCompletedOnExit: true, cancellationToken);
|
||||
|
||||
@@ -42,11 +71,13 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
RecordSession recordSession,
|
||||
RecordTask initialTask,
|
||||
StreamUrlResult streamUrlResult,
|
||||
RecordingExecutionSettings recordingSettings,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
StartInternalAsync(
|
||||
recordSession,
|
||||
initialTask,
|
||||
streamUrlResult,
|
||||
recordingSettings,
|
||||
inputOptionProfile: FfmpegInputOptionProfile.Baseline,
|
||||
hasRetriedWithCompatibilityProfile: false,
|
||||
hasRetriedWithRefreshedStream: false,
|
||||
@@ -57,6 +88,7 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
RecordSession recordSession,
|
||||
RecordTask initialTask,
|
||||
StreamUrlResult streamUrlResult,
|
||||
RecordingExecutionSettings recordingSettings,
|
||||
FfmpegInputOptionProfile inputOptionProfile,
|
||||
bool hasRetriedWithCompatibilityProfile,
|
||||
bool hasRetriedWithRefreshedStream,
|
||||
@@ -66,6 +98,7 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
ArgumentNullException.ThrowIfNull(recordSession);
|
||||
ArgumentNullException.ThrowIfNull(initialTask);
|
||||
ArgumentNullException.ThrowIfNull(streamUrlResult);
|
||||
ArgumentNullException.ThrowIfNull(recordingSettings);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(streamUrlResult.SelectedUrl) || string.IsNullOrWhiteSpace(recordSession.OutputPathPattern))
|
||||
{
|
||||
@@ -106,10 +139,12 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
recordSession.SaveMode,
|
||||
initialTask.Id,
|
||||
Math.Max(1, initialTask.SegmentIndex),
|
||||
initialTask.OutputFilePath ?? outputPathPattern,
|
||||
ResolveRecorderSegmentOutputPath(outputPathPattern, recordSession.OutputFormat, recordSession.SaveMode, Math.Max(1, initialTask.SegmentIndex)),
|
||||
streamUrlResult.SelectedQuality,
|
||||
streamUrlResult.SelectedProtocol,
|
||||
streamUrlResult.SelectedVideoCodec,
|
||||
streamUrlResult.InputHeaders,
|
||||
recordingSettings,
|
||||
inputOptionProfile,
|
||||
hasRetriedWithCompatibilityProfile,
|
||||
hasRetriedWithRefreshedStream,
|
||||
@@ -120,12 +155,14 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
recorderOutputPath,
|
||||
recordSession.OutputFormat,
|
||||
recordSession.SaveMode,
|
||||
settings.RecordingTemplate,
|
||||
settings.EnableAutoReconnect,
|
||||
settings.ReconnectDelayMaxSeconds,
|
||||
settings.ReadWriteTimeoutMilliseconds,
|
||||
settings.SegmentDurationMinutes,
|
||||
recordingSettings.RecordingTemplate,
|
||||
recordingSettings.EnableAutoReconnect,
|
||||
recordingSettings.ReconnectDelayMaxSeconds,
|
||||
recordingSettings.ReadWriteTimeoutMilliseconds,
|
||||
recordingSettings.SegmentDurationMinutes,
|
||||
streamUrlResult.InputHeaders,
|
||||
streamUrlResult.SelectedProtocol,
|
||||
streamUrlResult.SelectedVideoCodec,
|
||||
inputOptionProfile))
|
||||
{
|
||||
process.StartInfo.ArgumentList.Add(argument);
|
||||
@@ -191,6 +228,7 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
{
|
||||
if (!process.HasExited)
|
||||
{
|
||||
runtime.MarkForceKilled();
|
||||
process.Kill(true);
|
||||
}
|
||||
}
|
||||
@@ -204,7 +242,7 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
|
||||
public async Task<bool> TryReconcileInactiveSessionAsync(Guid recordSessionId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (IsRunning(recordSessionId))
|
||||
if (IsRunning(recordSessionId) || IsSessionUnderPostProcessing(recordSessionId))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
@@ -232,37 +270,39 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
.ToList();
|
||||
|
||||
var anyUsableOutput = false;
|
||||
string? finalizationError = null;
|
||||
string? sessionFinalizationError = null;
|
||||
|
||||
foreach (var task in tasks.Where(item => IsActiveTaskStatus(item.Status)))
|
||||
{
|
||||
var effectiveOutputPath = task.OutputFilePath ?? string.Empty;
|
||||
if (recordSession.SaveMode == RecordSaveMode.SingleFile &&
|
||||
recordSession.OutputFormat == RecordOutputFormat.Mp4 &&
|
||||
!string.IsNullOrWhiteSpace(recordSession.OutputPathPattern))
|
||||
var durationSeconds = task.StartedAt.HasValue
|
||||
? (double?)Math.Max(0, (endedAt - task.StartedAt.Value).TotalSeconds)
|
||||
: null;
|
||||
var finalizationResult = await TryFinalizeTaskOutputAsync(
|
||||
settings.FfmpegPath,
|
||||
settings.MaxConcurrentFfmpegTranscodeTasks,
|
||||
settings.Mp4FinalizeTimeoutMinutes,
|
||||
recordSession,
|
||||
task,
|
||||
durationSeconds,
|
||||
cancellationToken);
|
||||
var effectiveOutputPath = finalizationResult.OutputPath;
|
||||
var taskFinalizationError = finalizationResult.ErrorMessage;
|
||||
if (!IsLowStoragePauseError(taskFinalizationError))
|
||||
{
|
||||
var finalOutputPath = Path.IsPathRooted(recordSession.OutputPathPattern)
|
||||
? recordSession.OutputPathPattern
|
||||
: Path.GetFullPath(recordSession.OutputPathPattern, AppContext.BaseDirectory);
|
||||
var recorderOutputPath = GetRecorderOutputPath(finalOutputPath, recordSession.OutputFormat, recordSession.SaveMode);
|
||||
if (File.Exists(recorderOutputPath))
|
||||
{
|
||||
var finalizationResult = await TryFinalizeMp4Async(settings.FfmpegPath, recorderOutputPath, finalOutputPath);
|
||||
effectiveOutputPath = finalizationResult.OutputPath;
|
||||
finalizationError ??= finalizationResult.ErrorMessage;
|
||||
}
|
||||
sessionFinalizationError ??= taskFinalizationError;
|
||||
}
|
||||
|
||||
var fileSize = CalculateFileSize(effectiveOutputPath);
|
||||
var danmakuPath = task.Result?.DanmakuFilePath ?? GuessDanmakuPath(task.OutputFilePath);
|
||||
var danmakuCount = CountDanmakuMessages(danmakuPath);
|
||||
var durationSeconds = task.StartedAt.HasValue
|
||||
? (double?)Math.Max(0, (endedAt - task.StartedAt.Value).TotalSeconds)
|
||||
: null;
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(finalizationError))
|
||||
if (IsLowStoragePauseError(taskFinalizationError))
|
||||
{
|
||||
task.MarkFailed(finalizationError, endedAt);
|
||||
task.MarkProcessing(taskFinalizationError, endedAt);
|
||||
}
|
||||
else if (!string.IsNullOrWhiteSpace(taskFinalizationError))
|
||||
{
|
||||
task.MarkFailed(taskFinalizationError, endedAt);
|
||||
}
|
||||
else if (HasUsableOutput(effectiveOutputPath, fileSize))
|
||||
{
|
||||
@@ -274,13 +314,14 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
task.MarkStopped(endedAt, durationSeconds, "Recording process was no longer running when the session was reconciled.");
|
||||
}
|
||||
|
||||
UpsertRecordResult(task, dbContext, effectiveOutputPath, fileSize, durationSeconds, danmakuPath, danmakuCount, endedAt);
|
||||
await UpsertRecordResultAsync(task, dbContext, effectiveOutputPath, fileSize, durationSeconds, danmakuPath, danmakuCount, endedAt, cancellationToken);
|
||||
ClearPostProcessState(task.Id);
|
||||
}
|
||||
|
||||
recordSession.SyncSegmentCount(tasks.Count, endedAt);
|
||||
if (!string.IsNullOrWhiteSpace(finalizationError))
|
||||
if (!string.IsNullOrWhiteSpace(sessionFinalizationError))
|
||||
{
|
||||
recordSession.MarkFailed(finalizationError, endedAt);
|
||||
recordSession.MarkFailed(sessionFinalizationError, endedAt);
|
||||
}
|
||||
else if (anyUsableOutput)
|
||||
{
|
||||
@@ -295,6 +336,176 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
return true;
|
||||
}
|
||||
|
||||
public async Task<bool> StartManualFinalizeTaskAsync(Guid recordTaskId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (_postProcessStates.ContainsKey(recordTaskId))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
using var scope = _serviceScopeFactory.CreateScope();
|
||||
var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
|
||||
var recordTask = await dbContext.RecordTasks
|
||||
.Include(item => item.RecordSession)
|
||||
.Include(item => item.Result)
|
||||
.FirstOrDefaultAsync(item => item.Id == recordTaskId, cancellationToken);
|
||||
|
||||
if (recordTask?.RecordSession is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (recordTask.OutputFormat != RecordOutputFormat.Mp4 ||
|
||||
IsActiveTaskStatus(recordTask.Status) ||
|
||||
IsRunning(recordTask.RecordSessionId))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(recordTask.OutputFilePath))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var recorderOutputPath = ResolveManualFinalizeSourcePath(recordTask, recordTask.RecordSession);
|
||||
if (!File.Exists(recorderOutputPath))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
SetPostProcessState(
|
||||
recordTask.RecordSessionId,
|
||||
recordTask.Id,
|
||||
"Queued",
|
||||
null,
|
||||
$"Manual MP4 finalization queued for {Path.GetFileName(recordTask.OutputFilePath)}");
|
||||
|
||||
_ = Task.Run(
|
||||
async () => await RunManualFinalizeTaskAsync(recordTask.Id),
|
||||
CancellationToken.None);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public async Task<int> ResumePausedFinalizationsAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
using var scope = _serviceScopeFactory.CreateScope();
|
||||
var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
|
||||
var settingsService = scope.ServiceProvider.GetRequiredService<ISystemSettingsService>();
|
||||
var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
|
||||
var settings = await settingsService.GetAsync(cancellationToken);
|
||||
|
||||
var storageCheck = _storageGuardService.CheckCanStartOrResume(settings);
|
||||
if (!storageCheck.HasEnoughSpace)
|
||||
{
|
||||
await logService.WriteAsync(
|
||||
SystemLogLevel.Warning,
|
||||
"Storage",
|
||||
"Paused MP4 finalization remains blocked because storage is below threshold.",
|
||||
storageCheck.Message,
|
||||
cancellationToken: cancellationToken);
|
||||
return 0;
|
||||
}
|
||||
|
||||
var candidates = await dbContext.RecordTasks
|
||||
.Include(item => item.RecordSession)
|
||||
.Include(item => item.Result)
|
||||
.Where(item => item.OutputFormat == RecordOutputFormat.Mp4 &&
|
||||
(item.Status == RecordTaskStatus.Processing ||
|
||||
item.Status == RecordTaskStatus.Completed))
|
||||
.OrderBy(static item => item.UpdatedAt)
|
||||
.Take(100)
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
var queuedTaskIds = new List<Guid>();
|
||||
var repairedInterruptedTasks = 0;
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
if (candidate.Status == RecordTaskStatus.Processing)
|
||||
{
|
||||
queuedTaskIds.Add(candidate.Id);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!NeedsInterruptedMp4Finalization(candidate))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
candidate.MarkProcessing("MP4 finalization was interrupted before completion. Re-queued after restart.", now);
|
||||
queuedTaskIds.Add(candidate.Id);
|
||||
repairedInterruptedTasks++;
|
||||
}
|
||||
|
||||
if (repairedInterruptedTasks > 0)
|
||||
{
|
||||
await dbContext.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
var started = 0;
|
||||
foreach (var taskId in queuedTaskIds.Take(20))
|
||||
{
|
||||
if (await StartManualFinalizeTaskAsync(taskId, cancellationToken))
|
||||
{
|
||||
started++;
|
||||
}
|
||||
}
|
||||
|
||||
if (started > 0)
|
||||
{
|
||||
await logService.WriteAsync(
|
||||
SystemLogLevel.Info,
|
||||
"Storage",
|
||||
"Storage is available. Resumed paused MP4 finalization tasks.",
|
||||
$"count={started}; repairedInterrupted={repairedInterruptedTasks}; {storageCheck.Message}",
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
return started;
|
||||
}
|
||||
|
||||
private static bool NeedsInterruptedMp4Finalization(RecordTask recordTask)
|
||||
{
|
||||
if (recordTask.Status != RecordTaskStatus.Completed ||
|
||||
recordTask.RecordSession is null ||
|
||||
recordTask.OutputFormat != RecordOutputFormat.Mp4 ||
|
||||
recordTask.RecordSession.OutputFormat != RecordOutputFormat.Mp4 ||
|
||||
string.IsNullOrWhiteSpace(recordTask.OutputFilePath))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var finalOutputPath = NormalizeAbsolutePath(recordTask.OutputFilePath);
|
||||
if (HasUsableOutput(finalOutputPath, CalculateFileSize(finalOutputPath)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var recorderOutputPath = ResolveManualFinalizeSourcePath(recordTask, recordTask.RecordSession);
|
||||
return File.Exists(recorderOutputPath);
|
||||
}
|
||||
|
||||
private static string ResolveManualFinalizeSourcePath(RecordTask recordTask, RecordSession recordSession)
|
||||
{
|
||||
var resultPath = recordTask.Result?.FilePath;
|
||||
if (!string.IsNullOrWhiteSpace(resultPath) &&
|
||||
resultPath.EndsWith(".ts", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
var normalizedResultPath = NormalizeAbsolutePath(resultPath);
|
||||
if (File.Exists(normalizedResultPath))
|
||||
{
|
||||
return normalizedResultPath;
|
||||
}
|
||||
}
|
||||
|
||||
return NormalizeAbsolutePath(
|
||||
GetRecorderOutputPath(
|
||||
recordTask.OutputFilePath ?? resultPath ?? string.Empty,
|
||||
recordSession.OutputFormat,
|
||||
recordSession.SaveMode));
|
||||
}
|
||||
|
||||
private async Task PersistStartupProfileAsync(SessionProcessRuntime runtime, CancellationToken cancellationToken)
|
||||
{
|
||||
using var scope = _serviceScopeFactory.CreateScope();
|
||||
@@ -303,7 +514,7 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
SystemLogLevel.Info,
|
||||
"FFmpeg",
|
||||
$"ffmpeg input profile={runtime.InputOptionProfile}.",
|
||||
detail: $"stream={runtime.SelectedProtocol}:{runtime.SelectedQuality}; compatibilityRetry={runtime.HasRetriedWithCompatibilityProfile}; refreshRetry={runtime.HasRetriedWithRefreshedStream}",
|
||||
detail: $"stream={runtime.SelectedProtocol}:{runtime.SelectedQuality}; videoCodec={runtime.SelectedVideoCodec ?? "unknown"}; compatibilityRetry={runtime.HasRetriedWithCompatibilityProfile}; refreshRetry={runtime.HasRetriedWithRefreshedStream}",
|
||||
liveRoomId: runtime.LiveRoomId,
|
||||
recordSessionId: runtime.RecordSessionId,
|
||||
recordTaskId: runtime.CurrentTaskId,
|
||||
@@ -351,29 +562,107 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
|
||||
await process.StandardInput.WriteLineAsync("q");
|
||||
await process.StandardInput.FlushAsync();
|
||||
|
||||
using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
timeoutCts.CancelAfter(TimeSpan.FromSeconds(12));
|
||||
|
||||
try
|
||||
{
|
||||
await process.WaitForExitAsync(timeoutCts.Token);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
if (!process.HasExited)
|
||||
{
|
||||
process.Kill(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Stop ffmpeg process failed for session {RecordSessionId}", recordSessionId);
|
||||
}
|
||||
}
|
||||
|
||||
if (!process.HasExited)
|
||||
private void SetPostProcessState(
|
||||
Guid recordSessionId,
|
||||
Guid recordTaskId,
|
||||
string stage,
|
||||
double? progressPercent,
|
||||
string? detail = null)
|
||||
{
|
||||
var normalizedProgress = progressPercent.HasValue
|
||||
? Math.Clamp(progressPercent.Value, 0d, 100d)
|
||||
: (double?)null;
|
||||
_postProcessStates[recordTaskId] = new PostProcessRuntimeEntry(
|
||||
recordSessionId,
|
||||
new RecordTaskRuntimeState(
|
||||
RecordTaskStatus.Processing,
|
||||
stage,
|
||||
normalizedProgress,
|
||||
detail));
|
||||
}
|
||||
|
||||
private void ClearPostProcessState(Guid recordTaskId) =>
|
||||
_postProcessStates.TryRemove(recordTaskId, out _);
|
||||
|
||||
private bool IsSessionUnderPostProcessing(Guid recordSessionId) =>
|
||||
_postProcessStates.Values.Any(entry => entry.RecordSessionId == recordSessionId);
|
||||
|
||||
private async Task<IDisposable> AcquireTranscodeSlotAsync(int maxConcurrentTasks, CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedMaxConcurrentTasks = Math.Clamp(maxConcurrentTasks, 1, 16);
|
||||
TaskCompletionSource<IDisposable> waiter;
|
||||
|
||||
lock (_transcodeConcurrencyLock)
|
||||
{
|
||||
_maxConcurrentTranscodeTasks = normalizedMaxConcurrentTasks;
|
||||
if (_transcodeWaiters.Count == 0 && _activeTranscodeTasks < _maxConcurrentTranscodeTasks)
|
||||
{
|
||||
process.Kill(true);
|
||||
_activeTranscodeTasks++;
|
||||
return new TranscodeSlotLease(this);
|
||||
}
|
||||
|
||||
waiter = new TaskCompletionSource<IDisposable>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
_transcodeWaiters.Enqueue(waiter);
|
||||
}
|
||||
|
||||
using var cancellationRegistration = cancellationToken.Register(
|
||||
static state => ((TaskCompletionSource<IDisposable>)state!).TrySetCanceled(),
|
||||
waiter);
|
||||
return await waiter.Task.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private void ReleaseTranscodeSlot()
|
||||
{
|
||||
lock (_transcodeConcurrencyLock)
|
||||
{
|
||||
if (_activeTranscodeTasks > 0)
|
||||
{
|
||||
_activeTranscodeTasks--;
|
||||
}
|
||||
|
||||
while (_activeTranscodeTasks < _maxConcurrentTranscodeTasks && _transcodeWaiters.Count > 0)
|
||||
{
|
||||
var waiter = _transcodeWaiters.Dequeue();
|
||||
if (waiter.Task.IsCompleted)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
_activeTranscodeTasks++;
|
||||
if (waiter.TrySetResult(new TranscodeSlotLease(this)))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_activeTranscodeTasks--;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record PostProcessRuntimeEntry(Guid RecordSessionId, RecordTaskRuntimeState State);
|
||||
|
||||
private sealed class TranscodeSlotLease : IDisposable
|
||||
{
|
||||
private readonly FfmpegService _owner;
|
||||
private int _disposed;
|
||||
|
||||
public TranscodeSlotLease(FfmpegService owner)
|
||||
{
|
||||
_owner = owner;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _disposed, 1) == 0)
|
||||
{
|
||||
_owner.ReleaseTranscodeSlot();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
using System.Collections.Concurrent;
|
||||
using LiveRecorder.Application.Abstractions.Logging;
|
||||
using LiveRecorder.Application.Abstractions.Notifications;
|
||||
using LiveRecorder.Application.Abstractions.Platforms;
|
||||
using LiveRecorder.Application.Abstractions.Recording;
|
||||
using LiveRecorder.Application.Abstractions.Settings;
|
||||
using LiveRecorder.Application.Abstractions.Storage;
|
||||
using LiveRecorder.Application.Models.Settings;
|
||||
using LiveRecorder.Application.Models.RecordTasks;
|
||||
using LiveRecorder.Application.Services;
|
||||
using LiveRecorder.Domain.Enums;
|
||||
using LiveRecorder.Infrastructure.Persistence;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
@@ -16,8 +20,13 @@ namespace LiveRecorder.Infrastructure.Services;
|
||||
|
||||
public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
{
|
||||
private static readonly TimeSpan OfflineGracefulStopTimeout = TimeSpan.FromSeconds(20);
|
||||
private static readonly TimeSpan OfflineForcedStopTimeout = TimeSpan.FromSeconds(8);
|
||||
private static readonly TimeSpan ExceptionEmailCooldown = TimeSpan.FromHours(6);
|
||||
|
||||
private readonly IServiceScopeFactory _serviceScopeFactory;
|
||||
private readonly ILogger<LiveRoomPollingBackgroundService> _logger;
|
||||
private readonly ConcurrentDictionary<string, DateTimeOffset> _exceptionEmailSentAt = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public LiveRoomPollingBackgroundService(
|
||||
IServiceScopeFactory serviceScopeFactory,
|
||||
@@ -38,9 +47,15 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
using var scope = _serviceScopeFactory.CreateScope();
|
||||
var settingsService = scope.ServiceProvider.GetRequiredService<ISystemSettingsService>();
|
||||
var settings = await settingsService.GetAsync(stoppingToken);
|
||||
var emailNotificationService = scope.ServiceProvider.GetRequiredService<IEmailNotificationService>();
|
||||
|
||||
delaySeconds = settings.PollingIntervalSeconds;
|
||||
var ffmpegService = scope.ServiceProvider.GetRequiredService<IFfmpegService>();
|
||||
var storageGuardService = scope.ServiceProvider.GetRequiredService<IStorageGuardService>();
|
||||
if (storageGuardService.CheckCanStartOrResume(settings).HasEnoughSpace)
|
||||
{
|
||||
await ffmpegService.ResumePausedFinalizationsAsync(stoppingToken);
|
||||
}
|
||||
|
||||
if (!settings.EnableBackgroundPolling)
|
||||
{
|
||||
await DelayAsync(delaySeconds, stoppingToken);
|
||||
@@ -48,104 +63,23 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
}
|
||||
|
||||
var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
|
||||
var adapterFactory = scope.ServiceProvider.GetRequiredService<ILivePlatformAdapterFactory>();
|
||||
var ffmpegService = scope.ServiceProvider.GetRequiredService<IFfmpegService>();
|
||||
var recordService = scope.ServiceProvider.GetRequiredService<RecordService>();
|
||||
var liveRoomStatusService = scope.ServiceProvider.GetRequiredService<LiveRoomStatusService>();
|
||||
var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
|
||||
|
||||
var liveRooms = (await dbContext.LiveRooms.ToListAsync(stoppingToken))
|
||||
.Where(static item => item.IsEnabled)
|
||||
var liveRoomIds = (await dbContext.LiveRooms
|
||||
.AsNoTracking()
|
||||
.Where(static item => item.IsEnabled)
|
||||
.Select(static item => new { item.Id, item.UpdatedAt })
|
||||
.ToListAsync(stoppingToken))
|
||||
.OrderBy(static item => item.UpdatedAt)
|
||||
.Select(static item => item.Id)
|
||||
.ToList();
|
||||
|
||||
foreach (var liveRoom in liveRooms)
|
||||
foreach (var liveRoomId in liveRoomIds)
|
||||
{
|
||||
if (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var adapter = adapterFactory.GetByPlatform(liveRoom.Platform);
|
||||
var liveStatus = await adapter.GetLiveStatusAsync(liveRoom.RoomId, stoppingToken);
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
await liveRoomStatusService.ApplySnapshotAsync(liveRoom, liveStatus, now, stoppingToken);
|
||||
|
||||
await dbContext.SaveChangesAsync(stoppingToken);
|
||||
|
||||
if (!liveStatus.IsLive)
|
||||
{
|
||||
await CompleteActiveSessionsForOfflineRoomAsync(
|
||||
dbContext,
|
||||
ffmpegService,
|
||||
logService,
|
||||
liveRoom.Id,
|
||||
stoppingToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!settings.AutoStartRecordingOnLive)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var hasRunningSession = await dbContext.RecordSessions.AnyAsync(
|
||||
item => item.LiveRoomId == liveRoom.Id &&
|
||||
(item.Status == RecordSessionStatus.Starting ||
|
||||
item.Status == RecordSessionStatus.Running ||
|
||||
item.Status == RecordSessionStatus.Stopping),
|
||||
stoppingToken);
|
||||
|
||||
if (hasRunningSession)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
_logger.LogInformation("Auto-start recording for live room {RoomId}", liveRoom.RoomId);
|
||||
await logService.WriteAsync(
|
||||
SystemLogLevel.Info,
|
||||
"Scheduler",
|
||||
"Live detected by background poller. Auto-starting recording task.",
|
||||
liveRoomId: liveRoom.Id,
|
||||
cancellationToken: stoppingToken);
|
||||
|
||||
await recordService.StartAsync(
|
||||
new StartRecordTaskRequest
|
||||
{
|
||||
LiveRoomId = liveRoom.Id,
|
||||
PreferredQuality = settings.DefaultQuality,
|
||||
OutputFormat = settings.DefaultOutputFormat
|
||||
},
|
||||
stoppingToken);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Polling room {RoomId} failed", liveRoom.RoomId);
|
||||
|
||||
var isTransient = IsTransientPollingException(ex, stoppingToken);
|
||||
await logService.WriteAsync(
|
||||
isTransient ? SystemLogLevel.Warning : SystemLogLevel.Error,
|
||||
"Scheduler",
|
||||
isTransient
|
||||
? "Transient background polling failure. The room will be retried on the next cycle."
|
||||
: "Background polling failed for a live room.",
|
||||
ex.ToString(),
|
||||
liveRoomId: liveRoom.Id,
|
||||
cancellationToken: stoppingToken);
|
||||
|
||||
if (!isTransient)
|
||||
{
|
||||
await emailNotificationService.SendExceptionAsync(
|
||||
"Scheduler",
|
||||
"Background polling failed for a live room.",
|
||||
ex.ToString(),
|
||||
liveRoom,
|
||||
cancellationToken: stoppingToken);
|
||||
}
|
||||
}
|
||||
await PollLiveRoomAsync(liveRoomId, settings, stoppingToken);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
@@ -160,11 +94,14 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
{
|
||||
using var notificationScope = _serviceScopeFactory.CreateScope();
|
||||
var emailNotificationService = notificationScope.ServiceProvider.GetRequiredService<IEmailNotificationService>();
|
||||
await emailNotificationService.SendExceptionAsync(
|
||||
"Scheduler",
|
||||
"Background live room polling failed.",
|
||||
ex.ToString(),
|
||||
cancellationToken: stoppingToken);
|
||||
if (ShouldSendExceptionEmail(BuildExceptionEmailKey("background-loop", ex)))
|
||||
{
|
||||
await emailNotificationService.SendExceptionAsync(
|
||||
"Scheduler",
|
||||
"Background live room polling failed.",
|
||||
ex.ToString(),
|
||||
cancellationToken: stoppingToken);
|
||||
}
|
||||
}
|
||||
catch (Exception notificationEx)
|
||||
{
|
||||
@@ -179,6 +116,128 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
private static Task DelayAsync(int delaySeconds, CancellationToken cancellationToken) =>
|
||||
Task.Delay(TimeSpan.FromSeconds(Math.Clamp(delaySeconds, 10, 3600)), cancellationToken);
|
||||
|
||||
private async Task PollLiveRoomAsync(Guid liveRoomId, SystemSettingsDto settings, CancellationToken cancellationToken)
|
||||
{
|
||||
using var scope = _serviceScopeFactory.CreateScope();
|
||||
var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
|
||||
var adapterFactory = scope.ServiceProvider.GetRequiredService<ILivePlatformAdapterFactory>();
|
||||
var ffmpegService = scope.ServiceProvider.GetRequiredService<IFfmpegService>();
|
||||
var recordService = scope.ServiceProvider.GetRequiredService<RecordService>();
|
||||
var liveRoomStatusService = scope.ServiceProvider.GetRequiredService<LiveRoomStatusService>();
|
||||
var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
|
||||
var emailNotificationService = scope.ServiceProvider.GetRequiredService<IEmailNotificationService>();
|
||||
var storageGuardService = scope.ServiceProvider.GetRequiredService<IStorageGuardService>();
|
||||
|
||||
var liveRoom = await dbContext.LiveRooms.FirstOrDefaultAsync(item => item.Id == liveRoomId, cancellationToken);
|
||||
if (liveRoom is null || !liveRoom.IsEnabled)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var adapter = adapterFactory.GetByPlatform(liveRoom.Platform);
|
||||
var liveStatus = await adapter.GetLiveStatusAsync(liveRoom.RoomId, cancellationToken);
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
await liveRoomStatusService.ApplySnapshotAsync(liveRoom, liveStatus, now, cancellationToken);
|
||||
await SaveChangesWithRetryAsync(dbContext, cancellationToken);
|
||||
|
||||
if (!liveStatus.IsLive)
|
||||
{
|
||||
await CompleteActiveSessionsForOfflineRoomAsync(
|
||||
dbContext,
|
||||
ffmpegService,
|
||||
logService,
|
||||
liveRoom.Id,
|
||||
cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
var pauseCheck = storageGuardService.CheckShouldPause(settings);
|
||||
if (!pauseCheck.HasEnoughSpace)
|
||||
{
|
||||
await PauseActiveSessionsForLowStorageAsync(
|
||||
dbContext,
|
||||
ffmpegService,
|
||||
logService,
|
||||
liveRoom.Id,
|
||||
pauseCheck.Message,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
if (!settings.AutoStartRecordingOnLive)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var startCheck = storageGuardService.CheckCanStartOrResume(settings);
|
||||
if (!startCheck.HasEnoughSpace)
|
||||
{
|
||||
await logService.WriteAsync(
|
||||
SystemLogLevel.Warning,
|
||||
"Storage",
|
||||
"Auto-start recording skipped because storage is below resume threshold.",
|
||||
startCheck.Message,
|
||||
liveRoomId: liveRoom.Id,
|
||||
cancellationToken: cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
var hasRunningSession = await dbContext.RecordSessions.AnyAsync(
|
||||
item => item.LiveRoomId == liveRoom.Id &&
|
||||
(item.Status == RecordSessionStatus.Starting ||
|
||||
item.Status == RecordSessionStatus.Running ||
|
||||
item.Status == RecordSessionStatus.Stopping),
|
||||
cancellationToken);
|
||||
|
||||
if (hasRunningSession)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation("Auto-start recording for live room {RoomId}", liveRoom.RoomId);
|
||||
await logService.WriteAsync(
|
||||
SystemLogLevel.Info,
|
||||
"Scheduler",
|
||||
"Live detected by background poller. Auto-starting recording task.",
|
||||
liveRoomId: liveRoom.Id,
|
||||
cancellationToken: cancellationToken);
|
||||
|
||||
await recordService.StartAsync(
|
||||
new StartRecordTaskRequest
|
||||
{
|
||||
LiveRoomId = liveRoom.Id
|
||||
},
|
||||
cancellationToken);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Polling room {RoomId} failed", liveRoom.RoomId);
|
||||
|
||||
var isTransient = IsTransientPollingException(ex, cancellationToken);
|
||||
await logService.WriteAsync(
|
||||
isTransient ? SystemLogLevel.Warning : SystemLogLevel.Error,
|
||||
"Scheduler",
|
||||
isTransient
|
||||
? "Transient background polling failure. The room will be retried on the next cycle."
|
||||
: "Background polling failed for a live room.",
|
||||
ex.ToString(),
|
||||
liveRoomId: liveRoom.Id,
|
||||
cancellationToken: cancellationToken);
|
||||
|
||||
if (!isTransient && ShouldSendExceptionEmail(BuildExceptionEmailKey("live-room-poll", ex, liveRoom.Id)))
|
||||
{
|
||||
await emailNotificationService.SendExceptionAsync(
|
||||
"Scheduler",
|
||||
"Background polling failed for a live room.",
|
||||
ex.ToString(),
|
||||
liveRoom,
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task CompleteActiveSessionsForOfflineRoomAsync(
|
||||
LiveRecorderDbContext dbContext,
|
||||
IFfmpegService ffmpegService,
|
||||
@@ -201,7 +260,40 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
{
|
||||
if (ffmpegService.IsRunning(activeSession.Id))
|
||||
{
|
||||
await ffmpegService.CompleteAsync(activeSession.Id, cancellationToken);
|
||||
var stopped = await ffmpegService.StopAndWaitAsync(
|
||||
activeSession.Id,
|
||||
markAsCompletedOnExit: true,
|
||||
OfflineGracefulStopTimeout,
|
||||
cancellationToken);
|
||||
|
||||
if (!stopped)
|
||||
{
|
||||
await logService.WriteAsync(
|
||||
SystemLogLevel.Warning,
|
||||
"Scheduler",
|
||||
"Live room is offline, but the recorder did not stop gracefully in time. Force killing the ffmpeg process.",
|
||||
liveRoomId: liveRoomId,
|
||||
recordSessionId: activeSession.Id,
|
||||
cancellationToken: cancellationToken);
|
||||
|
||||
var killed = await ffmpegService.KillAndWaitAsync(
|
||||
activeSession.Id,
|
||||
OfflineForcedStopTimeout,
|
||||
cancellationToken);
|
||||
if (!killed)
|
||||
{
|
||||
await logService.WriteAsync(
|
||||
SystemLogLevel.Error,
|
||||
"Scheduler",
|
||||
"Live room is offline, but the active recording session is still shutting down.",
|
||||
liveRoomId: liveRoomId,
|
||||
recordSessionId: activeSession.Id,
|
||||
cancellationToken: cancellationToken);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
await ffmpegService.TryReconcileInactiveSessionAsync(activeSession.Id, cancellationToken);
|
||||
await logService.WriteAsync(
|
||||
SystemLogLevel.Info,
|
||||
"Scheduler",
|
||||
@@ -225,6 +317,57 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task PauseActiveSessionsForLowStorageAsync(
|
||||
LiveRecorderDbContext dbContext,
|
||||
IFfmpegService ffmpegService,
|
||||
ISystemLogService logService,
|
||||
Guid liveRoomId,
|
||||
string detail,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var activeSessions = await dbContext.RecordSessions
|
||||
.Where(item => item.LiveRoomId == liveRoomId &&
|
||||
(item.Status == RecordSessionStatus.Starting ||
|
||||
item.Status == RecordSessionStatus.Running ||
|
||||
item.Status == RecordSessionStatus.Stopping))
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
foreach (var activeSession in activeSessions.OrderBy(static item => item.CreatedAt))
|
||||
{
|
||||
await logService.WriteAsync(
|
||||
SystemLogLevel.Warning,
|
||||
"Storage",
|
||||
"Storage is below threshold. Pausing active recording session.",
|
||||
detail,
|
||||
liveRoomId: liveRoomId,
|
||||
recordSessionId: activeSession.Id,
|
||||
cancellationToken: cancellationToken);
|
||||
|
||||
if (ffmpegService.IsRunning(activeSession.Id))
|
||||
{
|
||||
var stopped = await ffmpegService.StopAndWaitAsync(
|
||||
activeSession.Id,
|
||||
markAsCompletedOnExit: false,
|
||||
OfflineGracefulStopTimeout,
|
||||
cancellationToken);
|
||||
if (!stopped)
|
||||
{
|
||||
await logService.WriteAsync(
|
||||
SystemLogLevel.Warning,
|
||||
"Storage",
|
||||
"Recorder did not stop gracefully after low storage pause. Force killing the ffmpeg process.",
|
||||
detail,
|
||||
liveRoomId,
|
||||
activeSession.Id,
|
||||
cancellationToken: cancellationToken);
|
||||
await ffmpegService.KillAndWaitAsync(activeSession.Id, OfflineForcedStopTimeout, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
await ffmpegService.TryReconcileInactiveSessionAsync(activeSession.Id, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsTransientPollingException(Exception exception, CancellationToken cancellationToken)
|
||||
{
|
||||
if (exception is OperationCanceledException && cancellationToken.IsCancellationRequested)
|
||||
@@ -237,7 +380,99 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
return true;
|
||||
}
|
||||
|
||||
if (exception is DbUpdateException dbUpdateException &&
|
||||
IsSqliteLockException(dbUpdateException))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (exception is SqliteException sqliteException &&
|
||||
IsSqliteLockException(sqliteException))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return exception.InnerException is not null &&
|
||||
IsTransientPollingException(exception.InnerException, cancellationToken);
|
||||
}
|
||||
|
||||
private bool ShouldSendExceptionEmail(string key)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
while (true)
|
||||
{
|
||||
if (_exceptionEmailSentAt.TryGetValue(key, out var lastSentAt))
|
||||
{
|
||||
if (now - lastSentAt < ExceptionEmailCooldown)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Suppressed repeated scheduler exception email. Key={Key}; CooldownMinutes={CooldownMinutes}",
|
||||
key,
|
||||
ExceptionEmailCooldown.TotalMinutes);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (_exceptionEmailSentAt.TryUpdate(key, now, lastSentAt))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (_exceptionEmailSentAt.TryAdd(key, now))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static string BuildExceptionEmailKey(string scope, Exception exception, Guid? liveRoomId = null)
|
||||
{
|
||||
if (IsSqliteStorageFullException(exception))
|
||||
{
|
||||
return $"{scope}:sqlite-storage-full";
|
||||
}
|
||||
|
||||
var root = exception.GetBaseException();
|
||||
var message = root.Message.Length > 160 ? root.Message[..160] : root.Message;
|
||||
return $"{scope}:{liveRoomId?.ToString() ?? "global"}:{root.GetType().FullName}:{message}";
|
||||
}
|
||||
|
||||
private static bool IsSqliteStorageFullException(Exception exception)
|
||||
{
|
||||
if (exception is SqliteException sqliteException &&
|
||||
(sqliteException.SqliteErrorCode == 13 ||
|
||||
sqliteException.Message.Contains("database or disk is full", StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return exception.InnerException is not null && IsSqliteStorageFullException(exception.InnerException);
|
||||
}
|
||||
|
||||
private static async Task SaveChangesWithRetryAsync(LiveRecorderDbContext dbContext, CancellationToken cancellationToken)
|
||||
{
|
||||
for (var attempt = 1; attempt <= 5; attempt++)
|
||||
{
|
||||
try
|
||||
{
|
||||
await dbContext.SaveChangesAsync(cancellationToken);
|
||||
return;
|
||||
}
|
||||
catch (DbUpdateException ex) when (attempt < 5 && IsSqliteLockException(ex))
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(300 * Math.Pow(2, attempt - 1)), cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsSqliteLockException(DbUpdateException exception) =>
|
||||
exception.InnerException is SqliteException sqliteException &&
|
||||
IsSqliteLockException(sqliteException);
|
||||
|
||||
private static bool IsSqliteLockException(SqliteException exception) =>
|
||||
exception.SqliteErrorCode is 5 or 6 ||
|
||||
exception.Message.Contains("database is locked", StringComparison.OrdinalIgnoreCase) ||
|
||||
exception.Message.Contains("database table is locked", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
@@ -29,9 +29,9 @@ public sealed class RecordMediaService : IRecordMediaService
|
||||
var recordTask = await _recordTaskRepository.GetByIdAsync(recordTaskId, cancellationToken)
|
||||
?? throw new KeyNotFoundException("Recording task was not found.");
|
||||
|
||||
if (recordTask.Status != RecordTaskStatus.Completed)
|
||||
if (recordTask.Status is not (RecordTaskStatus.Completed or RecordTaskStatus.Stopped))
|
||||
{
|
||||
throw new InvalidOperationException("Preview is only available for completed tasks.");
|
||||
throw new InvalidOperationException("Preview is only available for finalized MP4 tasks.");
|
||||
}
|
||||
|
||||
if (recordTask.OutputFormat != RecordOutputFormat.Mp4)
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
using LiveRecorder.Application.Abstractions.Storage;
|
||||
using LiveRecorder.Application.Models.Settings;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace LiveRecorder.Infrastructure.Services;
|
||||
|
||||
public sealed class StorageGuardService : IStorageGuardService
|
||||
{
|
||||
private const long Megabyte = 1024L * 1024L;
|
||||
private readonly ILogger<StorageGuardService> _logger;
|
||||
|
||||
public StorageGuardService(ILogger<StorageGuardService> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public StorageGuardResult CheckCanStartOrResume(SystemSettingsDto settings, long additionalRequiredBytes = 0) =>
|
||||
Check(settings, Math.Max(settings.PauseRecordingWhenFreeSpaceBelowMegabytes, settings.ResumeRecordingWhenFreeSpaceAboveMegabytes), additionalRequiredBytes);
|
||||
|
||||
public StorageGuardResult CheckShouldPause(SystemSettingsDto settings) =>
|
||||
Check(settings, settings.PauseRecordingWhenFreeSpaceBelowMegabytes, additionalRequiredBytes: 0);
|
||||
|
||||
private StorageGuardResult Check(SystemSettingsDto settings, int freeSpaceThresholdMegabytes, long additionalRequiredBytes)
|
||||
{
|
||||
if (!settings.EnableStorageGuard)
|
||||
{
|
||||
return new StorageGuardResult(false, true, ResolveOutputRoot(settings.OutputRoot), long.MaxValue, 0, "Storage guard is disabled.");
|
||||
}
|
||||
|
||||
var checkedPath = ResolveOutputRoot(settings.OutputRoot);
|
||||
var thresholdBytes = Math.Max(0, freeSpaceThresholdMegabytes) * Megabyte;
|
||||
var requiredBytes = thresholdBytes + Math.Max(0, additionalRequiredBytes);
|
||||
|
||||
try
|
||||
{
|
||||
var drive = new DriveInfo(Path.GetPathRoot(checkedPath) ?? checkedPath);
|
||||
var availableBytes = drive.AvailableFreeSpace;
|
||||
var hasEnoughSpace = availableBytes >= requiredBytes;
|
||||
var message = hasEnoughSpace
|
||||
? $"Storage is available. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}"
|
||||
: $"Storage is below threshold. free={FormatBytes(availableBytes)}, required={FormatBytes(requiredBytes)}, path={checkedPath}";
|
||||
|
||||
return new StorageGuardResult(true, hasEnoughSpace, checkedPath, availableBytes, requiredBytes, message);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Storage guard failed to inspect output root {OutputRoot}", checkedPath);
|
||||
return new StorageGuardResult(true, false, checkedPath, 0, requiredBytes, $"Unable to inspect storage path {checkedPath}: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private static string ResolveOutputRoot(string outputRoot)
|
||||
{
|
||||
var root = string.IsNullOrWhiteSpace(outputRoot) ? "records" : outputRoot.Trim();
|
||||
return Path.IsPathRooted(root)
|
||||
? root
|
||||
: Path.GetFullPath(root, AppContext.BaseDirectory);
|
||||
}
|
||||
|
||||
private static string FormatBytes(long bytes)
|
||||
{
|
||||
if (bytes == long.MaxValue)
|
||||
{
|
||||
return "unlimited";
|
||||
}
|
||||
|
||||
string[] units = ["B", "KB", "MB", "GB", "TB"];
|
||||
var value = Math.Max(0, bytes);
|
||||
var unitIndex = 0;
|
||||
var display = (double)value;
|
||||
while (display >= 1024 && unitIndex < units.Length - 1)
|
||||
{
|
||||
display /= 1024;
|
||||
unitIndex++;
|
||||
}
|
||||
|
||||
return $"{display:0.##} {units[unitIndex]}";
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user