fix: include ffmpeg exit diagnostics

This commit is contained in:
2026-05-06 19:40:59 +08:00
parent 8626af6d80
commit 8f5e63cffd
@@ -28,6 +28,7 @@ public sealed partial class FfmpegService
return; return;
} }
runtime.RememberOutputLine(line, isError);
_logger.LogDebug("ffmpeg[{SessionId}] {Line}", runtime.RecordSessionId, line); _logger.LogDebug("ffmpeg[{SessionId}] {Line}", runtime.RecordSessionId, line);
if (TryParseSegmentOpenPath(line, out var openedPath)) if (TryParseSegmentOpenPath(line, out var openedPath))
{ {
@@ -921,9 +922,7 @@ public sealed partial class FfmpegService
SystemLogLevel.Error, SystemLogLevel.Error,
"FFmpeg", "FFmpeg",
$"Recording session exited with status={session.Status}.", $"Recording session exited with status={session.Status}.",
toleratedNonZeroExit BuildExitLogDetail(runtime, process.ExitCode, effectiveOutputPath, toleratedNonZeroExit, finalizationError),
? $"exitCode={process.ExitCode}; output={effectiveOutputPath}; recorderOutput={runtime.RecorderOutputPath}; non-zero exit was tolerated because completion was requested and the output is usable"
: $"exitCode={process.ExitCode}; output={effectiveOutputPath}; recorderOutput={runtime.RecorderOutputPath}",
session.LiveRoomId, session.LiveRoomId,
session.Id, session.Id,
currentTask.Id); currentTask.Id);
@@ -935,21 +934,60 @@ public sealed partial class FfmpegService
if (session.Status == RecordSessionStatus.Failed && session.LiveRoom is not null) if (session.Status == RecordSessionStatus.Failed && session.LiveRoom is not null)
{ {
var failureDetail = BuildExitLogDetail(runtime, process.ExitCode, effectiveOutputPath, toleratedNonZeroExit: false, finalizationError);
await emailNotificationService.SendExceptionAsync( await emailNotificationService.SendExceptionAsync(
"FFmpeg", "FFmpeg",
"Recording session exited abnormally.", "Recording session exited abnormally.",
$"exitCode={process.ExitCode}; output={effectiveOutputPath}", failureDetail,
session.LiveRoom, session.LiveRoom,
currentTask); currentTask);
await webhookNotificationService.SendExceptionAsync( await webhookNotificationService.SendExceptionAsync(
"FFmpeg", "FFmpeg",
"Recording session exited abnormally.", "Recording session exited abnormally.",
$"exitCode={process.ExitCode}; output={effectiveOutputPath}", failureDetail,
session.LiveRoom, session.LiveRoom,
currentTask); currentTask);
} }
} }
private static string BuildExitLogDetail(
SessionProcessRuntime runtime,
int exitCode,
string? effectiveOutputPath,
bool toleratedNonZeroExit,
string? finalizationError)
{
var parts = new List<string>
{
$"exitCode={exitCode}",
$"output={effectiveOutputPath}",
$"recorderOutput={runtime.RecorderOutputPath}"
};
if (toleratedNonZeroExit)
{
parts.Add("non-zero exit was tolerated because completion was requested and the output is usable");
}
if (!string.IsNullOrWhiteSpace(finalizationError))
{
parts.Add($"finalizationError={finalizationError}");
}
if (!string.IsNullOrWhiteSpace(runtime.LastStartupFailureLine))
{
parts.Add($"startupFailure={runtime.LastStartupFailureLine}");
}
var recentOutput = runtime.GetRecentOutputSummary();
if (!string.IsNullOrWhiteSpace(recentOutput))
{
parts.Add($"recentOutput={recentOutput}");
}
return string.Join("; ", parts);
}
private async Task FinalizeCompletedSegmentAsync( private async Task FinalizeCompletedSegmentAsync(
Guid recordSessionId, Guid recordSessionId,
Guid recordTaskId, Guid recordTaskId,
@@ -1566,6 +1604,8 @@ public sealed partial class FfmpegService
public Task? DanmakuPumpTask { get; set; } public Task? DanmakuPumpTask { get; set; }
private List<string> CurrentRecorderSegmentPaths { get; } = []; private List<string> CurrentRecorderSegmentPaths { get; } = [];
private object RuntimeSourceFailureSync { get; } = new(); private object RuntimeSourceFailureSync { get; } = new();
private object RecentOutputSync { get; } = new();
private Queue<string> RecentOutputLines { get; } = new();
private DateTimeOffset RuntimeSourceFailureWindowStartedAt { get; set; } private DateTimeOffset RuntimeSourceFailureWindowStartedAt { get; set; }
private int RuntimeSourceFailureCount { get; set; } private int RuntimeSourceFailureCount { get; set; }
private bool RuntimeSourceFailureVerificationInProgress { get; set; } private bool RuntimeSourceFailureVerificationInProgress { get; set; }
@@ -1600,6 +1640,28 @@ public sealed partial class FfmpegService
public void MarkForceKilled() => WasForceKilled = true; public void MarkForceKilled() => WasForceKilled = true;
public void RememberOutputLine(string line, bool isError)
{
lock (RecentOutputSync)
{
RecentOutputLines.Enqueue($"{(isError ? "stderr" : "stdout")}={line.Trim()}");
while (RecentOutputLines.Count > 8)
{
RecentOutputLines.Dequeue();
}
}
}
public string? GetRecentOutputSummary()
{
lock (RecentOutputSync)
{
return RecentOutputLines.Count == 0
? null
: string.Join(" | ", RecentOutputLines);
}
}
public void ResetCurrentRecorderSegmentPaths(string openedPath) public void ResetCurrentRecorderSegmentPaths(string openedPath)
{ {
CurrentRecorderSegmentPaths.Clear(); CurrentRecorderSegmentPaths.Clear();