fix: include ffmpeg exit diagnostics
This commit is contained in:
@@ -28,6 +28,7 @@ public sealed partial class FfmpegService
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
runtime.RememberOutputLine(line, isError);
|
||||||
_logger.LogDebug("ffmpeg[{SessionId}] {Line}", runtime.RecordSessionId, line);
|
_logger.LogDebug("ffmpeg[{SessionId}] {Line}", runtime.RecordSessionId, line);
|
||||||
if (TryParseSegmentOpenPath(line, out var openedPath))
|
if (TryParseSegmentOpenPath(line, out var openedPath))
|
||||||
{
|
{
|
||||||
@@ -921,9 +922,7 @@ public sealed partial class FfmpegService
|
|||||||
SystemLogLevel.Error,
|
SystemLogLevel.Error,
|
||||||
"FFmpeg",
|
"FFmpeg",
|
||||||
$"Recording session exited with status={session.Status}.",
|
$"Recording session exited with status={session.Status}.",
|
||||||
toleratedNonZeroExit
|
BuildExitLogDetail(runtime, process.ExitCode, effectiveOutputPath, toleratedNonZeroExit, finalizationError),
|
||||||
? $"exitCode={process.ExitCode}; output={effectiveOutputPath}; recorderOutput={runtime.RecorderOutputPath}; non-zero exit was tolerated because completion was requested and the output is usable"
|
|
||||||
: $"exitCode={process.ExitCode}; output={effectiveOutputPath}; recorderOutput={runtime.RecorderOutputPath}",
|
|
||||||
session.LiveRoomId,
|
session.LiveRoomId,
|
||||||
session.Id,
|
session.Id,
|
||||||
currentTask.Id);
|
currentTask.Id);
|
||||||
@@ -935,21 +934,60 @@ public sealed partial class FfmpegService
|
|||||||
|
|
||||||
if (session.Status == RecordSessionStatus.Failed && session.LiveRoom is not null)
|
if (session.Status == RecordSessionStatus.Failed && session.LiveRoom is not null)
|
||||||
{
|
{
|
||||||
|
var failureDetail = BuildExitLogDetail(runtime, process.ExitCode, effectiveOutputPath, toleratedNonZeroExit: false, finalizationError);
|
||||||
await emailNotificationService.SendExceptionAsync(
|
await emailNotificationService.SendExceptionAsync(
|
||||||
"FFmpeg",
|
"FFmpeg",
|
||||||
"Recording session exited abnormally.",
|
"Recording session exited abnormally.",
|
||||||
$"exitCode={process.ExitCode}; output={effectiveOutputPath}",
|
failureDetail,
|
||||||
session.LiveRoom,
|
session.LiveRoom,
|
||||||
currentTask);
|
currentTask);
|
||||||
await webhookNotificationService.SendExceptionAsync(
|
await webhookNotificationService.SendExceptionAsync(
|
||||||
"FFmpeg",
|
"FFmpeg",
|
||||||
"Recording session exited abnormally.",
|
"Recording session exited abnormally.",
|
||||||
$"exitCode={process.ExitCode}; output={effectiveOutputPath}",
|
failureDetail,
|
||||||
session.LiveRoom,
|
session.LiveRoom,
|
||||||
currentTask);
|
currentTask);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static string BuildExitLogDetail(
|
||||||
|
SessionProcessRuntime runtime,
|
||||||
|
int exitCode,
|
||||||
|
string? effectiveOutputPath,
|
||||||
|
bool toleratedNonZeroExit,
|
||||||
|
string? finalizationError)
|
||||||
|
{
|
||||||
|
var parts = new List<string>
|
||||||
|
{
|
||||||
|
$"exitCode={exitCode}",
|
||||||
|
$"output={effectiveOutputPath}",
|
||||||
|
$"recorderOutput={runtime.RecorderOutputPath}"
|
||||||
|
};
|
||||||
|
|
||||||
|
if (toleratedNonZeroExit)
|
||||||
|
{
|
||||||
|
parts.Add("non-zero exit was tolerated because completion was requested and the output is usable");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!string.IsNullOrWhiteSpace(finalizationError))
|
||||||
|
{
|
||||||
|
parts.Add($"finalizationError={finalizationError}");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!string.IsNullOrWhiteSpace(runtime.LastStartupFailureLine))
|
||||||
|
{
|
||||||
|
parts.Add($"startupFailure={runtime.LastStartupFailureLine}");
|
||||||
|
}
|
||||||
|
|
||||||
|
var recentOutput = runtime.GetRecentOutputSummary();
|
||||||
|
if (!string.IsNullOrWhiteSpace(recentOutput))
|
||||||
|
{
|
||||||
|
parts.Add($"recentOutput={recentOutput}");
|
||||||
|
}
|
||||||
|
|
||||||
|
return string.Join("; ", parts);
|
||||||
|
}
|
||||||
|
|
||||||
private async Task FinalizeCompletedSegmentAsync(
|
private async Task FinalizeCompletedSegmentAsync(
|
||||||
Guid recordSessionId,
|
Guid recordSessionId,
|
||||||
Guid recordTaskId,
|
Guid recordTaskId,
|
||||||
@@ -1566,6 +1604,8 @@ public sealed partial class FfmpegService
|
|||||||
public Task? DanmakuPumpTask { get; set; }
|
public Task? DanmakuPumpTask { get; set; }
|
||||||
private List<string> CurrentRecorderSegmentPaths { get; } = [];
|
private List<string> CurrentRecorderSegmentPaths { get; } = [];
|
||||||
private object RuntimeSourceFailureSync { get; } = new();
|
private object RuntimeSourceFailureSync { get; } = new();
|
||||||
|
private object RecentOutputSync { get; } = new();
|
||||||
|
private Queue<string> RecentOutputLines { get; } = new();
|
||||||
private DateTimeOffset RuntimeSourceFailureWindowStartedAt { get; set; }
|
private DateTimeOffset RuntimeSourceFailureWindowStartedAt { get; set; }
|
||||||
private int RuntimeSourceFailureCount { get; set; }
|
private int RuntimeSourceFailureCount { get; set; }
|
||||||
private bool RuntimeSourceFailureVerificationInProgress { get; set; }
|
private bool RuntimeSourceFailureVerificationInProgress { get; set; }
|
||||||
@@ -1600,6 +1640,28 @@ public sealed partial class FfmpegService
|
|||||||
|
|
||||||
public void MarkForceKilled() => WasForceKilled = true;
|
public void MarkForceKilled() => WasForceKilled = true;
|
||||||
|
|
||||||
|
public void RememberOutputLine(string line, bool isError)
|
||||||
|
{
|
||||||
|
lock (RecentOutputSync)
|
||||||
|
{
|
||||||
|
RecentOutputLines.Enqueue($"{(isError ? "stderr" : "stdout")}={line.Trim()}");
|
||||||
|
while (RecentOutputLines.Count > 8)
|
||||||
|
{
|
||||||
|
RecentOutputLines.Dequeue();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public string? GetRecentOutputSummary()
|
||||||
|
{
|
||||||
|
lock (RecentOutputSync)
|
||||||
|
{
|
||||||
|
return RecentOutputLines.Count == 0
|
||||||
|
? null
|
||||||
|
: string.Join(" | ", RecentOutputLines);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public void ResetCurrentRecorderSegmentPaths(string openedPath)
|
public void ResetCurrentRecorderSegmentPaths(string openedPath)
|
||||||
{
|
{
|
||||||
CurrentRecorderSegmentPaths.Clear();
|
CurrentRecorderSegmentPaths.Clear();
|
||||||
|
|||||||
Reference in New Issue
Block a user