nanxun/live_recorder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: retry segmented sessions and restore settings text

Browse Source
This commit is contained in:
nanxun
2026-05-08 23:27:01 +08:00
parent 93e39b2c1c
commit ea41d3e7b4
4 changed files with 339 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/LiveRecorder.Infrastructure/Services/FfmpegService.Runtime.cs
+223
View File
@@ -17,6 +17,7 @@ namespace LiveRecorder.Infrastructure.Services;
public sealed partial class FfmpegService
{
private const int MaxInSessionRetryAttempts = 3;
private static readonly TimeSpan RuntimeSourceFailureWindow = TimeSpan.FromSeconds(20);
private static readonly TimeSpan RuntimeOfflineVerificationStopTimeout = TimeSpan.FromSeconds(20);
private static readonly TimeSpan RuntimeOfflineVerificationKillTimeout = TimeSpan.FromSeconds(8);
@@ -472,6 +473,11 @@ public sealed partial class FfmpegService
return;
}
if (await TryRecoverUnexpectedExitAsync(runtime, activeDanmakuSummary))
{
return;
}
await FinalizeExitedSessionAsync(runtime, process, activeDanmakuSummary);
}
catch (Exception ex)
@@ -522,6 +528,11 @@ public sealed partial class FfmpegService
return false;
}
if (runtime.RetryAttemptCount >= MaxInSessionRetryAttempts)
{
return false;
}
var observedAt = DateTimeOffset.UtcNow;
if (runtime.StartupFailureKind == StartupFailureKind.InputOptionCompatibility)
{
@@ -689,6 +700,182 @@ public sealed partial class FfmpegService
}
}
private async Task<bool> TryRecoverUnexpectedExitAsync(
SessionProcessRuntime runtime,
SessionDanmakuXmlRecorder.DanmakuSegmentSummary? activeDanmakuSummary)
{
if (!runtime.HasOpenedFirstSegment ||
runtime.StopRequested ||
runtime.CompletionRequested ||
runtime.SaveMode != RecordSaveMode.Segmented ||
runtime.RetryAttemptCount >= MaxInSessionRetryAttempts)
{
return false;
}
try
{
using var scope = _serviceScopeFactory.CreateScope();
var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
var adapterFactory = scope.ServiceProvider.GetRequiredService<ILivePlatformAdapterFactory>();
var liveRoomStatusService = scope.ServiceProvider.GetRequiredService<LiveRoomStatusService>();
var eventScriptService = scope.ServiceProvider.GetRequiredService<IEventScriptService>();
var recordUploadService = scope.ServiceProvider.GetRequiredService<RecordUploadService>();
var session = await dbContext.RecordSessions
.Include(item => item.LiveRoom)
.Include(item => item.RecordTasks)
.ThenInclude(item => item.Result)
.FirstOrDefaultAsync(item => item.Id == runtime.RecordSessionId);
if (session?.LiveRoom is null || !IsActiveSessionStatus(session.Status))
{
return false;
}
var currentTask = session.RecordTasks
.OrderBy(item => item.SegmentIndex)
.ThenBy(item => item.CreatedAt)
.FirstOrDefault(item => item.Id == runtime.CurrentTaskId);
if (currentTask is null || !IsActiveTaskStatus(currentTask.Status))
{
return false;
}
var observedAt = DateTimeOffset.UtcNow;
var adapter = adapterFactory.GetByPlatform(session.LiveRoom.Platform);
var liveStatus = await adapter.GetLiveStatusAsync(session.LiveRoom.RoomId);
await liveRoomStatusService.ApplySnapshotAsync(session.LiveRoom, liveStatus, observedAt);
await dbContext.SaveChangesAsync();
if (!liveStatus.IsLive)
{
return false;
}
var retryAttempt = runtime.RetryAttemptCount + 1;
var refreshedStream = await adapter.GetStreamUrlAsync(session.LiveRoom.RoomId, session.PreferredQuality);
var retryStream = SelectRetryStreamForCurrentSession(refreshedStream, runtime);
var nextSegmentIndex = Math.Max(currentTask.SegmentIndex + 1, session.ActiveSegmentIndex + 1);
var outputPathPattern = session.OutputPathPattern ?? runtime.OutputPathPattern;
var retryOutputPath = NormalizeAbsolutePath(
ResolveSegmentOutputPath(outputPathPattern, session.SaveMode, nextSegmentIndex));
var retryTask = new RecordTask(
session.LiveRoomId,
session.Id,
nextSegmentIndex,
session.PreferredQuality,
session.OutputFormat,
observedAt);
retryTask.MarkStarting(retryStream.SelectedUrl, retryOutputPath, observedAt);
await dbContext.RecordTasks.AddAsync(retryTask);
await dbContext.SaveChangesAsync();
try
{
await StartInternalAsync(
session,
retryTask,
retryStream,
runtime.RecordingSettings,
runtime.InputOptionProfile,
hasRetriedWithCompatibilityProfile: false,
hasRetriedWithRefreshedStream: false,
retryAttempt);
}
catch
{
dbContext.RecordTasks.Remove(retryTask);
await dbContext.SaveChangesAsync();
throw;
}
var restartedAt = DateTimeOffset.UtcNow;
var previousTaskId = currentTask.Id;
var previousDurationSeconds = currentTask.StartedAt.HasValue
? Math.Max(0, (restartedAt - currentTask.StartedAt.Value).TotalSeconds)
: (double?)null;
var previousEffectiveOutputPath = currentTask.OutputFilePath ??
NormalizeAbsolutePath(ResolveSegmentOutputPath(outputPathPattern, session.SaveMode, currentTask.SegmentIndex));
var previousRecorderSegmentPaths = runtime.CaptureCurrentRecorderSegments();
if (session.OutputFormat == RecordOutputFormat.Mp4)
{
currentTask.MarkProcessing("Recorder exited unexpectedly. Segment finalization was queued before retry.", restartedAt);
SetPostProcessState(
session.Id,
previousTaskId,
"Queued",
0,
$"Waiting for {Path.GetFileName(GetRecorderOutputPath(previousEffectiveOutputPath, session.OutputFormat, session.SaveMode))} to finish writing");
}
else
{
currentTask.MarkCompleted(restartedAt, previousDurationSeconds);
}
currentTask.DetachProcess(restartedAt);
session.MarkRunning(restartedAt);
session.ActivateSegment(nextSegmentIndex, restartedAt);
retryTask.MarkRunning(restartedAt);
await dbContext.SaveChangesAsync();
if (session.OutputFormat == RecordOutputFormat.Mp4)
{
_ = FinalizeCompletedSegmentAsync(
session.Id,
previousTaskId,
previousDurationSeconds,
restartedAt,
activeDanmakuSummary,
previousRecorderSegmentPaths);
}
else
{
await UpsertRecordResultAsync(
currentTask,
dbContext,
previousEffectiveOutputPath,
CalculateFileSize(previousEffectiveOutputPath),
currentTask.DurationSeconds,
activeDanmakuSummary?.FilePath,
activeDanmakuSummary?.MessageCount ?? 0,
restartedAt);
await dbContext.SaveChangesAsync();
ClearPostProcessState(currentTask.Id);
var previousResult = await LoadRecordResultAsync(dbContext, currentTask.Id);
await eventScriptService.RunSegmentCompletedAsync(
session.LiveRoom,
session,
currentTask,
previousResult,
previousEffectiveOutputPath,
restartedAt);
await recordUploadService.TryAutoUploadTaskAsync(currentTask.Id, CancellationToken.None);
}
await logService.WriteAsync(
SystemLogLevel.Warning,
"FFmpeg",
$"ffmpeg exited unexpectedly. Retrying within the current session ({retryAttempt}/{MaxInSessionRetryAttempts}).",
runtime.GetRecentOutputSummary(),
session.LiveRoomId,
session.Id,
retryTask.Id);
return true;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "In-session ffmpeg retry failed for session {RecordSessionId}", runtime.RecordSessionId);
return false;
}
}
private async Task<bool> TryRetryWithAlternateProtocolAsync(
SessionProcessRuntime runtime,
RecordSession session,
@@ -790,6 +977,38 @@ public sealed partial class FfmpegService
}
}
private static StreamUrlResult SelectRetryStreamForCurrentSession(
StreamUrlResult refreshedStream,
SessionProcessRuntime runtime)
{
var matchedOption = refreshedStream.AvailableQualities
.Where(item => item.Protocol.Equals(runtime.SelectedProtocol, StringComparison.OrdinalIgnoreCase) &&
item.QualityKey.Equals(runtime.SelectedQuality, StringComparison.OrdinalIgnoreCase))
.OrderByDescending(item => item.Rank)
.FirstOrDefault();
if (matchedOption is null)
{
matchedOption = refreshedStream.AvailableQualities
.Where(item => item.Protocol.Equals(runtime.SelectedProtocol, StringComparison.OrdinalIgnoreCase))
.OrderByDescending(item => item.Rank)
.FirstOrDefault();
}
if (matchedOption is null)
{
return refreshedStream;
}
return new StreamUrlResult(
matchedOption.QualityKey,
matchedOption.Protocol,
matchedOption.Url,
refreshedStream.InputHeaders,
refreshedStream.AvailableQualities,
refreshedStream.SelectedVideoCodec);
}
private async Task FinalizeExitedSessionAsync(
SessionProcessRuntime runtime,
Process process,
@@ -1327,11 +1546,15 @@ public sealed partial class FfmpegService
private static bool IsRetryableStartupFailureLine(string line) =>
line.Contains("Error reading HTTP response", StringComparison.OrdinalIgnoreCase) ||
line.Contains("overlong headers", StringComparison.OrdinalIgnoreCase) ||
line.Contains("unexpected EOF", StringComparison.OrdinalIgnoreCase) ||
line.Contains("Connection reset", StringComparison.OrdinalIgnoreCase) ||
line.Contains("Connection refused", StringComparison.OrdinalIgnoreCase) ||
line.Contains("I/O error", StringComparison.OrdinalIgnoreCase) ||
line.Contains("Invalid data found when processing input", StringComparison.OrdinalIgnoreCase) ||
(line.Contains("Invalid argument", StringComparison.OrdinalIgnoreCase) &&
(line.Contains("http://", StringComparison.OrdinalIgnoreCase) ||
line.Contains("https://", StringComparison.OrdinalIgnoreCase))) ||
line.Contains("Server returned 4", StringComparison.OrdinalIgnoreCase) ||
line.Contains("Server returned 5", StringComparison.OrdinalIgnoreCase) ||
line.Contains("HTTP error 4", StringComparison.OrdinalIgnoreCase) ||