fix: resolve low-storage deadlock by always resuming MP4 finalization
Under the Red storage tier, MP4 finalization (TS->MP4 remux) was being skipped, so tasks never reached Completed and the segment_completed event script — which uploads the file and deletes the local source to free space — never ran. The disk could never recover, deadlocking all recording and transcoding. Two reversed checks caused this: (1) FfmpegService gated finalization on the legacy HasEnoughSpace MB threshold (effectively 4GB) instead of the tier system, and (2) the polling loop only resumed paused finalizations when NOT in the Red tier. Now finalization is gated solely on ShouldPauseActive (true Red only) and the polling loop always attempts to resume it every cycle, since finalization is the very mechanism that frees space. Once any segment finalizes, the upload+delete script runs and the disk recovers, letting the rest finish. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
90fd9f0ba8
commit
f9c7ec5d43
@ -1383,7 +1383,7 @@ public sealed partial class FfmpegService
|
||||
IsLowStoragePauseError(finalizationError) ? SystemLogLevel.Warning : SystemLogLevel.Warning,
|
||||
"FFmpeg",
|
||||
IsLowStoragePauseError(finalizationError)
|
||||
? "Segment MP4 finalization paused because storage is below threshold."
|
||||
? "Segment MP4 finalization paused because storage tier is Red."
|
||||
: "Segment MP4 finalization failed after rollover.",
|
||||
finalizationError,
|
||||
session.LiveRoomId,
|
||||
|
||||
@ -13,7 +13,7 @@ namespace LiveRecorder.Infrastructure.Services;
|
||||
|
||||
public sealed partial class FfmpegService
|
||||
{
|
||||
private const string LowStoragePauseErrorPrefix = "MP4 finalization paused because storage is below threshold.";
|
||||
private const string LowStoragePauseErrorPrefix = "MP4 finalization paused because storage tier is Red (critically low).";
|
||||
private static readonly TimeSpan Mp4FinalizeInactivityTimeout = TimeSpan.FromMinutes(10);
|
||||
private static readonly TimeSpan Mp4FinalizePollInterval = TimeSpan.FromSeconds(1);
|
||||
|
||||
@ -49,7 +49,7 @@ public sealed partial class FfmpegService
|
||||
var settings = await settingsService.GetAsync(cancellationToken);
|
||||
var sourceSizeBytes = new FileInfo(sourcePath).Length;
|
||||
var storageCheck = _storageGuardService.CheckCanStartOrResume(settings, sourceSizeBytes);
|
||||
return storageCheck.HasEnoughSpace
|
||||
return storageCheck.ShouldPauseActive
|
||||
? null
|
||||
: $"{LowStoragePauseErrorPrefix} {storageCheck.Message}";
|
||||
}
|
||||
|
||||
@ -542,12 +542,12 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
var settings = await settingsService.GetAsync(cancellationToken);
|
||||
|
||||
var storageCheck = _storageGuardService.CheckCanStartOrResume(settings);
|
||||
if (!storageCheck.HasEnoughSpace)
|
||||
if (storageCheck.ShouldPauseActive)
|
||||
{
|
||||
await logService.WriteAsync(
|
||||
SystemLogLevel.Warning,
|
||||
"Storage",
|
||||
"Paused MP4 finalization remains blocked because storage is below threshold.",
|
||||
"MP4 finalization remains paused because storage tier is Red.",
|
||||
storageCheck.Message,
|
||||
cancellationToken: cancellationToken);
|
||||
return 0;
|
||||
|
||||
@ -74,12 +74,13 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
|
||||
var settingsService = scope.ServiceProvider.GetRequiredService<ISystemSettingsService>();
|
||||
var settings = await settingsService.GetAsync(stoppingToken);
|
||||
var ffmpegService = scope.ServiceProvider.GetRequiredService<IFfmpegService>();
|
||||
var storageGuardService = scope.ServiceProvider.GetRequiredService<IStorageGuardService>();
|
||||
var storageCheck = storageGuardService.CheckCanStartOrResume(settings);
|
||||
if (storageCheck.Tier != StorageTier.Red)
|
||||
{
|
||||
await ffmpegService.ResumePausedFinalizationsAsync(stoppingToken);
|
||||
}
|
||||
// Always try to resume paused MP4 finalizations — even (especially) under the Red
|
||||
// tier. Finalization is what flips a task to Completed, which fires the
|
||||
// segment_completed script (upload + delete source) that frees disk space. Skipping
|
||||
// it while storage is low is exactly what caused the low-storage deadlock.
|
||||
// ResumePausedFinalizationsAsync internally no-ops only when space is truly
|
||||
// insufficient to remux, and it is retried every poll cycle.
|
||||
await ffmpegService.ResumePausedFinalizationsAsync(stoppingToken);
|
||||
|
||||
if (!settings.EnableBackgroundPolling)
|
||||
{
|
||||
|
||||
Loading…
Reference in New Issue
Block a user