fix: resolve low-storage deadlock by always resuming MP4 finalization
Under the Red storage tier, MP4 finalization (TS->MP4 remux) was being skipped, so tasks never reached Completed and the segment_completed event script — which uploads the file and deletes the local source to free space — never ran. The disk could never recover, deadlocking all recording and transcoding. Two reversed checks caused this: (1) FfmpegService gated finalization on the legacy HasEnoughSpace MB threshold (effectively 4GB) instead of the tier system, and (2) the polling loop only resumed paused finalizations when NOT in the Red tier. Now finalization is gated solely on ShouldPauseActive (true Red only) and the polling loop always attempts to resume it every cycle, since finalization is the very mechanism that frees space. Once any segment finalizes, the upload+delete script runs and the disk recovers, letting the rest finish. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -1383,7 +1383,7 @@ public sealed partial class FfmpegService
|
|||||||
IsLowStoragePauseError(finalizationError) ? SystemLogLevel.Warning : SystemLogLevel.Warning,
|
IsLowStoragePauseError(finalizationError) ? SystemLogLevel.Warning : SystemLogLevel.Warning,
|
||||||
"FFmpeg",
|
"FFmpeg",
|
||||||
IsLowStoragePauseError(finalizationError)
|
IsLowStoragePauseError(finalizationError)
|
||||||
? "Segment MP4 finalization paused because storage is below threshold."
|
? "Segment MP4 finalization paused because storage tier is Red."
|
||||||
: "Segment MP4 finalization failed after rollover.",
|
: "Segment MP4 finalization failed after rollover.",
|
||||||
finalizationError,
|
finalizationError,
|
||||||
session.LiveRoomId,
|
session.LiveRoomId,
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ namespace LiveRecorder.Infrastructure.Services;
|
|||||||
|
|
||||||
public sealed partial class FfmpegService
|
public sealed partial class FfmpegService
|
||||||
{
|
{
|
||||||
private const string LowStoragePauseErrorPrefix = "MP4 finalization paused because storage is below threshold.";
|
private const string LowStoragePauseErrorPrefix = "MP4 finalization paused because storage tier is Red (critically low).";
|
||||||
private static readonly TimeSpan Mp4FinalizeInactivityTimeout = TimeSpan.FromMinutes(10);
|
private static readonly TimeSpan Mp4FinalizeInactivityTimeout = TimeSpan.FromMinutes(10);
|
||||||
private static readonly TimeSpan Mp4FinalizePollInterval = TimeSpan.FromSeconds(1);
|
private static readonly TimeSpan Mp4FinalizePollInterval = TimeSpan.FromSeconds(1);
|
||||||
|
|
||||||
@@ -49,7 +49,7 @@ public sealed partial class FfmpegService
|
|||||||
var settings = await settingsService.GetAsync(cancellationToken);
|
var settings = await settingsService.GetAsync(cancellationToken);
|
||||||
var sourceSizeBytes = new FileInfo(sourcePath).Length;
|
var sourceSizeBytes = new FileInfo(sourcePath).Length;
|
||||||
var storageCheck = _storageGuardService.CheckCanStartOrResume(settings, sourceSizeBytes);
|
var storageCheck = _storageGuardService.CheckCanStartOrResume(settings, sourceSizeBytes);
|
||||||
return storageCheck.HasEnoughSpace
|
return storageCheck.ShouldPauseActive
|
||||||
? null
|
? null
|
||||||
: $"{LowStoragePauseErrorPrefix} {storageCheck.Message}";
|
: $"{LowStoragePauseErrorPrefix} {storageCheck.Message}";
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -542,12 +542,12 @@ public sealed partial class FfmpegService : IFfmpegService
|
|||||||
var settings = await settingsService.GetAsync(cancellationToken);
|
var settings = await settingsService.GetAsync(cancellationToken);
|
||||||
|
|
||||||
var storageCheck = _storageGuardService.CheckCanStartOrResume(settings);
|
var storageCheck = _storageGuardService.CheckCanStartOrResume(settings);
|
||||||
if (!storageCheck.HasEnoughSpace)
|
if (storageCheck.ShouldPauseActive)
|
||||||
{
|
{
|
||||||
await logService.WriteAsync(
|
await logService.WriteAsync(
|
||||||
SystemLogLevel.Warning,
|
SystemLogLevel.Warning,
|
||||||
"Storage",
|
"Storage",
|
||||||
"Paused MP4 finalization remains blocked because storage is below threshold.",
|
"MP4 finalization remains paused because storage tier is Red.",
|
||||||
storageCheck.Message,
|
storageCheck.Message,
|
||||||
cancellationToken: cancellationToken);
|
cancellationToken: cancellationToken);
|
||||||
return 0;
|
return 0;
|
||||||
|
|||||||
@@ -74,12 +74,13 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
|
|||||||
var settingsService = scope.ServiceProvider.GetRequiredService<ISystemSettingsService>();
|
var settingsService = scope.ServiceProvider.GetRequiredService<ISystemSettingsService>();
|
||||||
var settings = await settingsService.GetAsync(stoppingToken);
|
var settings = await settingsService.GetAsync(stoppingToken);
|
||||||
var ffmpegService = scope.ServiceProvider.GetRequiredService<IFfmpegService>();
|
var ffmpegService = scope.ServiceProvider.GetRequiredService<IFfmpegService>();
|
||||||
var storageGuardService = scope.ServiceProvider.GetRequiredService<IStorageGuardService>();
|
// Always try to resume paused MP4 finalizations — even (especially) under the Red
|
||||||
var storageCheck = storageGuardService.CheckCanStartOrResume(settings);
|
// tier. Finalization is what flips a task to Completed, which fires the
|
||||||
if (storageCheck.Tier != StorageTier.Red)
|
// segment_completed script (upload + delete source) that frees disk space. Skipping
|
||||||
{
|
// it while storage is low is exactly what caused the low-storage deadlock.
|
||||||
await ffmpegService.ResumePausedFinalizationsAsync(stoppingToken);
|
// ResumePausedFinalizationsAsync internally no-ops only when space is truly
|
||||||
}
|
// insufficient to remux, and it is retried every poll cycle.
|
||||||
|
await ffmpegService.ResumePausedFinalizationsAsync(stoppingToken);
|
||||||
|
|
||||||
if (!settings.EnableBackgroundPolling)
|
if (!settings.EnableBackgroundPolling)
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user