fix: harden recording recovery and dashboard metrics

2026-08-14 19:20:24 +08:00
parent 506dca898e
commit 90a67bdf72
24 changed files with 607 additions and 46 deletions
@@ -272,7 +272,7 @@ public sealed class FfmpegFailureClassificationTests
    }

    [Fact]
-    public void RecoveryContext_LeavesCurlAfterExitWithoutResettingOneShotBudget()
+    public void RefreshedFlvRecovery_ReEnablesCurlAfterOverlongHeaders()
    {
        var curlContext = FfmpegService.InitialRecoveryContext with
        {
@@ -280,18 +280,65 @@ public sealed class FfmpegFailureClassificationTests
            HasTriedCurlFallback = true
        };

-        var nextContext = FfmpegService.AdvanceRecoveryContext(
-            curlContext,
-            curlContext.InputOptionProfile,
+        var nextContext = FfmpegService.PrepareRefreshedStreamRecoveryContext(
+            FfmpegService.AdvanceRecoveryContext(
+                curlContext,
+                curlContext.InputOptionProfile,
+                "flv",
+                "flv",
+                refreshedStream: true),
            "flv",
-            "flv",
-            refreshedStream: true);
+            hasOverlongHeadersFailure: true);

-        Assert.False(nextContext.UseCurlFallback);
+        Assert.True(nextContext.UseCurlFallback);
        Assert.True(nextContext.HasTriedCurlFallback);
        Assert.True(nextContext.HasRetriedWithRefreshedStream);
        Assert.False(FfmpegService.ShouldFallbackToCurl("flv", true, nextContext.HasTriedCurlFallback));
    }
+
+    [Fact]
+    public void RefreshedNonFlvRecovery_DisablesCurl()
+    {
+        var nextContext = FfmpegService.PrepareRefreshedStreamRecoveryContext(
+            FfmpegService.InitialRecoveryContext with
+            {
+                UseCurlFallback = true,
+                HasTriedCurlFallback = true
+            },
+            "hls",
+            hasOverlongHeadersFailure: true);
+
+        Assert.False(nextContext.UseCurlFallback);
+        Assert.True(nextContext.HasTriedCurlFallback);
+    }
+
+    [Fact]
+    public void RefreshedFlvRecovery_PreservesCurlRequirementAcrossLaterProcesses()
+    {
+        var priorContext = FfmpegService.InitialRecoveryContext with
+        {
+            UseCurlFallback = false,
+            HasTriedCurlFallback = true
+        };
+
+        var nextContext = FfmpegService.PrepareRefreshedStreamRecoveryContext(
+            priorContext,
+            "flv",
+            hasOverlongHeadersFailure: priorContext.HasTriedCurlFallback);
+
+        Assert.True(nextContext.UseCurlFallback);
+        Assert.True(nextContext.HasTriedCurlFallback);
+    }
+
+    [Theory]
+    [InlineData(false, true, false)]
+    [InlineData(false, false, true)]
+    [InlineData(true, true, true)]
+    public void FfmpegLinePersistence_SuppressesRecoverableStartupHandshakeNoise(
+        bool hasOpenedFirstSegment,
+        bool isStreamHandshakeFailure,
+        bool expected) =>
+        Assert.Equal(expected, FfmpegService.ShouldPersistFfmpegLine(hasOpenedFirstSegment, isStreamHandshakeFailure));
    [Fact]
    public void RecoveryContext_PreservesBudgetAcrossHlsFallbackAndReachesTimestampTranscode()
    {
@@ -615,6 +615,62 @@ public sealed class OpenListUploadTests
        Assert.Equal(1, await repository.CountFailedArtifactAsync());
    }

+    [Fact]
+    public async Task PendingUploadMetrics_IncludeValidVisibleArtifactWithActualFileSize()
+    {
+        await using var fixture = await QueueFixture.CreateAsync();
+        var result = await fixture.Context.RecordResults.AsNoTracking().SingleAsync();
+        var repository = new RecordResultRepository(fixture.Context);
+
+        var metrics = await repository.GetPendingUploadMetricsAsync();
+
+        Assert.Equal(1, metrics.Count);
+        Assert.Equal(new FileInfo(result.FilePath).Length, metrics.TotalBytes);
+    }
+
+    [Fact]
+    public async Task PendingUploadMetrics_ExcludeHiddenMergedSource()
+    {
+        await using var fixture = await QueueFixture.CreateAsync();
+        var task = await fixture.Context.RecordTasks.SingleAsync();
+        task.MarkMergedSource(Guid.NewGuid(), DateTimeOffset.UtcNow);
+        await fixture.Context.SaveChangesAsync();
+        var repository = new RecordResultRepository(fixture.Context);
+
+        var metrics = await repository.GetPendingUploadMetricsAsync();
+
+        Assert.Equal(0, metrics.Count);
+        Assert.Equal(0, metrics.TotalBytes);
+    }
+
+    [Fact]
+    public async Task PendingUploadMetrics_ExcludeMissingLocalFile()
+    {
+        await using var fixture = await QueueFixture.CreateAsync();
+        var result = await fixture.Context.RecordResults.AsNoTracking().SingleAsync();
+        File.Delete(result.FilePath);
+        var repository = new RecordResultRepository(fixture.Context);
+
+        var metrics = await repository.GetPendingUploadMetricsAsync();
+
+        Assert.Equal(0, metrics.Count);
+        Assert.Equal(0, metrics.TotalBytes);
+    }
+
+    [Fact]
+    public async Task PendingUploadMetrics_ExcludeEmptyLocalFile()
+    {
+        await using var fixture = await QueueFixture.CreateAsync();
+        var result = await fixture.Context.RecordResults.AsNoTracking().SingleAsync();
+        File.WriteAllBytes(result.FilePath, []);
+        var repository = new RecordResultRepository(fixture.Context);
+
+        var metrics = await repository.GetPendingUploadMetricsAsync();
+
+        Assert.Equal(0, metrics.Count);
+        Assert.Equal(0, metrics.TotalBytes);
+    }
+
    private static OpenListClient CreateClient(HttpMessageHandler handler) =>
        new(new StubHttpClientFactory(handler));

@@ -20,8 +20,11 @@ public sealed class RecordingContinuityTests
    [InlineData(3, 30)]
    [InlineData(4, 60)]
    [InlineData(5, 120)]
-    [InlineData(50, 120)]
-    public void RuntimeRecoveryBackoff_IsBoundedAtTwoMinutes(int attempt, int seconds) =>
+    [InlineData(6, 300)]
+    [InlineData(7, 600)]
+    [InlineData(8, 900)]
+    [InlineData(50, 900)]
+    public void RuntimeRecoveryBackoff_IsBoundedAtFifteenMinutes(int attempt, int seconds) =>
        Assert.Equal(TimeSpan.FromSeconds(seconds), FfmpegService.GetRuntimeRecoveryDelay(attempt));

    [Theory]
@@ -0,0 +1,120 @@
+using LiveRecorder.Application.Abstractions.Logging;
+using LiveRecorder.Application.Models.Logs;
+using LiveRecorder.Domain.Entities;
+using LiveRecorder.Domain.Enums;
+using LiveRecorder.Infrastructure.Persistence;
+using LiveRecorder.Infrastructure.Services;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Storage;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace LiveRecorder.Tests;
+
+public sealed class StalePendingRecoveryTests
+{
+    [Fact]
+    public async Task StalePendingSessionWithoutMedia_IsFailedAndRetainedForAudit()
+    {
+        var services = new ServiceCollection();
+        var databaseRoot = new InMemoryDatabaseRoot();
+        var databaseName = $"stale-pending-{Guid.NewGuid():N}";
+        services.AddDbContext<LiveRecorderDbContext>(options =>
+            options.UseInMemoryDatabase(databaseName, databaseRoot));
+        var logService = new CapturingSystemLogService();
+        services.AddSingleton<ISystemLogService>(logService);
+        await using var provider = services.BuildServiceProvider();
+
+        var createdAt = DateTimeOffset.UtcNow.AddMinutes(-11);
+        Guid sessionId;
+        Guid taskId;
+        await using (var scope = provider.CreateAsyncScope())
+        {
+            var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
+            var liveRoom = new LiveRoom(
+                LivePlatformType.Douyin,
+                "https://live.example/room",
+                "room-1",
+                "https://live.example/room",
+                createdAt);
+            var session = new RecordSession(
+                liveRoom.Id,
+                "origin",
+                RecordOutputFormat.Mp4,
+                RecordSaveMode.Segmented,
+                createdAt);
+            var task = new RecordTask(
+                liveRoom.Id,
+                session.Id,
+                1,
+                "origin",
+                RecordOutputFormat.Mp4,
+                createdAt);
+            sessionId = session.Id;
+            taskId = task.Id;
+            dbContext.AddRange(liveRoom, session, task);
+            await dbContext.SaveChangesAsync();
+        }
+
+        var service = new FfmpegService(
+            provider.GetRequiredService<IServiceScopeFactory>(),
+            null!,
+            null!,
+            NullLogger<FfmpegService>.Instance);
+
+        await using (var preconditionScope = provider.CreateAsyncScope())
+        {
+            var preconditionContext = preconditionScope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
+            var persisted = await preconditionContext.RecordSessions.SingleAsync();
+            Assert.Equal(RecordSessionStatus.Pending, persisted.Status);
+            Assert.True(
+                persisted.UpdatedAt <= DateTimeOffset.UtcNow.AddMinutes(-10),
+                $"updatedAt={persisted.UpdatedAt:O}");
+        }
+
+        var recovered = await service.RecoverStalePendingSessionsAsync();
+
+        Assert.Equal(1, recovered);
+        await using var verificationScope = provider.CreateAsyncScope();
+        var verificationContext = verificationScope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
+        var persistedSession = await verificationContext.RecordSessions.SingleAsync(item => item.Id == sessionId);
+        var persistedTask = await verificationContext.RecordTasks
+            .Include(item => item.Result)
+            .SingleAsync(item => item.Id == taskId);
+        Assert.Equal(RecordSessionStatus.Failed, persistedSession.Status);
+        Assert.Equal(RecordTaskStatus.Failed, persistedTask.Status);
+        Assert.Contains("interrupted before stream initialization", persistedSession.ErrorMessage);
+        Assert.Null(persistedTask.Result);
+        Assert.Single(logService.Entries);
+        Assert.Equal(SystemLogLevel.Warning, logService.Entries[0].Level);
+    }
+
+    private sealed class CapturingSystemLogService : ISystemLogService
+    {
+        public List<(SystemLogLevel Level, string Message)> Entries { get; } = [];
+
+        public Task WriteAsync(
+            SystemLogLevel level,
+            string category,
+            string message,
+            string? detail = null,
+            Guid? liveRoomId = null,
+            Guid? recordSessionId = null,
+            Guid? recordTaskId = null,
+            CancellationToken cancellationToken = default)
+        {
+            Entries.Add((level, message));
+            return Task.CompletedTask;
+        }
+
+        public Task<IReadOnlyList<SystemLogDto>> ListAsync(
+            Guid? liveRoomId = null,
+            Guid? recordSessionId = null,
+            Guid? recordTaskId = null,
+            SystemLogLevel? level = null,
+            string? content = null,
+            int take = 200,
+            CancellationToken cancellationToken = default) =>
+            Task.FromResult<IReadOnlyList<SystemLogDto>>([]);
+    }
+}
@@ -7,7 +7,26 @@ namespace LiveRecorder.Tests;

 public sealed class StorageGuardServiceTests
 {
-    private readonly StorageGuardService _service = new(NullLogger<StorageGuardService>.Instance);
+    private readonly StorageGuardService _service = new(
+        NullLogger<StorageGuardService>.Instance,
+        new DriveInfoStorageCapacityProvider());
+
+    [Fact]
+    public void CapacityProvider_SelectsLongestContainingMountRoot()
+    {
+        var selected = DriveInfoStorageCapacityProvider.SelectBestVolume(
+            "/vol00/live-recorder/Record",
+            [
+                new StorageVolumeCapacity("/", 100, 10),
+                new StorageVolumeCapacity("/vol00", 200, 20),
+                new StorageVolumeCapacity("/vol00/live-recorder", 300, 30),
+                new StorageVolumeCapacity("/vol0", 400, 40)
+            ]);
+
+        Assert.NotNull(selected);
+        Assert.Equal("/vol00/live-recorder", selected.VolumeRoot);
+        Assert.Equal(300, selected.TotalBytes);
+    }

    [Fact]
    public void DisabledProtection_StillReportsActualCapacityWithoutBlockingRecording()