feat: improve recording recovery and upload workflow

2026-08-10 11:14:34 +08:00
parent e0d3969e46
commit 19d358b312
58 changed files with 5979 additions and 586 deletions
@@ -103,6 +103,16 @@ public sealed class FfmpegFailureClassificationTests
            FfmpegService.IsMeaningfulUnexpectedExitArtifact(durationSeconds, fileSizeBytes));
    }

+    [Theory]
+    [InlineData(59, false)]
+    [InlineData(60, true)]
+    [InlineData(3600, true)]
+    public void StableRuntime_ResetsInSessionRecoveryBudget(int seconds, bool expected)
+    {
+        var now = DateTimeOffset.Parse("2026-08-04T20:00:00+08:00");
+        Assert.Equal(expected, FfmpegService.CanResetInSessionRetryBudget(now.AddSeconds(-seconds), now));
+    }
+
    [Theory]
    [InlineData("https://example.test/live.flv", "flv", true)]
    [InlineData("https://example.test/live.m3u8", "hls", false)]
@@ -299,6 +299,105 @@ public sealed class OpenListUploadTests
        Assert.Equal(now.AddSeconds(3), job.VerificationStartedAt);
    }

+    [Fact]
+    public void UploadJob_ImmediateRetryResetsBudgetButPreservesExternalTask()
+    {
+        var now = DateTimeOffset.Parse("2026-08-04T10:00:00+08:00");
+        var job = new RecordUploadJob(
+            Guid.NewGuid(),
+            "https://openlist.example.com",
+            "/source/video.mp4",
+            "/archive/video.mp4",
+            100,
+            null,
+            null,
+            null,
+            false,
+            now);
+        job.BeginAttempt(now.AddSeconds(1));
+        job.TrackExternalTask("copy-1", "copy", 30, now.AddSeconds(2));
+        job.ScheduleRetry("temporary", now.AddMinutes(5), now.AddSeconds(3), clearExternalTask: false);
+
+        job.RequestImmediateRetry(now.AddSeconds(4));
+
+        Assert.Equal(RecordArtifactUploadStatus.Queued, job.Status);
+        Assert.Equal(0, job.AttemptCount);
+        Assert.Null(job.NextAttemptAt);
+        Assert.Equal("copy-1", job.ExternalTaskId);
+        Assert.Equal(30, job.ProgressPercent);
+    }
+
+    [Fact]
+    public async Task LoginRateLimit_OpensProviderCircuit()
+    {
+        var state = new OpenListUploadHealthState();
+        var loginCount = 0;
+        var handler = new StubHttpMessageHandler(_ =>
+        {
+            loginCount++;
+            return Task.FromResult(new HttpResponseMessage(HttpStatusCode.TooManyRequests)
+            {
+                Content = new StringContent(
+                    "{\"code\":429,\"message\":\"错误账号密码尝试过多\",\"data\":null}",
+                    Encoding.UTF8,
+                    "application/json")
+            });
+        });
+        var client = new OpenListClient(new StubHttpClientFactory(handler), state);
+
+        var error = await Assert.ThrowsAsync<OpenListApiException>(() =>
+            client.EnsureAuthenticatedAsync(Connection()));
+
+        Assert.True(error.IsRateLimited);
+        Assert.Equal(1, loginCount);
+        var snapshot = state.GetSnapshot();
+        Assert.Equal(OpenListQueueHealthStatus.RateLimited, snapshot.Status);
+        Assert.True(snapshot.RetryAt > DateTimeOffset.UtcNow);
+        Assert.False(state.CanProcess(DateTimeOffset.UtcNow));
+    }
+
+    [Fact]
+    public async Task NonJsonRateLimit_StillOpensProviderCircuit()
+    {
+        var state = new OpenListUploadHealthState();
+        var handler = new StubHttpMessageHandler(_ => Task.FromResult(new HttpResponseMessage(HttpStatusCode.TooManyRequests)
+        {
+            Content = new StringContent("rate limited by reverse proxy", Encoding.UTF8, "text/plain")
+        }));
+        var client = new OpenListClient(new StubHttpClientFactory(handler), state);
+
+        var error = await Assert.ThrowsAsync<OpenListApiException>(() =>
+            client.EnsureAuthenticatedAsync(Connection()));
+
+        Assert.True(error.IsRateLimited);
+        Assert.Equal(OpenListQueueHealthStatus.RateLimited, state.GetSnapshot().Status);
+    }
+
+    [Fact]
+    public void UploadJob_ProviderPauseRollsBackOnlyTheCurrentAttempt()
+    {
+        var now = DateTimeOffset.Parse("2026-08-04T10:00:00+08:00");
+        var job = new RecordUploadJob(
+            Guid.NewGuid(),
+            "https://openlist.example.com",
+            "/source/video.mp4",
+            "/archive/video.mp4",
+            100,
+            null,
+            null,
+            null,
+            false,
+            now);
+        job.BeginAttempt(now.AddSeconds(1));
+
+        job.SuspendForProviderFailure("rate limited", now.AddSeconds(2), rollbackAttempt: true);
+
+        Assert.Equal(RecordArtifactUploadStatus.WaitingRetry, job.Status);
+        Assert.Equal(0, job.AttemptCount);
+        Assert.Null(job.NextAttemptAt);
+        Assert.Equal("rate limited", job.ErrorMessage);
+    }
+
    [Fact]
    public async Task Queue_MapsOutputRelativePathAndTreatsMatchingTargetAsIdempotentSuccess()
    {
@@ -648,7 +747,13 @@ public sealed class OpenListUploadTests
        }

        private OpenListUploadQueueService CreateQueue(LiveRecorderDbContext context) =>
-            new(context, _settingsService, OpenList, new NullSystemLogService(), _videoMetadataService);
+            new(
+                context,
+                _settingsService,
+                OpenList,
+                new OpenListUploadHealthState(),
+                new NullSystemLogService(),
+                _videoMetadataService);
    }

    private sealed class FixedVideoMetadataService : IVideoMetadataService
@@ -723,6 +828,11 @@ public sealed class OpenListUploadTests

        public OpenListCopyResult NextCopyResult { get; set; } = new([]);

+        public Task EnsureAuthenticatedAsync(
+            OpenListConnectionRequest connection,
+            bool forceRefresh = false,
+            CancellationToken cancellationToken = default) => Task.CompletedTask;
+
        public Task<OpenListConnectionTestDto> TestConnectionAsync(
            OpenListConnectionRequest connection,
            CancellationToken cancellationToken = default) => throw new NotSupportedException();
@@ -141,6 +141,7 @@ public sealed class RecordSessionRepositoryTests
        Assert.Equal(30, firstPage.TotalCount);
        Assert.Equal(10, firstPage.Items.Count);
        Assert.Equal(oldActive.Id, firstPage.Items[0].Id);
+        Assert.Equal("分页主播", firstPage.Items[0].LiveRoom?.AnchorName);
        Assert.Equal(29, completedPage.TotalCount);
        Assert.DoesNotContain(completedPage.Items, item => item.Id == oldActive.Id);
        Assert.Single(searchPage.Items);
@@ -0,0 +1,110 @@
+using LiveRecorder.Domain.Entities;
+using LiveRecorder.Domain.Enums;
+using LiveRecorder.Infrastructure.Services;
+using System.Xml.Linq;
+
+namespace LiveRecorder.Tests;
+
+public sealed class RecordingContinuityTests
+{
+    [Theory]
+    [InlineData(true, false, true)]
+    [InlineData(false, true, true)]
+    [InlineData(false, false, false)]
+    public void SessionTransition_IsTreatedAsActive(bool process, bool transition, bool expected) =>
+        Assert.Equal(expected, FfmpegService.IsSessionRuntimeActive(process, transition));
+
+    [Theory]
+    [InlineData(1, 5)]
+    [InlineData(2, 15)]
+    [InlineData(3, 30)]
+    [InlineData(4, 60)]
+    [InlineData(5, 120)]
+    [InlineData(50, 120)]
+    public void RuntimeRecoveryBackoff_IsBoundedAtTwoMinutes(int attempt, int seconds) =>
+        Assert.Equal(TimeSpan.FromSeconds(seconds), FfmpegService.GetRuntimeRecoveryDelay(attempt));
+
+    [Theory]
+    [InlineData(null, false, false)]
+    [InlineData(0.0, false, false)]
+    [InlineData(0.1, true, false)]
+    [InlineData(4.99, true, false)]
+    [InlineData(5.0, true, true)]
+    public void MediaDuration_SeparatesReadableFromStandalone(double? duration, bool readable, bool standalone)
+    {
+        Assert.Equal(readable, FfmpegService.IsReadableMediaDuration(duration));
+        Assert.Equal(standalone, FfmpegService.IsStandaloneMediaDuration(duration));
+    }
+
+    [Fact]
+    public void OfflineSample_RequiresConfirmation_ButLiveDoesNot()
+    {
+        Assert.True(LiveRoomPollingBackgroundService.RequiresOfflineConfirmation(false));
+        Assert.False(LiveRoomPollingBackgroundService.RequiresOfflineConfirmation(true));
+    }
+
+    [Theory]
+    [InlineData(false, RecordTaskStatus.Starting, true)]
+    [InlineData(false, RecordTaskStatus.Running, true)]
+    [InlineData(true, RecordTaskStatus.Running, false)]
+    [InlineData(false, RecordTaskStatus.Completed, false)]
+    public void ArtifactlessRecovery_ReusesOnlyActiveTask(bool hasMedia, RecordTaskStatus status, bool expected) =>
+        Assert.Equal(expected, FfmpegService.ShouldReuseRecoveryTask(hasMedia, status));
+
+    [Fact]
+    public void SessionRecoveryMarker_KeepsSessionActive()
+    {
+        var session = new RecordSession(Guid.NewGuid(), "origin", RecordOutputFormat.Mp4, RecordSaveMode.Segmented, DateTimeOffset.UtcNow);
+        session.MarkRecovering("[runtime-recovery] attempt=4; curl EOF", DateTimeOffset.UtcNow);
+
+        Assert.Equal(RecordSessionStatus.Starting, session.Status);
+        Assert.StartsWith("[runtime-recovery]", session.ErrorMessage);
+        Assert.Null(session.EndedAt);
+    }
+
+    [Fact]
+    public void MergedSource_PointsAtTargetAndBecomesHidden()
+    {
+        var source = new RecordTask(Guid.NewGuid(), Guid.NewGuid(), 2, "origin", RecordOutputFormat.Mp4, DateTimeOffset.UtcNow);
+        var targetId = Guid.NewGuid();
+        source.MarkMergedSource(targetId, DateTimeOffset.UtcNow);
+
+        Assert.True(source.IsHiddenArtifactSource);
+        Assert.Equal(targetId, source.MergedIntoRecordTaskId);
+    }
+
+    [Fact]
+    public void MergeWindow_FirstShortFragment_MergesForward()
+    {
+        var window = ShortFragmentConsolidationService.ResolveMergeWindow([1.2, 60], 0);
+        Assert.Equal((0, 1), window);
+    }
+
+    [Fact]
+    public void MergeWindow_TrailingShortFragment_MergesBackward()
+    {
+        var window = ShortFragmentConsolidationService.ResolveMergeWindow([60, 1.2], 0);
+        Assert.Equal((0, 1), window);
+    }
+
+    [Fact]
+    public void MergeWindow_ConsecutiveShortFragments_AreAccumulated()
+    {
+        var window = ShortFragmentConsolidationService.ResolveMergeWindow([1.2, 2.4, 60], 1);
+        Assert.Equal((0, 2), window);
+    }
+
+    [Fact]
+    public void DanmakuMerge_ShiftsRelativeOffset_ButPreservesAbsoluteTimestamp()
+    {
+        var chat = XElement.Parse("<d p=\"1.5,1,25,16777215,1720000000000,0,1,0\">hi</d>");
+        var shiftedChat = ShortFragmentConsolidationService.AdjustDanmakuElementOffsets(chat, 4.25);
+        Assert.StartsWith("5.8,", shiftedChat.Attribute("p")!.Value);
+        Assert.Contains("1720000000000", shiftedChat.Attribute("p")!.Value);
+
+        var eventElement = XElement.Parse("<event ts=\"1720000000123\" offset=\"2.0\" />");
+        var shiftedEvent = ShortFragmentConsolidationService.AdjustDanmakuElementOffsets(eventElement, 4.25);
+        Assert.Equal("6.2", shiftedEvent.Attribute("offset")!.Value);
+        Assert.Equal("1720000000123", shiftedEvent.Attribute("ts")!.Value);
+    }
+}
@@ -0,0 +1,170 @@
+using LiveRecorder.Application.Abstractions.Logging;
+using LiveRecorder.Application.Abstractions.Recording;
+using LiveRecorder.Application.Models.Logs;
+using LiveRecorder.Domain.Entities;
+using LiveRecorder.Domain.Enums;
+using LiveRecorder.Infrastructure.Persistence;
+using LiveRecorder.Infrastructure.Services;
+using Microsoft.EntityFrameworkCore;
+
+namespace LiveRecorder.Tests;
+
+public sealed class RecoveryServiceTests : IDisposable
+{
+    private readonly string _temporaryDirectory = Path.Combine(
+        Path.GetTempPath(),
+        $"liverecorder-artifact-recovery-{Guid.NewGuid():N}");
+
+    [Fact]
+    public async Task AcceptShortArtifact_RequiresConfirmationThenLeavesItForManualUpload()
+    {
+        Directory.CreateDirectory(_temporaryDirectory);
+        var mediaPath = Path.Combine(_temporaryDirectory, "short-fragment.mp4");
+        await File.WriteAllBytesAsync(mediaPath, [1, 2, 3, 4]);
+        await using var context = CreateContext();
+        var taskId = await SeedFailedTaskAsync(context, mediaPath, durationSeconds: 3);
+        var service = CreateService(context, new VideoMetadata(3, 1920, 1080, "h264", "aac", 30, 2_000_000));
+
+        var rejected = await service.AcceptRecordingArtifactAsync(taskId, confirmShortArtifact: false);
+
+        Assert.Equal(0, rejected.SuccessCount);
+        Assert.Contains("需要确认短分片", rejected.Messages.Single());
+        Assert.Equal(RecordTaskStatus.Failed, (await context.RecordTasks.FindAsync(taskId))!.Status);
+
+        var accepted = await service.AcceptRecordingArtifactAsync(taskId, confirmShortArtifact: true);
+
+        Assert.Equal(1, accepted.SuccessCount);
+        var task = await context.RecordTasks.Include(item => item.Result).SingleAsync(item => item.Id == taskId);
+        Assert.Equal(RecordTaskStatus.Completed, task.Status);
+        Assert.Equal(RecordArtifactUploadStatus.NotUploaded, task.Result!.UploadStatus);
+        Assert.Null(task.Result.LastUploadProvider);
+        Assert.Null(task.Result.LastUploadedAt);
+        Assert.Null(task.Result.UploadErrorMessage);
+        Assert.True(File.Exists(mediaPath));
+    }
+
+    [Fact]
+    public async Task ListFailures_IdentifiesMissingMediaAsUnrecoverable()
+    {
+        Directory.CreateDirectory(_temporaryDirectory);
+        var missingPath = Path.Combine(_temporaryDirectory, "missing.mp4");
+        await using var context = CreateContext();
+        await SeedFailedTaskAsync(context, missingPath, durationSeconds: 120);
+        var service = CreateService(context, metadata: null);
+
+        var response = await service.ListRecordingFailuresAsync("MissingMedia", 0, 20);
+
+        var item = Assert.Single(response.Items);
+        Assert.Equal("MissingMedia", item.FailureKind);
+        Assert.False(item.FileExists);
+        Assert.False(item.CanAccept);
+        Assert.False(item.CanRepair);
+    }
+
+    private static LiveRecorderDbContext CreateContext()
+    {
+        var options = new DbContextOptionsBuilder<LiveRecorderDbContext>()
+            .UseInMemoryDatabase($"artifact-recovery-{Guid.NewGuid():N}")
+            .Options;
+        return new LiveRecorderDbContext(options);
+    }
+
+    private static async Task<Guid> SeedFailedTaskAsync(
+        LiveRecorderDbContext context,
+        string mediaPath,
+        double durationSeconds)
+    {
+        var now = DateTimeOffset.UtcNow;
+        var room = new LiveRoom(
+            LivePlatformType.Douyin,
+            "https://live.example/recovery",
+            "recovery-room",
+            "https://live.example/recovery",
+            now.AddHours(-1));
+        room.UpdateMetadata("恢复测试直播间", "恢复主播", null, null, null, now.AddHours(-1));
+        var session = new RecordSession(
+            room.Id,
+            "origin",
+            RecordOutputFormat.Mp4,
+            RecordSaveMode.Segmented,
+            now.AddMinutes(-10));
+        session.MarkFailed("ffmpeg exited unexpectedly", now);
+        var task = new RecordTask(
+            room.Id,
+            session.Id,
+            1,
+            "origin",
+            RecordOutputFormat.Mp4,
+            now.AddMinutes(-10));
+        task.MarkStarting("https://stream.example/recovery", mediaPath, now.AddMinutes(-10));
+        task.MarkFailed("ffmpeg exited unexpectedly", now, durationSeconds);
+        var result = new RecordResult(
+            task.Id,
+            mediaPath,
+            File.Exists(mediaPath) ? new FileInfo(mediaPath).Length : null,
+            durationSeconds,
+            null,
+            0,
+            RecordTaskStatus.Failed,
+            task.ErrorMessage,
+            now);
+        result.MarkUploadFailed("openlist", "previous upload error", now);
+
+        context.AddRange(room, session, task, result);
+        await context.SaveChangesAsync();
+        return task.Id;
+    }
+
+    private static RecoveryService CreateService(LiveRecorderDbContext context, VideoMetadata? metadata) =>
+        new(
+            context,
+            systemSettingsService: null!,
+            storageGuardService: null!,
+            ffmpegService: null!,
+            recordService: null!,
+            videoMetadataService: new FixedVideoMetadataService(metadata),
+            systemLogService: new NullSystemLogService());
+
+    public void Dispose()
+    {
+        if (Directory.Exists(_temporaryDirectory))
+        {
+            Directory.Delete(_temporaryDirectory, recursive: true);
+        }
+    }
+
+    private sealed class FixedVideoMetadataService(VideoMetadata? metadata) : IVideoMetadataService
+    {
+        public Task<VideoMetadata?> ExtractMetadataAsync(
+            string filePath,
+            CancellationToken cancellationToken = default) => Task.FromResult(metadata);
+
+        public Task<string?> GenerateThumbnailAsync(
+            string filePath,
+            string outputDir,
+            CancellationToken cancellationToken = default) => Task.FromResult<string?>(null);
+    }
+
+    private sealed class NullSystemLogService : ISystemLogService
+    {
+        public Task WriteAsync(
+            SystemLogLevel level,
+            string category,
+            string message,
+            string? detail = null,
+            Guid? liveRoomId = null,
+            Guid? recordSessionId = null,
+            Guid? recordTaskId = null,
+            CancellationToken cancellationToken = default) => Task.CompletedTask;
+
+        public Task<IReadOnlyList<SystemLogDto>> ListAsync(
+            Guid? liveRoomId = null,
+            Guid? recordSessionId = null,
+            Guid? recordTaskId = null,
+            SystemLogLevel? level = null,
+            string? content = null,
+            int take = 200,
+            CancellationToken cancellationToken = default) =>
+            Task.FromResult<IReadOnlyList<SystemLogDto>>([]);
+    }
+}