fix: guard polling hangs with per-room timeout

2026-05-06 09:50:08 +08:00
parent 56432a9ece
commit 8626af6d80
1 changed files with 48 additions and 1 deletions
@@ -27,6 +27,7 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
    private static readonly TimeSpan ExceptionEmailCooldown = TimeSpan.FromHours(6);
    private static readonly TimeSpan PollDispatchSpacing = TimeSpan.FromMilliseconds(400);
    private static readonly TimeSpan MinimumIdleDelay = TimeSpan.FromSeconds(2);
+    private static readonly TimeSpan PerRoomPollingTimeout = TimeSpan.FromMinutes(2);
    private const int MaxConcurrentLiveRoomPolls = 2;

    private readonly IServiceScopeFactory _serviceScopeFactory;
@@ -252,7 +253,17 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
    {
        try
        {
-            await PollLiveRoomAsync(liveRoom.LiveRoomId, liveRoom.IntervalSeconds, settings, cancellationToken);
+            using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+            timeoutCts.CancelAfter(PerRoomPollingTimeout);
+
+            try
+            {
+                await PollLiveRoomAsync(liveRoom.LiveRoomId, liveRoom.IntervalSeconds, settings, timeoutCts.Token);
+            }
+            catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested && timeoutCts.IsCancellationRequested)
+            {
+                await HandlePollingTimeoutAsync(liveRoom.LiveRoomId, cancellationToken);
+            }
        }
        finally
        {
@@ -260,6 +271,42 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
        }
    }

+    private async Task HandlePollingTimeoutAsync(Guid liveRoomId, CancellationToken cancellationToken)
+    {
+        RequestImmediatePoll(liveRoomId, TransientRetryDelay);
+
+        try
+        {
+            using var scope = _serviceScopeFactory.CreateScope();
+            var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
+            var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
+
+            var liveRoom = await dbContext.LiveRooms.FirstOrDefaultAsync(item => item.Id == liveRoomId, cancellationToken);
+            if (liveRoom is not null)
+            {
+                await UpdateAutoStartDecisionAsync(
+                    dbContext,
+                    liveRoom,
+                    AutoStartDecisionCodes.PollFailedTransient,
+                    "Auto-start is pending because live status polling timed out.",
+                    $"Polling exceeded {PerRoomPollingTimeout.TotalSeconds:0} seconds and was cancelled for recovery.",
+                    CancellationToken.None);
+
+                await logService.WriteAsync(
+                    SystemLogLevel.Warning,
+                    "Scheduler",
+                    "Background polling timed out for a live room. The room will be retried shortly.",
+                    $"Polling exceeded {PerRoomPollingTimeout.TotalSeconds:0} seconds and was cancelled for recovery.",
+                    liveRoomId: liveRoomId,
+                    cancellationToken: CancellationToken.None);
+            }
+        }
+        catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
+        {
+            _logger.LogWarning(ex, "Polling timeout recovery failed for live room {LiveRoomId}", liveRoomId);
+        }
+    }
+
    private async Task PollLiveRoomAsync(
        Guid liveRoomId,
        int intervalSeconds,