diff --git a/src/LiveRecorder.Infrastructure/Services/LiveRoomPollingBackgroundService.cs b/src/LiveRecorder.Infrastructure/Services/LiveRoomPollingBackgroundService.cs index 74e31bd..538e8b0 100644 --- a/src/LiveRecorder.Infrastructure/Services/LiveRoomPollingBackgroundService.cs +++ b/src/LiveRecorder.Infrastructure/Services/LiveRoomPollingBackgroundService.cs @@ -27,6 +27,7 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR private static readonly TimeSpan ExceptionEmailCooldown = TimeSpan.FromHours(6); private static readonly TimeSpan PollDispatchSpacing = TimeSpan.FromMilliseconds(400); private static readonly TimeSpan MinimumIdleDelay = TimeSpan.FromSeconds(2); + private static readonly TimeSpan PerRoomPollingTimeout = TimeSpan.FromMinutes(2); private const int MaxConcurrentLiveRoomPolls = 2; private readonly IServiceScopeFactory _serviceScopeFactory; @@ -252,7 +253,17 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR { try { - await PollLiveRoomAsync(liveRoom.LiveRoomId, liveRoom.IntervalSeconds, settings, cancellationToken); + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(PerRoomPollingTimeout); + + try + { + await PollLiveRoomAsync(liveRoom.LiveRoomId, liveRoom.IntervalSeconds, settings, timeoutCts.Token); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested && timeoutCts.IsCancellationRequested) + { + await HandlePollingTimeoutAsync(liveRoom.LiveRoomId, cancellationToken); + } } finally { @@ -260,6 +271,42 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR } } + private async Task HandlePollingTimeoutAsync(Guid liveRoomId, CancellationToken cancellationToken) + { + RequestImmediatePoll(liveRoomId, TransientRetryDelay); + + try + { + using var scope = _serviceScopeFactory.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var logService = scope.ServiceProvider.GetRequiredService(); + + var liveRoom = await dbContext.LiveRooms.FirstOrDefaultAsync(item => item.Id == liveRoomId, cancellationToken); + if (liveRoom is not null) + { + await UpdateAutoStartDecisionAsync( + dbContext, + liveRoom, + AutoStartDecisionCodes.PollFailedTransient, + "Auto-start is pending because live status polling timed out.", + $"Polling exceeded {PerRoomPollingTimeout.TotalSeconds:0} seconds and was cancelled for recovery.", + CancellationToken.None); + + await logService.WriteAsync( + SystemLogLevel.Warning, + "Scheduler", + "Background polling timed out for a live room. The room will be retried shortly.", + $"Polling exceeded {PerRoomPollingTimeout.TotalSeconds:0} seconds and was cancelled for recovery.", + liveRoomId: liveRoomId, + cancellationToken: CancellationToken.None); + } + } + catch (Exception ex) when (!cancellationToken.IsCancellationRequested) + { + _logger.LogWarning(ex, "Polling timeout recovery failed for live room {LiveRoomId}", liveRoomId); + } + } + private async Task PollLiveRoomAsync( Guid liveRoomId, int intervalSeconds,