fix: guard polling hangs with per-room timeout
This commit is contained in:
@@ -27,6 +27,7 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
|
|||||||
private static readonly TimeSpan ExceptionEmailCooldown = TimeSpan.FromHours(6);
|
private static readonly TimeSpan ExceptionEmailCooldown = TimeSpan.FromHours(6);
|
||||||
private static readonly TimeSpan PollDispatchSpacing = TimeSpan.FromMilliseconds(400);
|
private static readonly TimeSpan PollDispatchSpacing = TimeSpan.FromMilliseconds(400);
|
||||||
private static readonly TimeSpan MinimumIdleDelay = TimeSpan.FromSeconds(2);
|
private static readonly TimeSpan MinimumIdleDelay = TimeSpan.FromSeconds(2);
|
||||||
|
private static readonly TimeSpan PerRoomPollingTimeout = TimeSpan.FromMinutes(2);
|
||||||
private const int MaxConcurrentLiveRoomPolls = 2;
|
private const int MaxConcurrentLiveRoomPolls = 2;
|
||||||
|
|
||||||
private readonly IServiceScopeFactory _serviceScopeFactory;
|
private readonly IServiceScopeFactory _serviceScopeFactory;
|
||||||
@@ -252,7 +253,17 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
|
|||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
await PollLiveRoomAsync(liveRoom.LiveRoomId, liveRoom.IntervalSeconds, settings, cancellationToken);
|
using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||||
|
timeoutCts.CancelAfter(PerRoomPollingTimeout);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await PollLiveRoomAsync(liveRoom.LiveRoomId, liveRoom.IntervalSeconds, settings, timeoutCts.Token);
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested && timeoutCts.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
await HandlePollingTimeoutAsync(liveRoom.LiveRoomId, cancellationToken);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
finally
|
finally
|
||||||
{
|
{
|
||||||
@@ -260,6 +271,42 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async Task HandlePollingTimeoutAsync(Guid liveRoomId, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
RequestImmediatePoll(liveRoomId, TransientRetryDelay);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var scope = _serviceScopeFactory.CreateScope();
|
||||||
|
var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
|
||||||
|
var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
|
||||||
|
|
||||||
|
var liveRoom = await dbContext.LiveRooms.FirstOrDefaultAsync(item => item.Id == liveRoomId, cancellationToken);
|
||||||
|
if (liveRoom is not null)
|
||||||
|
{
|
||||||
|
await UpdateAutoStartDecisionAsync(
|
||||||
|
dbContext,
|
||||||
|
liveRoom,
|
||||||
|
AutoStartDecisionCodes.PollFailedTransient,
|
||||||
|
"Auto-start is pending because live status polling timed out.",
|
||||||
|
$"Polling exceeded {PerRoomPollingTimeout.TotalSeconds:0} seconds and was cancelled for recovery.",
|
||||||
|
CancellationToken.None);
|
||||||
|
|
||||||
|
await logService.WriteAsync(
|
||||||
|
SystemLogLevel.Warning,
|
||||||
|
"Scheduler",
|
||||||
|
"Background polling timed out for a live room. The room will be retried shortly.",
|
||||||
|
$"Polling exceeded {PerRoomPollingTimeout.TotalSeconds:0} seconds and was cancelled for recovery.",
|
||||||
|
liveRoomId: liveRoomId,
|
||||||
|
cancellationToken: CancellationToken.None);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(ex, "Polling timeout recovery failed for live room {LiveRoomId}", liveRoomId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private async Task PollLiveRoomAsync(
|
private async Task PollLiveRoomAsync(
|
||||||
Guid liveRoomId,
|
Guid liveRoomId,
|
||||||
int intervalSeconds,
|
int intervalSeconds,
|
||||||
|
|||||||
Reference in New Issue
Block a user