1111
This commit is contained in:
@@ -0,0 +1,243 @@
|
||||
using LiveRecorder.Application.Abstractions.Logging;
|
||||
using LiveRecorder.Application.Abstractions.Notifications;
|
||||
using LiveRecorder.Application.Abstractions.Platforms;
|
||||
using LiveRecorder.Application.Abstractions.Recording;
|
||||
using LiveRecorder.Application.Abstractions.Settings;
|
||||
using LiveRecorder.Application.Models.RecordTasks;
|
||||
using LiveRecorder.Application.Services;
|
||||
using LiveRecorder.Domain.Enums;
|
||||
using LiveRecorder.Infrastructure.Persistence;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace LiveRecorder.Infrastructure.Services;
|
||||
|
||||
public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
{
|
||||
private readonly IServiceScopeFactory _serviceScopeFactory;
|
||||
private readonly ILogger<LiveRoomPollingBackgroundService> _logger;
|
||||
|
||||
public LiveRoomPollingBackgroundService(
|
||||
IServiceScopeFactory serviceScopeFactory,
|
||||
ILogger<LiveRoomPollingBackgroundService> logger)
|
||||
{
|
||||
_serviceScopeFactory = serviceScopeFactory;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
var delaySeconds = 60;
|
||||
|
||||
try
|
||||
{
|
||||
using var scope = _serviceScopeFactory.CreateScope();
|
||||
var settingsService = scope.ServiceProvider.GetRequiredService<ISystemSettingsService>();
|
||||
var settings = await settingsService.GetAsync(stoppingToken);
|
||||
var emailNotificationService = scope.ServiceProvider.GetRequiredService<IEmailNotificationService>();
|
||||
|
||||
delaySeconds = settings.PollingIntervalSeconds;
|
||||
if (!settings.EnableBackgroundPolling)
|
||||
{
|
||||
await DelayAsync(delaySeconds, stoppingToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
var dbContext = scope.ServiceProvider.GetRequiredService<LiveRecorderDbContext>();
|
||||
var adapterFactory = scope.ServiceProvider.GetRequiredService<ILivePlatformAdapterFactory>();
|
||||
var ffmpegService = scope.ServiceProvider.GetRequiredService<IFfmpegService>();
|
||||
var recordService = scope.ServiceProvider.GetRequiredService<RecordService>();
|
||||
var liveRoomStatusService = scope.ServiceProvider.GetRequiredService<LiveRoomStatusService>();
|
||||
var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
|
||||
|
||||
var liveRooms = (await dbContext.LiveRooms.ToListAsync(stoppingToken))
|
||||
.Where(static item => item.IsEnabled)
|
||||
.OrderBy(static item => item.UpdatedAt)
|
||||
.ToList();
|
||||
|
||||
foreach (var liveRoom in liveRooms)
|
||||
{
|
||||
if (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var adapter = adapterFactory.GetByPlatform(liveRoom.Platform);
|
||||
var liveStatus = await adapter.GetLiveStatusAsync(liveRoom.RoomId, stoppingToken);
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
await liveRoomStatusService.ApplySnapshotAsync(liveRoom, liveStatus, now, stoppingToken);
|
||||
|
||||
await dbContext.SaveChangesAsync(stoppingToken);
|
||||
|
||||
if (!liveStatus.IsLive)
|
||||
{
|
||||
await CompleteActiveSessionsForOfflineRoomAsync(
|
||||
dbContext,
|
||||
ffmpegService,
|
||||
logService,
|
||||
liveRoom.Id,
|
||||
stoppingToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!settings.AutoStartRecordingOnLive)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var hasRunningSession = await dbContext.RecordSessions.AnyAsync(
|
||||
item => item.LiveRoomId == liveRoom.Id &&
|
||||
(item.Status == RecordSessionStatus.Starting ||
|
||||
item.Status == RecordSessionStatus.Running ||
|
||||
item.Status == RecordSessionStatus.Stopping),
|
||||
stoppingToken);
|
||||
|
||||
if (hasRunningSession)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
_logger.LogInformation("Auto-start recording for live room {RoomId}", liveRoom.RoomId);
|
||||
await logService.WriteAsync(
|
||||
SystemLogLevel.Info,
|
||||
"Scheduler",
|
||||
"Live detected by background poller. Auto-starting recording task.",
|
||||
liveRoomId: liveRoom.Id,
|
||||
cancellationToken: stoppingToken);
|
||||
|
||||
await recordService.StartAsync(
|
||||
new StartRecordTaskRequest
|
||||
{
|
||||
LiveRoomId = liveRoom.Id,
|
||||
PreferredQuality = settings.DefaultQuality,
|
||||
OutputFormat = settings.DefaultOutputFormat
|
||||
},
|
||||
stoppingToken);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Polling room {RoomId} failed", liveRoom.RoomId);
|
||||
|
||||
var isTransient = IsTransientPollingException(ex, stoppingToken);
|
||||
await logService.WriteAsync(
|
||||
isTransient ? SystemLogLevel.Warning : SystemLogLevel.Error,
|
||||
"Scheduler",
|
||||
isTransient
|
||||
? "Transient background polling failure. The room will be retried on the next cycle."
|
||||
: "Background polling failed for a live room.",
|
||||
ex.ToString(),
|
||||
liveRoomId: liveRoom.Id,
|
||||
cancellationToken: stoppingToken);
|
||||
|
||||
if (!isTransient)
|
||||
{
|
||||
await emailNotificationService.SendExceptionAsync(
|
||||
"Scheduler",
|
||||
"Background polling failed for a live room.",
|
||||
ex.ToString(),
|
||||
liveRoom,
|
||||
cancellationToken: stoppingToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Background live room polling failed");
|
||||
|
||||
try
|
||||
{
|
||||
using var notificationScope = _serviceScopeFactory.CreateScope();
|
||||
var emailNotificationService = notificationScope.ServiceProvider.GetRequiredService<IEmailNotificationService>();
|
||||
await emailNotificationService.SendExceptionAsync(
|
||||
"Scheduler",
|
||||
"Background live room polling failed.",
|
||||
ex.ToString(),
|
||||
cancellationToken: stoppingToken);
|
||||
}
|
||||
catch (Exception notificationEx)
|
||||
{
|
||||
_logger.LogWarning(notificationEx, "Scheduler failure notification send failed");
|
||||
}
|
||||
}
|
||||
|
||||
await DelayAsync(delaySeconds, stoppingToken);
|
||||
}
|
||||
}
|
||||
|
||||
private static Task DelayAsync(int delaySeconds, CancellationToken cancellationToken) =>
|
||||
Task.Delay(TimeSpan.FromSeconds(Math.Clamp(delaySeconds, 10, 3600)), cancellationToken);
|
||||
|
||||
private static async Task CompleteActiveSessionsForOfflineRoomAsync(
|
||||
LiveRecorderDbContext dbContext,
|
||||
IFfmpegService ffmpegService,
|
||||
ISystemLogService logService,
|
||||
Guid liveRoomId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var activeSessions = await dbContext.RecordSessions
|
||||
.Where(item => item.LiveRoomId == liveRoomId &&
|
||||
(item.Status == RecordSessionStatus.Starting ||
|
||||
item.Status == RecordSessionStatus.Running ||
|
||||
item.Status == RecordSessionStatus.Stopping))
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
activeSessions = activeSessions
|
||||
.OrderBy(item => item.CreatedAt)
|
||||
.ToList();
|
||||
|
||||
foreach (var activeSession in activeSessions)
|
||||
{
|
||||
if (ffmpegService.IsRunning(activeSession.Id))
|
||||
{
|
||||
await ffmpegService.CompleteAsync(activeSession.Id, cancellationToken);
|
||||
await logService.WriteAsync(
|
||||
SystemLogLevel.Info,
|
||||
"Scheduler",
|
||||
"Live room is offline. Completing the active recording session.",
|
||||
liveRoomId: liveRoomId,
|
||||
recordSessionId: activeSession.Id,
|
||||
cancellationToken: cancellationToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (await ffmpegService.TryReconcileInactiveSessionAsync(activeSession.Id, cancellationToken))
|
||||
{
|
||||
await logService.WriteAsync(
|
||||
SystemLogLevel.Warning,
|
||||
"Scheduler",
|
||||
"Recovered a stale active recording session after the room was detected offline.",
|
||||
liveRoomId: liveRoomId,
|
||||
recordSessionId: activeSession.Id,
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsTransientPollingException(Exception exception, CancellationToken cancellationToken)
|
||||
{
|
||||
if (exception is OperationCanceledException && cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (exception is HttpRequestException or IOException or TimeoutException)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return exception.InnerException is not null &&
|
||||
IsTransientPollingException(exception.InnerException, cancellationToken);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user