feat: improve polling recovery and session cleanup

src/LiveRecorder.Infrastructure/Services/EmailNotificationService.cs

@@ -4,6 +4,7 @@ using System.Text;
 using System.Text.RegularExpressions;
 using LiveRecorder.Application.Abstractions.Notifications;
 using LiveRecorder.Application.Abstractions.Settings;
+using LiveRecorder.Application.Common;
 using LiveRecorder.Application.Models.Settings;
 using LiveRecorder.Domain.Entities;
 using Microsoft.Extensions.Logging;
@@ -41,7 +42,7 @@ public sealed class EmailNotificationService : IEmailNotificationService
             ["title"] = liveRoom.Title,
             ["anchor"] = liveRoom.AnchorName,
             ["sourceUrl"] = liveRoom.SourceUrl,
-            ["detectedAtUtc"] = DateTimeOffset.UtcNow.ToString("O")
+            ["detectedAtUtc"] = ChinaTime.ToBeijingTime(DateTimeOffset.UtcNow).ToString("O")
         });
 
         var subject = RenderSubject(settings.EmailLiveStartedSubjectTemplate, tokens);
@@ -73,7 +74,7 @@ public sealed class EmailNotificationService : IEmailNotificationService
             ["roomId"] = liveRoom?.RoomId,
             ["recordTaskId"] = recordTask?.Id.ToString(),
             ["taskStatus"] = recordTask?.Status.ToString(),
-            ["occurredAtUtc"] = DateTimeOffset.UtcNow.ToString("O")
+            ["occurredAtUtc"] = ChinaTime.ToBeijingTime(DateTimeOffset.UtcNow).ToString("O")
         });
 
         var subject = RenderSubject(settings.EmailExceptionSubjectTemplate, tokens);
@@ -112,7 +113,7 @@ public sealed class EmailNotificationService : IEmailNotificationService
             ["title"] = "Sample Live Title",
             ["anchor"] = "Sample Anchor",
             ["sourceUrl"] = "https://live.douyin.com/123456789",
-            ["detectedAtUtc"] = DateTimeOffset.UtcNow.ToString("O")
+            ["detectedAtUtc"] = ChinaTime.ToBeijingTime(DateTimeOffset.UtcNow).ToString("O")
         });
         var sampleExceptionTokens = CreateTokenMap(new Dictionary<string, string?>
         {
@@ -123,7 +124,7 @@ public sealed class EmailNotificationService : IEmailNotificationService
             ["roomId"] = "123456789",
             ["recordTaskId"] = Guid.NewGuid().ToString(),
             ["taskStatus"] = "Running",
-            ["occurredAtUtc"] = DateTimeOffset.UtcNow.ToString("O")
+            ["occurredAtUtc"] = ChinaTime.ToBeijingTime(DateTimeOffset.UtcNow).ToString("O")
         });
 
         var body = $$"""

src/LiveRecorder.Infrastructure/Services/EventScriptService.cs

@@ -2,6 +2,7 @@ using System.Diagnostics;
 using LiveRecorder.Application.Abstractions.Logging;
 using LiveRecorder.Application.Abstractions.Scripting;
 using LiveRecorder.Application.Abstractions.Settings;
+using LiveRecorder.Application.Common;
 using LiveRecorder.Application.Models.Settings;
 using LiveRecorder.Domain.Entities;
 using LiveRecorder.Domain.Enums;
@@ -354,6 +355,7 @@ public sealed class EventScriptService : IEventScriptService
 
     private static Dictionary<string, string> BuildLiveRoomEnvironment(LiveRoom? liveRoom, DateTimeOffset occurredAt)
     {
+        var localOccurredAt = ChinaTime.ToBeijingTime(occurredAt);
         return new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
         {
             ["LIVE_RECORDER_EVENT"] = string.Empty,
@@ -363,12 +365,13 @@ public sealed class EventScriptService : IEventScriptService
             ["LIVE_RECORDER_TITLE"] = liveRoom?.Title ?? string.Empty,
             ["LIVE_RECORDER_ANCHOR"] = liveRoom?.AnchorName ?? string.Empty,
             ["LIVE_RECORDER_SOURCE_URL"] = liveRoom?.SourceUrl ?? liveRoom?.NormalizedUrl ?? string.Empty,
-            ["LIVE_RECORDER_OCCURRED_AT_UTC"] = occurredAt.ToString("O")
+            ["LIVE_RECORDER_OCCURRED_AT_UTC"] = localOccurredAt.ToString("O")
         };
     }
 
     private static IReadOnlyDictionary<string, string> BuildTestEnvironment(string eventName, DateTimeOffset occurredAt)
     {
+        var localOccurredAt = ChinaTime.ToBeijingTime(occurredAt);
         var environment = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
         {
             ["LIVE_RECORDER_EVENT"] = eventName,
@@ -378,7 +381,7 @@ public sealed class EventScriptService : IEventScriptService
             ["LIVE_RECORDER_TITLE"] = "Sample live title",
             ["LIVE_RECORDER_ANCHOR"] = "Sample anchor",
             ["LIVE_RECORDER_SOURCE_URL"] = "https://live.douyin.com/676493068539",
-            ["LIVE_RECORDER_OCCURRED_AT_UTC"] = occurredAt.ToString("O")
+            ["LIVE_RECORDER_OCCURRED_AT_UTC"] = localOccurredAt.ToString("O")
         };
 
         if (string.Equals(eventName, "segment_completed", StringComparison.OrdinalIgnoreCase))

src/LiveRecorder.Infrastructure/Services/FfmpegService.Runtime.cs

@@ -780,6 +780,11 @@ public sealed partial class FfmpegService
             session.Id,
             currentTask.Id);
 
+        if (!runtime.StopRequested && session.LiveRoomId != Guid.Empty)
+        {
+            _liveRoomPollingSignal.RequestImmediatePoll(session.LiveRoomId, TimeSpan.FromSeconds(2));
+        }
+
         if (session.Status == RecordSessionStatus.Failed && session.LiveRoom is not null)
         {
             await emailNotificationService.SendExceptionAsync(

src/LiveRecorder.Infrastructure/Services/FfmpegService.cs

@@ -31,15 +31,18 @@ public sealed partial class FfmpegService : IFfmpegService
     private int _maxConcurrentTranscodeTasks = 1;
     private readonly IServiceScopeFactory _serviceScopeFactory;
     private readonly IStorageGuardService _storageGuardService;
+    private readonly ILiveRoomPollingSignal _liveRoomPollingSignal;
     private readonly ILogger<FfmpegService> _logger;
 
     public FfmpegService(
         IServiceScopeFactory serviceScopeFactory,
         IStorageGuardService storageGuardService,
+        ILiveRoomPollingSignal liveRoomPollingSignal,
         ILogger<FfmpegService> logger)
     {
         _serviceScopeFactory = serviceScopeFactory;
         _storageGuardService = storageGuardService;
+        _liveRoomPollingSignal = liveRoomPollingSignal;
         _logger = logger;
     }
 

src/LiveRecorder.Infrastructure/Services/ILiveRoomPollingSignal.cs

@@ -0,0 +1,6 @@
+namespace LiveRecorder.Infrastructure.Services;
+
+public interface ILiveRoomPollingSignal
+{
+    void RequestImmediatePoll(Guid liveRoomId, TimeSpan? delay = null);
+}

src/LiveRecorder.Infrastructure/Services/LiveRoomPollingBackgroundService.cs

@@ -19,8 +19,9 @@ using Microsoft.Extensions.Logging;
 
 namespace LiveRecorder.Infrastructure.Services;
 
-public sealed class LiveRoomPollingBackgroundService : BackgroundService
+public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveRoomPollingSignal
 {
+    private static readonly TimeSpan TransientRetryDelay = TimeSpan.FromSeconds(10);
     private static readonly TimeSpan OfflineGracefulStopTimeout = TimeSpan.FromSeconds(20);
     private static readonly TimeSpan OfflineForcedStopTimeout = TimeSpan.FromSeconds(8);
     private static readonly TimeSpan ExceptionEmailCooldown = TimeSpan.FromHours(6);
@@ -32,6 +33,7 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
     private readonly ILogger<LiveRoomPollingBackgroundService> _logger;
     private readonly ConcurrentDictionary<string, DateTimeOffset> _exceptionEmailSentAt = new(StringComparer.OrdinalIgnoreCase);
     private readonly ConcurrentDictionary<Guid, DateTimeOffset> _nextPollDueAt = new();
+    private readonly SemaphoreSlim _wakeSignal = new(0);
 
     public LiveRoomPollingBackgroundService(
         IServiceScopeFactory serviceScopeFactory,
@@ -41,6 +43,23 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
         _logger = logger;
     }
 
+    public void RequestImmediatePoll(Guid liveRoomId, TimeSpan? delay = null)
+    {
+        var requestedDueAt = DateTimeOffset.UtcNow + (delay ?? MinimumIdleDelay);
+        _nextPollDueAt.AddOrUpdate(
+            liveRoomId,
+            requestedDueAt,
+            (_, current) => requestedDueAt < current ? requestedDueAt : current);
+
+        try
+        {
+            _wakeSignal.Release();
+        }
+        catch (SemaphoreFullException)
+        {
+        }
+    }
+
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
         while (!stoppingToken.IsCancellationRequested)
@@ -61,7 +80,7 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
 
                 if (!settings.EnableBackgroundPolling)
                 {
-                    await DelayAsync(TimeSpan.FromSeconds(Math.Clamp(settings.PollingIntervalSeconds, 10, 3600)), stoppingToken);
+                    await WaitForNextRunAsync(TimeSpan.FromSeconds(Math.Clamp(settings.PollingIntervalSeconds, 10, 3600)), stoppingToken);
                     continue;
                 }
 
@@ -160,12 +179,23 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
                 }
             }
 
-            await DelayAsync(delay, stoppingToken);
+            await WaitForNextRunAsync(delay, stoppingToken);
         }
     }
 
-    private static Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken) =>
-        Task.Delay(ClampDelay(delay), cancellationToken);
+    private async Task WaitForNextRunAsync(TimeSpan delay, CancellationToken cancellationToken)
+    {
+        while (_wakeSignal.CurrentCount > 0)
+        {
+            await _wakeSignal.WaitAsync(CancellationToken.None);
+        }
+
+        var effectiveDelay = ClampDelay(delay);
+        var delayTask = Task.Delay(effectiveDelay, cancellationToken);
+        var wakeTask = _wakeSignal.WaitAsync(cancellationToken);
+        var completedTask = await Task.WhenAny(delayTask, wakeTask);
+        await completedTask;
+    }
 
     private static TimeSpan ClampDelay(TimeSpan delay)
     {
@@ -254,6 +284,8 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
             return;
         }
 
+        var nextDueAt = DateTimeOffset.UtcNow.AddSeconds(Math.Clamp(intervalSeconds, 10, 3600));
+
         try
         {
             var adapter = adapterFactory.GetByPlatform(liveRoom.Platform);
@@ -380,6 +412,21 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
             _logger.LogWarning(ex, "Polling room {RoomId} failed", liveRoom.RoomId);
 
             var isTransient = IsTransientPollingException(ex, cancellationToken);
+            if (isTransient)
+            {
+                nextDueAt = DateTimeOffset.UtcNow.Add(TransientRetryDelay);
+            }
+
+            await UpdateAutoStartDecisionAsync(
+                dbContext,
+                liveRoom,
+                isTransient ? AutoStartDecisionCodes.PollFailedTransient : AutoStartDecisionCodes.PollFailed,
+                isTransient
+                    ? "Auto-start is pending because live status polling failed temporarily."
+                    : "Auto-start failed because live status polling failed.",
+                BuildPollingFailureDetail(ex),
+                cancellationToken);
+
             await logService.WriteAsync(
                 isTransient ? SystemLogLevel.Warning : SystemLogLevel.Error,
                 "Scheduler",
@@ -408,7 +455,7 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
         }
         finally
         {
-            _nextPollDueAt[liveRoomId] = DateTimeOffset.UtcNow.AddSeconds(Math.Clamp(intervalSeconds, 10, 3600));
+            _nextPollDueAt[liveRoomId] = nextDueAt;
         }
     }
 
@@ -605,6 +652,12 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
                IsTransientPollingException(exception.InnerException, cancellationToken);
     }
 
+    private static string BuildPollingFailureDetail(Exception exception)
+    {
+        var root = exception.GetBaseException();
+        return Truncate($"{root.GetType().Name}: {root.Message}", 2048) ?? exception.GetType().Name;
+    }
+
     private bool ShouldSendExceptionEmail(string key)
     {
         var now = DateTimeOffset.UtcNow;