feat: improve polling recovery and session cleanup
This commit is contained in:
@@ -15,4 +15,8 @@ public static class AutoStartDecisionCodes
|
||||
public const string SkippedDebounce = "skipped_debounce";
|
||||
|
||||
public const string FailedStartup = "failed_startup";
|
||||
|
||||
public const string PollFailedTransient = "poll_failed_transient";
|
||||
|
||||
public const string PollFailed = "poll_failed";
|
||||
}
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
namespace LiveRecorder.Application.Common;
|
||||
|
||||
public static class ChinaTime
|
||||
{
|
||||
private static readonly Lazy<TimeZoneInfo> TimeZoneInfoLazy = new(ResolveTimeZoneInfo);
|
||||
|
||||
public static TimeZoneInfo Zone => TimeZoneInfoLazy.Value;
|
||||
|
||||
public static DateTimeOffset ToBeijingTime(DateTimeOffset value)
|
||||
{
|
||||
var local = TimeZoneInfo.ConvertTime(value, Zone);
|
||||
return new DateTimeOffset(local.DateTime, Zone.GetUtcOffset(local.DateTime));
|
||||
}
|
||||
|
||||
private static TimeZoneInfo ResolveTimeZoneInfo()
|
||||
{
|
||||
foreach (var id in new[] { "Asia/Shanghai", "China Standard Time" })
|
||||
{
|
||||
try
|
||||
{
|
||||
return TimeZoneInfo.FindSystemTimeZoneById(id);
|
||||
}
|
||||
catch (TimeZoneNotFoundException)
|
||||
{
|
||||
}
|
||||
catch (InvalidTimeZoneException)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
return TimeZoneInfo.CreateCustomTimeZone(
|
||||
"UTC+08",
|
||||
TimeSpan.FromHours(8),
|
||||
"UTC+08",
|
||||
"UTC+08");
|
||||
}
|
||||
}
|
||||
@@ -129,3 +129,8 @@ public sealed class DeleteRecordSessionsRequest
|
||||
|
||||
public bool DeleteFiles { get; set; }
|
||||
}
|
||||
|
||||
public sealed class DeleteMissingFileRecordSessionsRequest
|
||||
{
|
||||
public bool DeleteFiles { get; set; }
|
||||
}
|
||||
|
||||
@@ -181,7 +181,7 @@ public sealed class SystemSettingsDto
|
||||
<li><strong>Room ID:</strong> {{roomId}}</li>
|
||||
<li><strong>Title:</strong> {{title}}</li>
|
||||
<li><strong>Anchor:</strong> {{anchor}}</li>
|
||||
<li><strong>Detected At (UTC):</strong> {{detectedAtUtc}}</li>
|
||||
<li><strong>Detected At (Beijing Time):</strong> {{detectedAtUtc}}</li>
|
||||
</ul>
|
||||
<p><strong>Source URL:</strong> <a href="{{sourceUrl}}">{{sourceUrl}}</a></p>
|
||||
</div>
|
||||
@@ -199,7 +199,7 @@ public sealed class SystemSettingsDto
|
||||
<li><strong>Room ID:</strong> {{roomId}}</li>
|
||||
<li><strong>Record Task ID:</strong> {{recordTaskId}}</li>
|
||||
<li><strong>Task Status:</strong> {{taskStatus}}</li>
|
||||
<li><strong>Occurred At (UTC):</strong> {{occurredAtUtc}}</li>
|
||||
<li><strong>Occurred At (Beijing Time):</strong> {{occurredAtUtc}}</li>
|
||||
</ul>
|
||||
<div style="margin-top: 16px; padding: 12px 14px; border-radius: 8px; background: #f5f5f5; white-space: pre-wrap;">{{detail}}</div>
|
||||
</div>
|
||||
@@ -362,7 +362,7 @@ public sealed class UpdateSystemSettingsRequest
|
||||
<li><strong>Room ID:</strong> {{roomId}}</li>
|
||||
<li><strong>Title:</strong> {{title}}</li>
|
||||
<li><strong>Anchor:</strong> {{anchor}}</li>
|
||||
<li><strong>Detected At (UTC):</strong> {{detectedAtUtc}}</li>
|
||||
<li><strong>Detected At (Beijing Time):</strong> {{detectedAtUtc}}</li>
|
||||
</ul>
|
||||
<p><strong>Source URL:</strong> <a href="{{sourceUrl}}">{{sourceUrl}}</a></p>
|
||||
</div>
|
||||
@@ -380,7 +380,7 @@ public sealed class UpdateSystemSettingsRequest
|
||||
<li><strong>Room ID:</strong> {{roomId}}</li>
|
||||
<li><strong>Record Task ID:</strong> {{recordTaskId}}</li>
|
||||
<li><strong>Task Status:</strong> {{taskStatus}}</li>
|
||||
<li><strong>Occurred At (UTC):</strong> {{occurredAtUtc}}</li>
|
||||
<li><strong>Occurred At (Beijing Time):</strong> {{occurredAtUtc}}</li>
|
||||
</ul>
|
||||
<div style="margin-top: 16px; padding: 12px 14px; border-radius: 8px; background: #f5f5f5; white-space: pre-wrap;">{{detail}}</div>
|
||||
</div>
|
||||
@@ -435,7 +435,7 @@ public sealed class SendTestEmailRequest
|
||||
<li><strong>Room ID:</strong> {{roomId}}</li>
|
||||
<li><strong>Title:</strong> {{title}}</li>
|
||||
<li><strong>Anchor:</strong> {{anchor}}</li>
|
||||
<li><strong>Detected At (UTC):</strong> {{detectedAtUtc}}</li>
|
||||
<li><strong>Detected At (Beijing Time):</strong> {{detectedAtUtc}}</li>
|
||||
</ul>
|
||||
<p><strong>Source URL:</strong> <a href="{{sourceUrl}}">{{sourceUrl}}</a></p>
|
||||
</div>
|
||||
@@ -453,7 +453,7 @@ public sealed class SendTestEmailRequest
|
||||
<li><strong>Room ID:</strong> {{roomId}}</li>
|
||||
<li><strong>Record Task ID:</strong> {{recordTaskId}}</li>
|
||||
<li><strong>Task Status:</strong> {{taskStatus}}</li>
|
||||
<li><strong>Occurred At (UTC):</strong> {{occurredAtUtc}}</li>
|
||||
<li><strong>Occurred At (Beijing Time):</strong> {{occurredAtUtc}}</li>
|
||||
</ul>
|
||||
<div style="margin-top: 16px; padding: 12px 14px; border-radius: 8px; background: #f5f5f5; white-space: pre-wrap;">{{detail}}</div>
|
||||
</div>
|
||||
|
||||
@@ -774,6 +774,7 @@ public sealed class RecordService
|
||||
RecordSaveMode saveMode,
|
||||
DateTimeOffset now)
|
||||
{
|
||||
var localNow = ChinaTime.ToBeijingTime(now);
|
||||
var safeRoomId = SanitizeFileName(roomId, "room");
|
||||
var effectiveFileNameTemplate = EnsureSegmentSuffixTemplate(outputFileNameTemplate, saveMode);
|
||||
var baseFileStem = BuildFileNameStem(
|
||||
@@ -782,7 +783,7 @@ public sealed class RecordService
|
||||
safeRoomId,
|
||||
anchorName,
|
||||
title,
|
||||
now,
|
||||
localNow,
|
||||
segmentSuffix: string.Empty);
|
||||
var directoryPath = BuildDirectoryPath(
|
||||
outputDirectoryTemplate,
|
||||
@@ -790,7 +791,7 @@ public sealed class RecordService
|
||||
safeRoomId,
|
||||
anchorName,
|
||||
title,
|
||||
now,
|
||||
localNow,
|
||||
baseFileStem);
|
||||
var fileNameStem = BuildFileNameStem(
|
||||
effectiveFileNameTemplate,
|
||||
@@ -798,7 +799,7 @@ public sealed class RecordService
|
||||
safeRoomId,
|
||||
anchorName,
|
||||
title,
|
||||
now,
|
||||
localNow,
|
||||
saveMode == RecordSaveMode.Segmented ? "_%05d" : string.Empty);
|
||||
var folder = Path.Combine(outputRoot, directoryPath);
|
||||
var extension = outputFormat == RecordOutputFormat.Ts ? "ts" : "mp4";
|
||||
|
||||
@@ -253,6 +253,36 @@ public sealed class RecordSessionService
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<DeleteCompletedRecordTasksResultDto> DeleteMissingFilesAsync(
|
||||
DeleteMissingFileRecordSessionsRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
await _stoppedOrphanRecordSessionCleanupService.CleanupAsync(cancellationToken: cancellationToken);
|
||||
await ReconcileActiveSessionsAsync(null, cancellationToken);
|
||||
|
||||
var sessions = await _recordSessionRepository.ListAsync(null, cancellationToken);
|
||||
var missingFileSessionIds = sessions
|
||||
.Where(CanDeleteMissingFileSession)
|
||||
.Select(static item => item.Id)
|
||||
.Distinct()
|
||||
.ToArray();
|
||||
|
||||
if (missingFileSessionIds.Length == 0)
|
||||
{
|
||||
return RecordService.CreateEmptyDeleteResult();
|
||||
}
|
||||
|
||||
return await DeleteAsync(
|
||||
new DeleteRecordSessionsRequest
|
||||
{
|
||||
SessionIds = missingFileSessionIds,
|
||||
DeleteFiles = request.DeleteFiles
|
||||
},
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private async Task ReconcileActiveSessionsAsync(Guid? liveRoomId, CancellationToken cancellationToken)
|
||||
{
|
||||
var sessions = await _recordSessionRepository.ListAsync(liveRoomId, cancellationToken);
|
||||
@@ -271,6 +301,39 @@ public sealed class RecordSessionService
|
||||
private static bool IsActiveStatus(RecordSessionStatus status) =>
|
||||
status is RecordSessionStatus.Starting or RecordSessionStatus.Running or RecordSessionStatus.Stopping;
|
||||
|
||||
private static bool CanDeleteMissingFileSession(RecordSession session)
|
||||
{
|
||||
if (IsActiveStatus(session.Status))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (session.RecordTasks.Count == 0)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return session.RecordTasks.All(static task => !HasExistingVideoFile(task));
|
||||
}
|
||||
|
||||
private static bool HasExistingVideoFile(RecordTask task)
|
||||
{
|
||||
var candidatePath = !string.IsNullOrWhiteSpace(task.Result?.FilePath)
|
||||
? task.Result!.FilePath
|
||||
: task.OutputFilePath;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(candidatePath))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var resolvedPath = Path.IsPathRooted(candidatePath)
|
||||
? candidatePath
|
||||
: Path.GetFullPath(candidatePath, AppContext.BaseDirectory);
|
||||
|
||||
return File.Exists(resolvedPath);
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<SystemLogEntry>> ListRelatedLogsAsync(
|
||||
Guid recordSessionId,
|
||||
IReadOnlyCollection<Guid> taskIds,
|
||||
|
||||
@@ -225,7 +225,7 @@ public sealed class SystemSettingsService : ISystemSettingsService
|
||||
NotifyOnLiveStarted = bool.TryParse(GetValue(lookup, NotifyOnLiveStartedKey, "true"), out var notifyOnLiveStarted) && notifyOnLiveStarted,
|
||||
NotifyOnException = bool.TryParse(GetValue(lookup, NotifyOnExceptionKey, "true"), out var notifyOnException) && notifyOnException,
|
||||
EmailLiveStartedSubjectTemplate = GetValue(lookup, EmailLiveStartedSubjectTemplateKey, "[{{appName}}] Live started: {{anchor}} {{title}} ({{roomId}})"),
|
||||
EmailLiveStartedBodyTemplateHtml = GetValue(
|
||||
EmailLiveStartedBodyTemplateHtml = NormalizeBeijingTimeTemplate(GetValue(
|
||||
lookup,
|
||||
EmailLiveStartedBodyTemplateHtmlKey,
|
||||
"""
|
||||
@@ -237,13 +237,13 @@ public sealed class SystemSettingsService : ISystemSettingsService
|
||||
<li><strong>Room ID:</strong> {{roomId}}</li>
|
||||
<li><strong>Title:</strong> {{title}}</li>
|
||||
<li><strong>Anchor:</strong> {{anchor}}</li>
|
||||
<li><strong>Detected At (UTC):</strong> {{detectedAtUtc}}</li>
|
||||
<li><strong>Detected At (Beijing Time):</strong> {{detectedAtUtc}}</li>
|
||||
</ul>
|
||||
<p><strong>Source URL:</strong> <a href="{{sourceUrl}}">{{sourceUrl}}</a></p>
|
||||
</div>
|
||||
"""),
|
||||
""")),
|
||||
EmailExceptionSubjectTemplate = GetValue(lookup, EmailExceptionSubjectTemplateKey, "[{{appName}}] Exception: {{source}}"),
|
||||
EmailExceptionBodyTemplateHtml = GetValue(
|
||||
EmailExceptionBodyTemplateHtml = NormalizeBeijingTimeTemplate(GetValue(
|
||||
lookup,
|
||||
EmailExceptionBodyTemplateHtmlKey,
|
||||
"""
|
||||
@@ -256,11 +256,11 @@ public sealed class SystemSettingsService : ISystemSettingsService
|
||||
<li><strong>Room ID:</strong> {{roomId}}</li>
|
||||
<li><strong>Record Task ID:</strong> {{recordTaskId}}</li>
|
||||
<li><strong>Task Status:</strong> {{taskStatus}}</li>
|
||||
<li><strong>Occurred At (UTC):</strong> {{occurredAtUtc}}</li>
|
||||
<li><strong>Occurred At (Beijing Time):</strong> {{occurredAtUtc}}</li>
|
||||
</ul>
|
||||
<div style="margin-top: 16px; padding: 12px 14px; border-radius: 8px; background: #f5f5f5; white-space: pre-wrap;">{{detail}}</div>
|
||||
</div>
|
||||
"""),
|
||||
""")),
|
||||
EnableWebhookNotification = bool.TryParse(GetValue(lookup, EnableWebhookNotificationKey, "false"), out var enableWebhookNotification) && enableWebhookNotification,
|
||||
WebhookUrl = GetValue(lookup, WebhookUrlKey, string.Empty),
|
||||
WebhookHeaders = GetValue(lookup, WebhookHeadersKey, string.Empty),
|
||||
@@ -437,6 +437,11 @@ public sealed class SystemSettingsService : ISystemSettingsService
|
||||
? EventScriptSourceModes.Inline
|
||||
: EventScriptSourceModes.Path;
|
||||
|
||||
private static string NormalizeBeijingTimeTemplate(string value) =>
|
||||
value
|
||||
.Replace("Detected At (UTC)", "Detected At (Beijing Time)", StringComparison.Ordinal)
|
||||
.Replace("Occurred At (UTC)", "Occurred At (Beijing Time)", StringComparison.Ordinal);
|
||||
|
||||
private async Task UpsertAsync(string key, string value, DateTimeOffset updatedAt, CancellationToken cancellationToken)
|
||||
{
|
||||
var existing = await _appSettingRepository.GetByKeyAsync(key, cancellationToken);
|
||||
|
||||
@@ -107,7 +107,7 @@ public sealed class DatabaseInitializer
|
||||
<li><strong>Room ID:</strong> {{roomId}}</li>
|
||||
<li><strong>Title:</strong> {{title}}</li>
|
||||
<li><strong>Anchor:</strong> {{anchor}}</li>
|
||||
<li><strong>Detected At (UTC):</strong> {{detectedAtUtc}}</li>
|
||||
<li><strong>Detected At (Beijing Time):</strong> {{detectedAtUtc}}</li>
|
||||
</ul>
|
||||
<p><strong>Source URL:</strong> <a href="{{sourceUrl}}">{{sourceUrl}}</a></p>
|
||||
</div>
|
||||
@@ -123,7 +123,7 @@ public sealed class DatabaseInitializer
|
||||
<li><strong>Room ID:</strong> {{roomId}}</li>
|
||||
<li><strong>Record Task ID:</strong> {{recordTaskId}}</li>
|
||||
<li><strong>Task Status:</strong> {{taskStatus}}</li>
|
||||
<li><strong>Occurred At (UTC):</strong> {{occurredAtUtc}}</li>
|
||||
<li><strong>Occurred At (Beijing Time):</strong> {{occurredAtUtc}}</li>
|
||||
</ul>
|
||||
<div style="margin-top: 16px; padding: 12px 14px; border-radius: 8px; background: #f5f5f5; white-space: pre-wrap;">{{detail}}</div>
|
||||
</div>
|
||||
|
||||
@@ -4,6 +4,7 @@ using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using LiveRecorder.Application.Abstractions.Notifications;
|
||||
using LiveRecorder.Application.Abstractions.Settings;
|
||||
using LiveRecorder.Application.Common;
|
||||
using LiveRecorder.Application.Models.Settings;
|
||||
using LiveRecorder.Domain.Entities;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -41,7 +42,7 @@ public sealed class EmailNotificationService : IEmailNotificationService
|
||||
["title"] = liveRoom.Title,
|
||||
["anchor"] = liveRoom.AnchorName,
|
||||
["sourceUrl"] = liveRoom.SourceUrl,
|
||||
["detectedAtUtc"] = DateTimeOffset.UtcNow.ToString("O")
|
||||
["detectedAtUtc"] = ChinaTime.ToBeijingTime(DateTimeOffset.UtcNow).ToString("O")
|
||||
});
|
||||
|
||||
var subject = RenderSubject(settings.EmailLiveStartedSubjectTemplate, tokens);
|
||||
@@ -73,7 +74,7 @@ public sealed class EmailNotificationService : IEmailNotificationService
|
||||
["roomId"] = liveRoom?.RoomId,
|
||||
["recordTaskId"] = recordTask?.Id.ToString(),
|
||||
["taskStatus"] = recordTask?.Status.ToString(),
|
||||
["occurredAtUtc"] = DateTimeOffset.UtcNow.ToString("O")
|
||||
["occurredAtUtc"] = ChinaTime.ToBeijingTime(DateTimeOffset.UtcNow).ToString("O")
|
||||
});
|
||||
|
||||
var subject = RenderSubject(settings.EmailExceptionSubjectTemplate, tokens);
|
||||
@@ -112,7 +113,7 @@ public sealed class EmailNotificationService : IEmailNotificationService
|
||||
["title"] = "Sample Live Title",
|
||||
["anchor"] = "Sample Anchor",
|
||||
["sourceUrl"] = "https://live.douyin.com/123456789",
|
||||
["detectedAtUtc"] = DateTimeOffset.UtcNow.ToString("O")
|
||||
["detectedAtUtc"] = ChinaTime.ToBeijingTime(DateTimeOffset.UtcNow).ToString("O")
|
||||
});
|
||||
var sampleExceptionTokens = CreateTokenMap(new Dictionary<string, string?>
|
||||
{
|
||||
@@ -123,7 +124,7 @@ public sealed class EmailNotificationService : IEmailNotificationService
|
||||
["roomId"] = "123456789",
|
||||
["recordTaskId"] = Guid.NewGuid().ToString(),
|
||||
["taskStatus"] = "Running",
|
||||
["occurredAtUtc"] = DateTimeOffset.UtcNow.ToString("O")
|
||||
["occurredAtUtc"] = ChinaTime.ToBeijingTime(DateTimeOffset.UtcNow).ToString("O")
|
||||
});
|
||||
|
||||
var body = $$"""
|
||||
|
||||
@@ -2,6 +2,7 @@ using System.Diagnostics;
|
||||
using LiveRecorder.Application.Abstractions.Logging;
|
||||
using LiveRecorder.Application.Abstractions.Scripting;
|
||||
using LiveRecorder.Application.Abstractions.Settings;
|
||||
using LiveRecorder.Application.Common;
|
||||
using LiveRecorder.Application.Models.Settings;
|
||||
using LiveRecorder.Domain.Entities;
|
||||
using LiveRecorder.Domain.Enums;
|
||||
@@ -354,6 +355,7 @@ public sealed class EventScriptService : IEventScriptService
|
||||
|
||||
private static Dictionary<string, string> BuildLiveRoomEnvironment(LiveRoom? liveRoom, DateTimeOffset occurredAt)
|
||||
{
|
||||
var localOccurredAt = ChinaTime.ToBeijingTime(occurredAt);
|
||||
return new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["LIVE_RECORDER_EVENT"] = string.Empty,
|
||||
@@ -363,12 +365,13 @@ public sealed class EventScriptService : IEventScriptService
|
||||
["LIVE_RECORDER_TITLE"] = liveRoom?.Title ?? string.Empty,
|
||||
["LIVE_RECORDER_ANCHOR"] = liveRoom?.AnchorName ?? string.Empty,
|
||||
["LIVE_RECORDER_SOURCE_URL"] = liveRoom?.SourceUrl ?? liveRoom?.NormalizedUrl ?? string.Empty,
|
||||
["LIVE_RECORDER_OCCURRED_AT_UTC"] = occurredAt.ToString("O")
|
||||
["LIVE_RECORDER_OCCURRED_AT_UTC"] = localOccurredAt.ToString("O")
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, string> BuildTestEnvironment(string eventName, DateTimeOffset occurredAt)
|
||||
{
|
||||
var localOccurredAt = ChinaTime.ToBeijingTime(occurredAt);
|
||||
var environment = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["LIVE_RECORDER_EVENT"] = eventName,
|
||||
@@ -378,7 +381,7 @@ public sealed class EventScriptService : IEventScriptService
|
||||
["LIVE_RECORDER_TITLE"] = "Sample live title",
|
||||
["LIVE_RECORDER_ANCHOR"] = "Sample anchor",
|
||||
["LIVE_RECORDER_SOURCE_URL"] = "https://live.douyin.com/676493068539",
|
||||
["LIVE_RECORDER_OCCURRED_AT_UTC"] = occurredAt.ToString("O")
|
||||
["LIVE_RECORDER_OCCURRED_AT_UTC"] = localOccurredAt.ToString("O")
|
||||
};
|
||||
|
||||
if (string.Equals(eventName, "segment_completed", StringComparison.OrdinalIgnoreCase))
|
||||
|
||||
@@ -780,6 +780,11 @@ public sealed partial class FfmpegService
|
||||
session.Id,
|
||||
currentTask.Id);
|
||||
|
||||
if (!runtime.StopRequested && session.LiveRoomId != Guid.Empty)
|
||||
{
|
||||
_liveRoomPollingSignal.RequestImmediatePoll(session.LiveRoomId, TimeSpan.FromSeconds(2));
|
||||
}
|
||||
|
||||
if (session.Status == RecordSessionStatus.Failed && session.LiveRoom is not null)
|
||||
{
|
||||
await emailNotificationService.SendExceptionAsync(
|
||||
|
||||
@@ -31,15 +31,18 @@ public sealed partial class FfmpegService : IFfmpegService
|
||||
private int _maxConcurrentTranscodeTasks = 1;
|
||||
private readonly IServiceScopeFactory _serviceScopeFactory;
|
||||
private readonly IStorageGuardService _storageGuardService;
|
||||
private readonly ILiveRoomPollingSignal _liveRoomPollingSignal;
|
||||
private readonly ILogger<FfmpegService> _logger;
|
||||
|
||||
public FfmpegService(
|
||||
IServiceScopeFactory serviceScopeFactory,
|
||||
IStorageGuardService storageGuardService,
|
||||
ILiveRoomPollingSignal liveRoomPollingSignal,
|
||||
ILogger<FfmpegService> logger)
|
||||
{
|
||||
_serviceScopeFactory = serviceScopeFactory;
|
||||
_storageGuardService = storageGuardService;
|
||||
_liveRoomPollingSignal = liveRoomPollingSignal;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
namespace LiveRecorder.Infrastructure.Services;
|
||||
|
||||
public interface ILiveRoomPollingSignal
|
||||
{
|
||||
void RequestImmediatePoll(Guid liveRoomId, TimeSpan? delay = null);
|
||||
}
|
||||
@@ -19,8 +19,9 @@ using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace LiveRecorder.Infrastructure.Services;
|
||||
|
||||
public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveRoomPollingSignal
|
||||
{
|
||||
private static readonly TimeSpan TransientRetryDelay = TimeSpan.FromSeconds(10);
|
||||
private static readonly TimeSpan OfflineGracefulStopTimeout = TimeSpan.FromSeconds(20);
|
||||
private static readonly TimeSpan OfflineForcedStopTimeout = TimeSpan.FromSeconds(8);
|
||||
private static readonly TimeSpan ExceptionEmailCooldown = TimeSpan.FromHours(6);
|
||||
@@ -32,6 +33,7 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
private readonly ILogger<LiveRoomPollingBackgroundService> _logger;
|
||||
private readonly ConcurrentDictionary<string, DateTimeOffset> _exceptionEmailSentAt = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly ConcurrentDictionary<Guid, DateTimeOffset> _nextPollDueAt = new();
|
||||
private readonly SemaphoreSlim _wakeSignal = new(0);
|
||||
|
||||
public LiveRoomPollingBackgroundService(
|
||||
IServiceScopeFactory serviceScopeFactory,
|
||||
@@ -41,6 +43,23 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public void RequestImmediatePoll(Guid liveRoomId, TimeSpan? delay = null)
|
||||
{
|
||||
var requestedDueAt = DateTimeOffset.UtcNow + (delay ?? MinimumIdleDelay);
|
||||
_nextPollDueAt.AddOrUpdate(
|
||||
liveRoomId,
|
||||
requestedDueAt,
|
||||
(_, current) => requestedDueAt < current ? requestedDueAt : current);
|
||||
|
||||
try
|
||||
{
|
||||
_wakeSignal.Release();
|
||||
}
|
||||
catch (SemaphoreFullException)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
@@ -61,7 +80,7 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
|
||||
if (!settings.EnableBackgroundPolling)
|
||||
{
|
||||
await DelayAsync(TimeSpan.FromSeconds(Math.Clamp(settings.PollingIntervalSeconds, 10, 3600)), stoppingToken);
|
||||
await WaitForNextRunAsync(TimeSpan.FromSeconds(Math.Clamp(settings.PollingIntervalSeconds, 10, 3600)), stoppingToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -160,12 +179,23 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
}
|
||||
}
|
||||
|
||||
await DelayAsync(delay, stoppingToken);
|
||||
await WaitForNextRunAsync(delay, stoppingToken);
|
||||
}
|
||||
}
|
||||
|
||||
private static Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken) =>
|
||||
Task.Delay(ClampDelay(delay), cancellationToken);
|
||||
private async Task WaitForNextRunAsync(TimeSpan delay, CancellationToken cancellationToken)
|
||||
{
|
||||
while (_wakeSignal.CurrentCount > 0)
|
||||
{
|
||||
await _wakeSignal.WaitAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
var effectiveDelay = ClampDelay(delay);
|
||||
var delayTask = Task.Delay(effectiveDelay, cancellationToken);
|
||||
var wakeTask = _wakeSignal.WaitAsync(cancellationToken);
|
||||
var completedTask = await Task.WhenAny(delayTask, wakeTask);
|
||||
await completedTask;
|
||||
}
|
||||
|
||||
private static TimeSpan ClampDelay(TimeSpan delay)
|
||||
{
|
||||
@@ -254,6 +284,8 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
return;
|
||||
}
|
||||
|
||||
var nextDueAt = DateTimeOffset.UtcNow.AddSeconds(Math.Clamp(intervalSeconds, 10, 3600));
|
||||
|
||||
try
|
||||
{
|
||||
var adapter = adapterFactory.GetByPlatform(liveRoom.Platform);
|
||||
@@ -380,6 +412,21 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
_logger.LogWarning(ex, "Polling room {RoomId} failed", liveRoom.RoomId);
|
||||
|
||||
var isTransient = IsTransientPollingException(ex, cancellationToken);
|
||||
if (isTransient)
|
||||
{
|
||||
nextDueAt = DateTimeOffset.UtcNow.Add(TransientRetryDelay);
|
||||
}
|
||||
|
||||
await UpdateAutoStartDecisionAsync(
|
||||
dbContext,
|
||||
liveRoom,
|
||||
isTransient ? AutoStartDecisionCodes.PollFailedTransient : AutoStartDecisionCodes.PollFailed,
|
||||
isTransient
|
||||
? "Auto-start is pending because live status polling failed temporarily."
|
||||
: "Auto-start failed because live status polling failed.",
|
||||
BuildPollingFailureDetail(ex),
|
||||
cancellationToken);
|
||||
|
||||
await logService.WriteAsync(
|
||||
isTransient ? SystemLogLevel.Warning : SystemLogLevel.Error,
|
||||
"Scheduler",
|
||||
@@ -408,7 +455,7 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
}
|
||||
finally
|
||||
{
|
||||
_nextPollDueAt[liveRoomId] = DateTimeOffset.UtcNow.AddSeconds(Math.Clamp(intervalSeconds, 10, 3600));
|
||||
_nextPollDueAt[liveRoomId] = nextDueAt;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -605,6 +652,12 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
IsTransientPollingException(exception.InnerException, cancellationToken);
|
||||
}
|
||||
|
||||
private static string BuildPollingFailureDetail(Exception exception)
|
||||
{
|
||||
var root = exception.GetBaseException();
|
||||
return Truncate($"{root.GetType().Name}: {root.Message}", 2048) ?? exception.GetType().Name;
|
||||
}
|
||||
|
||||
private bool ShouldSendExceptionEmail(string key)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
@@ -81,6 +81,12 @@ public sealed class RecordSessionsController : ControllerBase
|
||||
CancellationToken cancellationToken) =>
|
||||
Ok(await _recordSessionService.DeleteAsync(request, cancellationToken));
|
||||
|
||||
[HttpPost("delete-missing-files")]
|
||||
public async Task<ActionResult<DeleteCompletedRecordTasksResultDto>> DeleteMissingFiles(
|
||||
[FromBody] DeleteMissingFileRecordSessionsRequest request,
|
||||
CancellationToken cancellationToken) =>
|
||||
Ok(await _recordSessionService.DeleteMissingFilesAsync(request, cancellationToken));
|
||||
|
||||
[HttpPost("{id:guid}/upload")]
|
||||
public async Task<ActionResult<RecordArtifactUploadBatchResultDto>> Upload(Guid id, CancellationToken cancellationToken) =>
|
||||
Ok(await _recordUploadService.UploadSessionAsync(id, cancellationToken));
|
||||
|
||||
@@ -167,7 +167,9 @@ builder.Services.AddScoped<ILiveDanmakuAdapterFactory, LiveDanmakuAdapterFactory
|
||||
builder.Services.AddSingleton<IFfmpegService, FfmpegService>();
|
||||
builder.Services.AddSingleton<IStorageGuardService, StorageGuardService>();
|
||||
builder.Services.AddScoped<IRecordMediaService, RecordMediaService>();
|
||||
builder.Services.AddHostedService<LiveRoomPollingBackgroundService>();
|
||||
builder.Services.AddSingleton<LiveRoomPollingBackgroundService>();
|
||||
builder.Services.AddSingleton<ILiveRoomPollingSignal>(provider => provider.GetRequiredService<LiveRoomPollingBackgroundService>());
|
||||
builder.Services.AddHostedService(provider => provider.GetRequiredService<LiveRoomPollingBackgroundService>());
|
||||
builder.Services.AddHostedService<RetentionCleanupBackgroundService>();
|
||||
|
||||
var app = builder.Build();
|
||||
|
||||
Reference in New Issue
Block a user