nanxun/live_recorder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add storage and script failure notifications

Browse Source
This commit is contained in:
nanxun
2026-05-31 19:35:06 +08:00
parent fd5be2cc8e
commit b82110461a
7 changed files with 381 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/LiveRecorder.Application/Models/Settings/SettingsModels.cs
+8
View File
@@ -177,6 +177,10 @@ public sealed class SystemSettingsDto
public int EventScriptTimeoutSeconds { get; set; } = 60;
public int EventScriptRetryAttempts { get; set; } = 3;
public int EventScriptRetryDelaySeconds { get; set; } = 10;
public bool EnableRetentionCleanup { get; set; } = false;
public int RetentionDays { get; set; } = 30;
@@ -446,6 +450,10 @@ public sealed class UpdateSystemSettingsRequest
public int EventScriptTimeoutSeconds { get; set; } = 60;
public int EventScriptRetryAttempts { get; set; } = 3;
public int EventScriptRetryDelaySeconds { get; set; } = 10;
public bool EnableRetentionCleanup { get; set; } = false;
public int RetentionDays { get; set; } = 30;
src/LiveRecorder.Application/Services/SystemSettingsService.cs
+6
View File
@@ -71,6 +71,8 @@ public sealed class SystemSettingsService : ISystemSettingsService
private const string SegmentCompletedScriptPathKey = "event_scripts.segment_completed.path";
private const string SegmentCompletedScriptContentKey = "event_scripts.segment_completed.content";
private const string EventScriptTimeoutSecondsKey = "event_scripts.timeout_seconds";
private const string EventScriptRetryAttemptsKey = "event_scripts.retry_attempts";
private const string EventScriptRetryDelaySecondsKey = "event_scripts.retry_delay_seconds";
private const string EnableRetentionCleanupKey = "retention.cleanup.enabled";
private const string RetentionDaysKey = "retention.cleanup.days";
private const string RetentionDeleteFilesKey = "retention.cleanup.delete_files";
@@ -202,6 +204,8 @@ public sealed class SystemSettingsService : ISystemSettingsService
SegmentCompletedScriptPath = GetValue(lookup, SegmentCompletedScriptPathKey, string.Empty),
SegmentCompletedScriptContent = GetValue(lookup, SegmentCompletedScriptContentKey, string.Empty),
EventScriptTimeoutSeconds = GetIntValue(lookup, EventScriptTimeoutSecondsKey, 60, 1, 3600),
EventScriptRetryAttempts = GetIntValue(lookup, EventScriptRetryAttemptsKey, 3, 0, 20),
EventScriptRetryDelaySeconds = GetIntValue(lookup, EventScriptRetryDelaySecondsKey, 10, 0, 3600),
EnableRetentionCleanup = bool.TryParse(GetValue(lookup, EnableRetentionCleanupKey, "false"), out var enableRetentionCleanup) && enableRetentionCleanup,
RetentionDays = GetIntValue(lookup, RetentionDaysKey, 30, 1, 3650),
RetentionDeleteFiles = bool.TryParse(GetValue(lookup, RetentionDeleteFilesKey, "false"), out var retentionDeleteFiles) && retentionDeleteFiles,
@@ -352,6 +356,8 @@ public sealed class SystemSettingsService : ISystemSettingsService
await UpsertAsync(SegmentCompletedScriptPathKey, request.SegmentCompletedScriptPath.Trim(), now, cancellationToken);
await UpsertAsync(SegmentCompletedScriptContentKey, request.SegmentCompletedScriptContent, now, cancellationToken);
await UpsertAsync(EventScriptTimeoutSecondsKey, Math.Clamp(request.EventScriptTimeoutSeconds, 1, 3600).ToString(), now, cancellationToken);
await UpsertAsync(EventScriptRetryAttemptsKey, Math.Clamp(request.EventScriptRetryAttempts, 0, 20).ToString(), now, cancellationToken);
await UpsertAsync(EventScriptRetryDelaySecondsKey, Math.Clamp(request.EventScriptRetryDelaySeconds, 0, 3600).ToString(), now, cancellationToken);
await UpsertAsync(EnableRetentionCleanupKey, request.EnableRetentionCleanup.ToString(), now, cancellationToken);
await UpsertAsync(RetentionDaysKey, Math.Clamp(request.RetentionDays, 1, 3650).ToString(), now, cancellationToken);
await UpsertAsync(RetentionDeleteFilesKey, request.RetentionDeleteFiles.ToString(), now, cancellationToken);
src/LiveRecorder.Infrastructure/Services/EventScriptService.cs
+226 -7
View File
@@ -1,5 +1,7 @@
using System.Diagnostics;
using System.Text;
using LiveRecorder.Application.Abstractions.Logging;
using LiveRecorder.Application.Abstractions.Notifications;
using LiveRecorder.Application.Abstractions.Scripting;
using LiveRecorder.Application.Abstractions.Settings;
using LiveRecorder.Application.Common;
@@ -18,15 +20,21 @@ public sealed class EventScriptService : IEventScriptService
private readonly ISystemSettingsService _settingsService;
private readonly ISystemLogService _systemLogService;
private readonly IEmailNotificationService _emailNotificationService;
private readonly IWebhookNotificationService _webhookNotificationService;
private readonly ILogger<EventScriptService> _logger;
public EventScriptService(
ISystemSettingsService settingsService,
ISystemLogService systemLogService,
IEmailNotificationService emailNotificationService,
IWebhookNotificationService webhookNotificationService,
ILogger<EventScriptService> logger)
{
_settingsService = settingsService;
_systemLogService = systemLogService;
_emailNotificationService = emailNotificationService;
_webhookNotificationService = webhookNotificationService;
_logger = logger;
}
@@ -44,8 +52,12 @@ public sealed class EventScriptService : IEventScriptService
settings.LiveStartedScriptPath,
settings.LiveStartedScriptContent,
settings.EventScriptTimeoutSeconds,
settings.EventScriptRetryAttempts,
settings.EventScriptRetryDelaySeconds,
"live_started",
environment,
liveRoom,
recordTask: null,
"Script",
liveRoom.Id,
recordSessionId: null,
@@ -67,8 +79,12 @@ public sealed class EventScriptService : IEventScriptService
settings.LiveEndedScriptPath,
settings.LiveEndedScriptContent,
settings.EventScriptTimeoutSeconds,
settings.EventScriptRetryAttempts,
settings.EventScriptRetryDelaySeconds,
"live_ended",
environment,
liveRoom,
recordTask: null,
"Script",
liveRoom.Id,
recordSessionId: null,
@@ -105,8 +121,12 @@ public sealed class EventScriptService : IEventScriptService
settings.SegmentCompletedScriptPath,
settings.SegmentCompletedScriptContent,
settings.EventScriptTimeoutSeconds,
settings.EventScriptRetryAttempts,
settings.EventScriptRetryDelaySeconds,
"segment_completed",
environment,
liveRoom,
recordTask,
"Script",
liveRoom?.Id ?? recordSession.LiveRoomId,
recordSession.Id,
@@ -176,8 +196,12 @@ public sealed class EventScriptService : IEventScriptService
string scriptPath,
string scriptContent,
int timeoutSeconds,
int retryAttempts,
int retryDelaySeconds,
string eventName,
IReadOnlyDictionary<string, string> environment,
LiveRoom? liveRoom,
RecordTask? recordTask,
string logCategory,
Guid? liveRoomId,
Guid? recordSessionId,
@@ -189,6 +213,43 @@ public sealed class EventScriptService : IEventScriptService
return null;
}
var outcome = await ExecuteWithRetryAsync(
scriptMode,
scriptPath,
scriptContent,
timeoutSeconds,
retryAttempts,
retryDelaySeconds,
eventName,
environment,
liveRoom,
recordTask,
logCategory,
liveRoomId,
recordSessionId,
recordTaskId,
cancellationToken);
return MapOutcome(outcome);
}
private async Task<ScriptExecutionOutcome> ExecuteWithRetryAsync(
string scriptMode,
string scriptPath,
string scriptContent,
int timeoutSeconds,
int retryAttempts,
int retryDelaySeconds,
string eventName,
IReadOnlyDictionary<string, string> environment,
LiveRoom? liveRoom,
RecordTask? recordTask,
string logCategory,
Guid? liveRoomId,
Guid? recordSessionId,
Guid? recordTaskId,
CancellationToken cancellationToken)
{
var outcome = await ExecuteAsync(
scriptMode,
scriptPath,
@@ -202,7 +263,53 @@ public sealed class EventScriptService : IEventScriptService
recordTaskId,
cancellationToken);
return MapOutcome(outcome);
if (outcome.Success || !IsRetryableFailure(outcome))
{
return outcome;
}
var maxRetryAttempts = Math.Clamp(retryAttempts, 0, 20);
var totalAttempts = 1;
for (var retryIndex = 1; retryIndex <= maxRetryAttempts; retryIndex++)
{
await _systemLogService.WriteAsync(
SystemLogLevel.Warning,
logCategory,
$"Event script retry {retryIndex} of {maxRetryAttempts} scheduled for {eventName}.",
BuildRetryAttemptDetail(eventName, retryIndex + 1, maxRetryAttempts + 1, retryDelaySeconds, outcome),
liveRoomId,
recordSessionId,
recordTaskId,
cancellationToken);
if (retryDelaySeconds > 0)
{
await Task.Delay(TimeSpan.FromSeconds(Math.Clamp(retryDelaySeconds, 0, 3600)), cancellationToken);
}
outcome = await ExecuteAsync(
scriptMode,
scriptPath,
scriptContent,
timeoutSeconds,
eventName,
environment,
logCategory,
liveRoomId,
recordSessionId,
recordTaskId,
cancellationToken);
totalAttempts++;
if (outcome.Success || !IsRetryableFailure(outcome))
{
return outcome;
}
}
await NotifyRetryExhaustedAsync(eventName, totalAttempts, outcome, liveRoom, recordTask, cancellationToken);
return outcome;
}
private async Task<ScriptExecutionOutcome> ExecuteAsync(
@@ -225,7 +332,9 @@ public sealed class EventScriptService : IEventScriptService
false,
$"Event script was not configured for {eventName}.",
null,
null);
null,
null,
ScriptFailureKind.MissingConfiguration);
await WriteOutcomeLogAsync(
missingConfiguration,
@@ -244,7 +353,9 @@ public sealed class EventScriptService : IEventScriptService
false,
$"Event script was not found for {eventName}.",
execution.Detail,
null);
null,
execution.Detail,
ScriptFailureKind.MissingScript);
await WriteOutcomeLogAsync(
missingScript,
@@ -295,7 +406,9 @@ public sealed class EventScriptService : IEventScriptService
? $"Event script completed for {eventName}."
: $"Event script exited with code {process.ExitCode} for {eventName}.",
execution.Detail,
null);
null,
execution.Detail,
process.ExitCode == 0 ? ScriptFailureKind.None : ScriptFailureKind.ExitCode);
outcomeLevel = process.ExitCode == 0 ? SystemLogLevel.Info : SystemLogLevel.Warning;
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
@@ -305,7 +418,9 @@ public sealed class EventScriptService : IEventScriptService
false,
$"Event script timed out for {eventName}.",
execution.Detail,
null);
null,
execution.Detail,
ScriptFailureKind.Timeout);
outcomeLevel = SystemLogLevel.Warning;
}
catch (Exception ex)
@@ -315,7 +430,9 @@ public sealed class EventScriptService : IEventScriptService
false,
$"Event script failed for {eventName}.",
ex.ToString(),
null);
null,
execution.Detail,
ScriptFailureKind.Exception);
outcomeLevel = SystemLogLevel.Warning;
}
finally
@@ -344,6 +461,31 @@ public sealed class EventScriptService : IEventScriptService
return outcome;
}
private async Task NotifyRetryExhaustedAsync(
string eventName,
int totalAttempts,
ScriptExecutionOutcome outcome,
LiveRoom? liveRoom,
RecordTask? recordTask,
CancellationToken cancellationToken)
{
var detail = BuildRetryFailureNotificationDetail(eventName, totalAttempts, outcome);
await _emailNotificationService.SendExceptionAsync(
"EventScript",
"Event script failed after retries.",
detail,
liveRoom,
recordTask,
cancellationToken);
await _webhookNotificationService.SendExceptionAsync(
"EventScript",
"Event script failed after retries.",
detail,
liveRoom,
recordTask,
cancellationToken);
}
private static EventScriptExecution? CreateExecution(string scriptMode, string scriptPath, string scriptContent)
{
if (string.Equals(scriptMode, EventScriptSourceModes.Inline, StringComparison.OrdinalIgnoreCase))
@@ -582,6 +724,71 @@ public sealed class EventScriptService : IEventScriptService
cancellationToken);
}
private static string BuildRetryAttemptDetail(
string eventName,
int nextAttempt,
int totalAttempts,
int retryDelaySeconds,
ScriptExecutionOutcome outcome)
{
var builder = new StringBuilder();
builder.AppendLine($"Event: {eventName}");
builder.AppendLine($"Next attempt: {nextAttempt}/{totalAttempts}");
builder.AppendLine($"Retry delay: {Math.Clamp(retryDelaySeconds, 0, 3600)} second(s)");
builder.AppendLine($"Last result: {outcome.Message}");
if (!string.IsNullOrWhiteSpace(outcome.ExecutionTarget))
{
builder.AppendLine($"Script: {outcome.ExecutionTarget}");
}
if (!string.IsNullOrWhiteSpace(outcome.Detail) &&
!string.Equals(outcome.Detail, outcome.ExecutionTarget, StringComparison.Ordinal))
{
builder.AppendLine();
builder.AppendLine("Detail:");
builder.AppendLine(outcome.Detail);
}
return TruncateSystemLogDetail(builder.ToString().Trim());
}
private static string BuildRetryFailureNotificationDetail(
string eventName,
int totalAttempts,
ScriptExecutionOutcome outcome)
{
var builder = new StringBuilder();
builder.AppendLine($"Event: {eventName}");
builder.AppendLine($"Attempts: {totalAttempts}");
builder.AppendLine($"Last result: {outcome.Message}");
if (!string.IsNullOrWhiteSpace(outcome.ExecutionTarget))
{
builder.AppendLine($"Script: {outcome.ExecutionTarget}");
}
if (!string.IsNullOrWhiteSpace(outcome.Detail) &&
!string.Equals(outcome.Detail, outcome.ExecutionTarget, StringComparison.Ordinal))
{
builder.AppendLine();
builder.AppendLine("Detail:");
builder.AppendLine(outcome.Detail);
}
if (!string.IsNullOrWhiteSpace(outcome.CustomLogOutput))
{
builder.AppendLine();
builder.AppendLine("Custom log output:");
builder.AppendLine(outcome.CustomLogOutput);
}
return TruncateSystemLogDetail(builder.ToString().Trim());
}
private static bool IsRetryableFailure(ScriptExecutionOutcome outcome) =>
outcome.FailureKind is ScriptFailureKind.ExitCode or ScriptFailureKind.Timeout or ScriptFailureKind.Exception;
private static string CreateScriptLogPath()
{
return Path.Combine(
@@ -628,11 +835,23 @@ public sealed class EventScriptService : IEventScriptService
}
}
private enum ScriptFailureKind
{
None,
MissingConfiguration,
MissingScript,
ExitCode,
Timeout,
Exception
}
private sealed record EventScriptExecution(ProcessStartInfo? StartInfo, string Detail, bool IsMissing = false);
private sealed record ScriptExecutionOutcome(
bool Success,
string Message,
string? Detail,
string? CustomLogOutput);
string? CustomLogOutput,
string? ExecutionTarget,
ScriptFailureKind FailureKind);
}
src/LiveRecorder.Infrastructure/Services/LiveRoomPollingBackgroundService.cs
+65
View File
@@ -1,4 +1,5 @@
using System.Collections.Concurrent;
using System.Text;
using LiveRecorder.Application.Common;
using LiveRecorder.Application.Abstractions.Logging;
using LiveRecorder.Application.Abstractions.Notifications;
@@ -360,6 +361,9 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
dbContext,
ffmpegService,
logService,
emailNotificationService,
webhookNotificationService,
liveRoom,
liveRoom.Id,
pauseCheck.Message,
cancellationToken);
@@ -589,6 +593,9 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
LiveRecorderDbContext dbContext,
IFfmpegService ffmpegService,
ISystemLogService logService,
IEmailNotificationService emailNotificationService,
IWebhookNotificationService webhookNotificationService,
Domain.Entities.LiveRoom liveRoom,
Guid liveRoomId,
string detail,
CancellationToken cancellationToken)
@@ -600,8 +607,13 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
item.Status == RecordSessionStatus.Stopping))
.ToListAsync(cancellationToken);
var affectedSessionCount = 0;
var forcedStopCount = 0;
foreach (var activeSession in activeSessions.OrderBy(static item => item.CreatedAt))
{
affectedSessionCount++;
await logService.WriteAsync(
SystemLogLevel.Warning,
"Storage",
@@ -628,12 +640,32 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
liveRoomId,
activeSession.Id,
cancellationToken: cancellationToken);
forcedStopCount++;
await ffmpegService.KillAndWaitAsync(activeSession.Id, OfflineForcedStopTimeout, cancellationToken);
}
}
await ffmpegService.TryReconcileInactiveSessionAsync(activeSession.Id, cancellationToken);
}
if (affectedSessionCount <= 0)
{
return;
}
var notificationDetail = BuildLowStorageNotificationDetail(liveRoom, affectedSessionCount, forcedStopCount, detail);
await emailNotificationService.SendExceptionAsync(
"StorageGuard",
"Low storage paused active recording sessions.",
notificationDetail,
liveRoom,
cancellationToken: cancellationToken);
await webhookNotificationService.SendExceptionAsync(
"StorageGuard",
"Low storage paused active recording sessions.",
notificationDetail,
liveRoom,
cancellationToken: cancellationToken);
}
private static async Task<IReadOnlyList<Guid>> ReconcileStaleActiveSessionsAsync(
@@ -699,6 +731,39 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
IsTransientPollingException(exception.InnerException, cancellationToken);
}
private static string BuildLowStorageNotificationDetail(
Domain.Entities.LiveRoom liveRoom,
int affectedSessionCount,
int forcedStopCount,
string storageDetail)
{
var builder = new StringBuilder();
builder.AppendLine($"Platform: {liveRoom.Platform}");
builder.AppendLine($"Room ID: {liveRoom.RoomId}");
if (!string.IsNullOrWhiteSpace(liveRoom.AnchorName))
{
builder.AppendLine($"Anchor: {liveRoom.AnchorName}");
}
if (!string.IsNullOrWhiteSpace(liveRoom.Title))
{
builder.AppendLine($"Title: {liveRoom.Title}");
}
builder.AppendLine($"Affected sessions: {affectedSessionCount}");
builder.AppendLine($"Forced stop attempts: {forcedStopCount}");
if (!string.IsNullOrWhiteSpace(storageDetail))
{
builder.AppendLine();
builder.AppendLine("Storage detail:");
builder.AppendLine(storageDetail);
}
return builder.ToString().Trim();
}
private static string BuildPollingFailureDetail(Exception exception)
{
var root = exception.GetBaseException();