feat: add storage and script failure notifications

This commit is contained in:
2026-05-31 19:35:06 +08:00
parent fd5be2cc8e
commit b82110461a
7 changed files with 381 additions and 11 deletions
+52
View File
@@ -61,9 +61,17 @@
--el-border-color-light: var(--border-subtle); --el-border-color-light: var(--border-subtle);
--el-border-radius-base: var(--radius-sm); --el-border-radius-base: var(--radius-sm);
--el-bg-color: transparent; --el-bg-color: transparent;
--el-bg-color-page: var(--bg-base);
--el-bg-color-overlay: var(--surface);
--el-fill-color: var(--surface-muted);
--el-fill-color-blank: var(--surface); --el-fill-color-blank: var(--surface);
--el-fill-color-light: var(--surface-muted); --el-fill-color-light: var(--surface-muted);
--el-fill-color-lighter: var(--surface-strong); --el-fill-color-lighter: var(--surface-strong);
--el-fill-color-dark: var(--surface-raised);
--el-fill-color-darker: var(--surface-strong);
--el-disabled-bg-color: var(--surface-muted);
--el-disabled-text-color: var(--text-soft);
--el-text-color-placeholder: var(--text-soft);
--el-mask-color: rgba(15, 23, 42, 0.54); --el-mask-color: rgba(15, 23, 42, 0.54);
} }
@@ -123,9 +131,17 @@ html[data-theme="dark"] {
--el-text-color-secondary: var(--text-muted); --el-text-color-secondary: var(--text-muted);
--el-border-color: var(--border-base); --el-border-color: var(--border-base);
--el-border-color-light: var(--border-subtle); --el-border-color-light: var(--border-subtle);
--el-bg-color-page: var(--bg-base);
--el-bg-color-overlay: var(--surface);
--el-fill-color: var(--surface-muted);
--el-fill-color-blank: var(--surface); --el-fill-color-blank: var(--surface);
--el-fill-color-light: var(--surface-muted); --el-fill-color-light: var(--surface-muted);
--el-fill-color-lighter: var(--surface-strong); --el-fill-color-lighter: var(--surface-strong);
--el-fill-color-dark: var(--surface-raised);
--el-fill-color-darker: var(--surface-strong);
--el-disabled-bg-color: var(--surface-muted);
--el-disabled-text-color: var(--text-soft);
--el-text-color-placeholder: var(--text-soft);
--el-mask-color: rgba(3, 7, 14, 0.72); --el-mask-color: rgba(3, 7, 14, 0.72);
} }
@@ -255,6 +271,7 @@ select:focus-visible {
.surface-card { .surface-card {
position: relative; position: relative;
overflow: hidden; overflow: hidden;
--el-card-bg-color: transparent;
border-radius: var(--radius-md); border-radius: var(--radius-md);
border: 1px solid var(--border-subtle); border: 1px solid var(--border-subtle);
background: background:
@@ -573,9 +590,33 @@ html[data-theme="dark"] .stat-card {
.el-card { .el-card {
--el-card-border-color: transparent; --el-card-border-color: transparent;
--el-card-bg-color: transparent;
background: transparent; background: transparent;
} }
.el-tabs--border-card {
border-color: var(--border-subtle);
background: var(--surface);
}
.el-tabs--border-card > .el-tabs__content {
background: transparent;
color: var(--text-primary);
}
.el-tabs--border-card > .el-tabs__header {
background: transparent;
}
.el-tabs--border-card > .el-tabs__header .el-tabs__item {
color: var(--text-secondary);
}
.el-tabs--border-card > .el-tabs__header .el-tabs__item.is-active {
color: var(--text-primary);
background: var(--surface-raised);
}
.el-button { .el-button {
min-height: var(--control-height); min-height: var(--control-height);
padding: 0 15px; padding: 0 15px;
@@ -802,6 +843,17 @@ html[data-theme="dark"] .stat-card {
box-shadow: var(--shadow-float); box-shadow: var(--shadow-float);
} }
.el-message-box,
.el-popover.el-popper,
.el-select__popper.el-popper,
.el-picker__popper.el-popper,
.el-dropdown__popper.el-popper .el-dropdown-menu {
border-color: var(--border-subtle);
background: linear-gradient(180deg, var(--surface-raised), var(--surface));
color: var(--text-primary);
box-shadow: var(--shadow-float);
}
.el-dialog__header { .el-dialog__header {
margin: 0; margin: 0;
padding: 22px 24px 10px; padding: 22px 24px 10px;
+2
View File
@@ -458,6 +458,8 @@ export interface SystemSettings {
segmentCompletedScriptPath: string; segmentCompletedScriptPath: string;
segmentCompletedScriptContent: string; segmentCompletedScriptContent: string;
eventScriptTimeoutSeconds: number; eventScriptTimeoutSeconds: number;
eventScriptRetryAttempts: number;
eventScriptRetryDelaySeconds: number;
enableEmailNotification: boolean; enableEmailNotification: boolean;
emailSmtpHost: string; emailSmtpHost: string;
emailSmtpPort: number; emailSmtpPort: number;
+22 -4
View File
@@ -168,6 +168,8 @@ const form = reactive<SettingsFormModel>({
segmentCompletedScriptPath: "", segmentCompletedScriptPath: "",
segmentCompletedScriptContent: "", segmentCompletedScriptContent: "",
eventScriptTimeoutSeconds: 60, eventScriptTimeoutSeconds: 60,
eventScriptRetryAttempts: 3,
eventScriptRetryDelaySeconds: 10,
enableEmailNotification: false, enableEmailNotification: false,
emailSmtpHost: "", emailSmtpHost: "",
emailSmtpPort: 587, emailSmtpPort: 587,
@@ -1480,16 +1482,26 @@ watch(
<el-form label-position="top"> <el-form label-position="top">
<el-row :gutter="16"> <el-row :gutter="16">
<el-col :span="8"> <el-col :span="6">
<el-form-item label="启用事件脚本"> <el-form-item label="启用事件脚本">
<el-switch v-model="form.enableEventScripts" /> <el-switch v-model="form.enableEventScripts" />
</el-form-item> </el-form-item>
</el-col> </el-col>
<el-col :span="8"> <el-col :span="6">
<el-form-item label="脚本超时(秒)"> <el-form-item label="脚本超时(秒)">
<el-input-number v-model="form.eventScriptTimeoutSeconds" :min="1" :max="3600" /> <el-input-number v-model="form.eventScriptTimeoutSeconds" :min="1" :max="3600" />
</el-form-item> </el-form-item>
</el-col> </el-col>
<el-col :span="5">
<el-form-item label="重试次数">
<el-input-number v-model="form.eventScriptRetryAttempts" :min="0" :max="20" />
</el-form-item>
</el-col>
<el-col :span="6">
<el-form-item label="重试间隔(秒)">
<el-input-number v-model="form.eventScriptRetryDelaySeconds" :min="0" :max="3600" />
</el-form-item>
</el-col>
</el-row> </el-row>
</el-form> </el-form>
@@ -1676,6 +1688,10 @@ watch(
The official Docker image includes <code>curl</code> and <code>jq</code> by default. If you run on a host machine or a custom image, rely on the commands available in that environment. The official Docker image includes <code>curl</code> and <code>jq</code> by default. If you run on a host machine or a custom image, rely on the commands available in that environment.
</div> </div>
<div class="event-script-help__intro">
Set <code>Retry attempts</code> to <code>0</code> to disable automatic retries. Retry exhaustion failures are sent through the existing exception notification channel.
</div>
<div class="event-script-example"> <div class="event-script-example">
<div class="event-script-example__label">Environment variable examples</div> <div class="event-script-example__label">Environment variable examples</div>
<div class="event-script-example__grid"> <div class="event-script-example__grid">
@@ -1699,7 +1715,8 @@ watch(
<el-card class="surface-card settings-card settings-grid__full" shadow="never"> <el-card class="surface-card settings-card settings-grid__full" shadow="never">
<h3 class="section-title">Webhook 閫氱煡</h3> <h3 class="section-title">Webhook 閫氱煡</h3>
<p class="section-subtitle">Send fixed JSON POST payloads with custom headers. Currently only live-started and exception events are sent.</p> <h3 class="section-title">Webhook 通知</h3>
<p class="section-subtitle">Send fixed JSON POST payloads with custom headers. Exception notifications also cover low-storage stop events and event-script failures after retries are exhausted.</p>
<el-form label-position="top"> <el-form label-position="top">
<el-row :gutter="16"> <el-row :gutter="16">
@@ -1772,7 +1789,8 @@ watch(
<el-card class="surface-card settings-card settings-grid__full" shadow="never"> <el-card class="surface-card settings-card settings-grid__full" shadow="never">
<h3 class="section-title">欢閫氱煡</h3> <h3 class="section-title">欢閫氱煡</h3>
<p class="section-subtitle">Configure SMTP plus HTML templates for live-started and exception alerts. Tests use the current unsaved form values directly.</p> <h3 class="section-title">邮件通知</h3>
<p class="section-subtitle">Configure SMTP plus HTML templates for live-started and exception alerts. Exception notifications also cover low-storage stop events and event-script failures after retries are exhausted.</p>
<el-form label-position="top"> <el-form label-position="top">
<el-row :gutter="16"> <el-row :gutter="16">
@@ -177,6 +177,10 @@ public sealed class SystemSettingsDto
public int EventScriptTimeoutSeconds { get; set; } = 60; public int EventScriptTimeoutSeconds { get; set; } = 60;
public int EventScriptRetryAttempts { get; set; } = 3;
public int EventScriptRetryDelaySeconds { get; set; } = 10;
public bool EnableRetentionCleanup { get; set; } = false; public bool EnableRetentionCleanup { get; set; } = false;
public int RetentionDays { get; set; } = 30; public int RetentionDays { get; set; } = 30;
@@ -446,6 +450,10 @@ public sealed class UpdateSystemSettingsRequest
public int EventScriptTimeoutSeconds { get; set; } = 60; public int EventScriptTimeoutSeconds { get; set; } = 60;
public int EventScriptRetryAttempts { get; set; } = 3;
public int EventScriptRetryDelaySeconds { get; set; } = 10;
public bool EnableRetentionCleanup { get; set; } = false; public bool EnableRetentionCleanup { get; set; } = false;
public int RetentionDays { get; set; } = 30; public int RetentionDays { get; set; } = 30;
@@ -71,6 +71,8 @@ public sealed class SystemSettingsService : ISystemSettingsService
private const string SegmentCompletedScriptPathKey = "event_scripts.segment_completed.path"; private const string SegmentCompletedScriptPathKey = "event_scripts.segment_completed.path";
private const string SegmentCompletedScriptContentKey = "event_scripts.segment_completed.content"; private const string SegmentCompletedScriptContentKey = "event_scripts.segment_completed.content";
private const string EventScriptTimeoutSecondsKey = "event_scripts.timeout_seconds"; private const string EventScriptTimeoutSecondsKey = "event_scripts.timeout_seconds";
private const string EventScriptRetryAttemptsKey = "event_scripts.retry_attempts";
private const string EventScriptRetryDelaySecondsKey = "event_scripts.retry_delay_seconds";
private const string EnableRetentionCleanupKey = "retention.cleanup.enabled"; private const string EnableRetentionCleanupKey = "retention.cleanup.enabled";
private const string RetentionDaysKey = "retention.cleanup.days"; private const string RetentionDaysKey = "retention.cleanup.days";
private const string RetentionDeleteFilesKey = "retention.cleanup.delete_files"; private const string RetentionDeleteFilesKey = "retention.cleanup.delete_files";
@@ -202,6 +204,8 @@ public sealed class SystemSettingsService : ISystemSettingsService
SegmentCompletedScriptPath = GetValue(lookup, SegmentCompletedScriptPathKey, string.Empty), SegmentCompletedScriptPath = GetValue(lookup, SegmentCompletedScriptPathKey, string.Empty),
SegmentCompletedScriptContent = GetValue(lookup, SegmentCompletedScriptContentKey, string.Empty), SegmentCompletedScriptContent = GetValue(lookup, SegmentCompletedScriptContentKey, string.Empty),
EventScriptTimeoutSeconds = GetIntValue(lookup, EventScriptTimeoutSecondsKey, 60, 1, 3600), EventScriptTimeoutSeconds = GetIntValue(lookup, EventScriptTimeoutSecondsKey, 60, 1, 3600),
EventScriptRetryAttempts = GetIntValue(lookup, EventScriptRetryAttemptsKey, 3, 0, 20),
EventScriptRetryDelaySeconds = GetIntValue(lookup, EventScriptRetryDelaySecondsKey, 10, 0, 3600),
EnableRetentionCleanup = bool.TryParse(GetValue(lookup, EnableRetentionCleanupKey, "false"), out var enableRetentionCleanup) && enableRetentionCleanup, EnableRetentionCleanup = bool.TryParse(GetValue(lookup, EnableRetentionCleanupKey, "false"), out var enableRetentionCleanup) && enableRetentionCleanup,
RetentionDays = GetIntValue(lookup, RetentionDaysKey, 30, 1, 3650), RetentionDays = GetIntValue(lookup, RetentionDaysKey, 30, 1, 3650),
RetentionDeleteFiles = bool.TryParse(GetValue(lookup, RetentionDeleteFilesKey, "false"), out var retentionDeleteFiles) && retentionDeleteFiles, RetentionDeleteFiles = bool.TryParse(GetValue(lookup, RetentionDeleteFilesKey, "false"), out var retentionDeleteFiles) && retentionDeleteFiles,
@@ -352,6 +356,8 @@ public sealed class SystemSettingsService : ISystemSettingsService
await UpsertAsync(SegmentCompletedScriptPathKey, request.SegmentCompletedScriptPath.Trim(), now, cancellationToken); await UpsertAsync(SegmentCompletedScriptPathKey, request.SegmentCompletedScriptPath.Trim(), now, cancellationToken);
await UpsertAsync(SegmentCompletedScriptContentKey, request.SegmentCompletedScriptContent, now, cancellationToken); await UpsertAsync(SegmentCompletedScriptContentKey, request.SegmentCompletedScriptContent, now, cancellationToken);
await UpsertAsync(EventScriptTimeoutSecondsKey, Math.Clamp(request.EventScriptTimeoutSeconds, 1, 3600).ToString(), now, cancellationToken); await UpsertAsync(EventScriptTimeoutSecondsKey, Math.Clamp(request.EventScriptTimeoutSeconds, 1, 3600).ToString(), now, cancellationToken);
await UpsertAsync(EventScriptRetryAttemptsKey, Math.Clamp(request.EventScriptRetryAttempts, 0, 20).ToString(), now, cancellationToken);
await UpsertAsync(EventScriptRetryDelaySecondsKey, Math.Clamp(request.EventScriptRetryDelaySeconds, 0, 3600).ToString(), now, cancellationToken);
await UpsertAsync(EnableRetentionCleanupKey, request.EnableRetentionCleanup.ToString(), now, cancellationToken); await UpsertAsync(EnableRetentionCleanupKey, request.EnableRetentionCleanup.ToString(), now, cancellationToken);
await UpsertAsync(RetentionDaysKey, Math.Clamp(request.RetentionDays, 1, 3650).ToString(), now, cancellationToken); await UpsertAsync(RetentionDaysKey, Math.Clamp(request.RetentionDays, 1, 3650).ToString(), now, cancellationToken);
await UpsertAsync(RetentionDeleteFilesKey, request.RetentionDeleteFiles.ToString(), now, cancellationToken); await UpsertAsync(RetentionDeleteFilesKey, request.RetentionDeleteFiles.ToString(), now, cancellationToken);
@@ -1,5 +1,7 @@
using System.Diagnostics; using System.Diagnostics;
using System.Text;
using LiveRecorder.Application.Abstractions.Logging; using LiveRecorder.Application.Abstractions.Logging;
using LiveRecorder.Application.Abstractions.Notifications;
using LiveRecorder.Application.Abstractions.Scripting; using LiveRecorder.Application.Abstractions.Scripting;
using LiveRecorder.Application.Abstractions.Settings; using LiveRecorder.Application.Abstractions.Settings;
using LiveRecorder.Application.Common; using LiveRecorder.Application.Common;
@@ -18,15 +20,21 @@ public sealed class EventScriptService : IEventScriptService
private readonly ISystemSettingsService _settingsService; private readonly ISystemSettingsService _settingsService;
private readonly ISystemLogService _systemLogService; private readonly ISystemLogService _systemLogService;
private readonly IEmailNotificationService _emailNotificationService;
private readonly IWebhookNotificationService _webhookNotificationService;
private readonly ILogger<EventScriptService> _logger; private readonly ILogger<EventScriptService> _logger;
public EventScriptService( public EventScriptService(
ISystemSettingsService settingsService, ISystemSettingsService settingsService,
ISystemLogService systemLogService, ISystemLogService systemLogService,
IEmailNotificationService emailNotificationService,
IWebhookNotificationService webhookNotificationService,
ILogger<EventScriptService> logger) ILogger<EventScriptService> logger)
{ {
_settingsService = settingsService; _settingsService = settingsService;
_systemLogService = systemLogService; _systemLogService = systemLogService;
_emailNotificationService = emailNotificationService;
_webhookNotificationService = webhookNotificationService;
_logger = logger; _logger = logger;
} }
@@ -44,8 +52,12 @@ public sealed class EventScriptService : IEventScriptService
settings.LiveStartedScriptPath, settings.LiveStartedScriptPath,
settings.LiveStartedScriptContent, settings.LiveStartedScriptContent,
settings.EventScriptTimeoutSeconds, settings.EventScriptTimeoutSeconds,
settings.EventScriptRetryAttempts,
settings.EventScriptRetryDelaySeconds,
"live_started", "live_started",
environment, environment,
liveRoom,
recordTask: null,
"Script", "Script",
liveRoom.Id, liveRoom.Id,
recordSessionId: null, recordSessionId: null,
@@ -67,8 +79,12 @@ public sealed class EventScriptService : IEventScriptService
settings.LiveEndedScriptPath, settings.LiveEndedScriptPath,
settings.LiveEndedScriptContent, settings.LiveEndedScriptContent,
settings.EventScriptTimeoutSeconds, settings.EventScriptTimeoutSeconds,
settings.EventScriptRetryAttempts,
settings.EventScriptRetryDelaySeconds,
"live_ended", "live_ended",
environment, environment,
liveRoom,
recordTask: null,
"Script", "Script",
liveRoom.Id, liveRoom.Id,
recordSessionId: null, recordSessionId: null,
@@ -105,8 +121,12 @@ public sealed class EventScriptService : IEventScriptService
settings.SegmentCompletedScriptPath, settings.SegmentCompletedScriptPath,
settings.SegmentCompletedScriptContent, settings.SegmentCompletedScriptContent,
settings.EventScriptTimeoutSeconds, settings.EventScriptTimeoutSeconds,
settings.EventScriptRetryAttempts,
settings.EventScriptRetryDelaySeconds,
"segment_completed", "segment_completed",
environment, environment,
liveRoom,
recordTask,
"Script", "Script",
liveRoom?.Id ?? recordSession.LiveRoomId, liveRoom?.Id ?? recordSession.LiveRoomId,
recordSession.Id, recordSession.Id,
@@ -176,8 +196,12 @@ public sealed class EventScriptService : IEventScriptService
string scriptPath, string scriptPath,
string scriptContent, string scriptContent,
int timeoutSeconds, int timeoutSeconds,
int retryAttempts,
int retryDelaySeconds,
string eventName, string eventName,
IReadOnlyDictionary<string, string> environment, IReadOnlyDictionary<string, string> environment,
LiveRoom? liveRoom,
RecordTask? recordTask,
string logCategory, string logCategory,
Guid? liveRoomId, Guid? liveRoomId,
Guid? recordSessionId, Guid? recordSessionId,
@@ -189,6 +213,43 @@ public sealed class EventScriptService : IEventScriptService
return null; return null;
} }
var outcome = await ExecuteWithRetryAsync(
scriptMode,
scriptPath,
scriptContent,
timeoutSeconds,
retryAttempts,
retryDelaySeconds,
eventName,
environment,
liveRoom,
recordTask,
logCategory,
liveRoomId,
recordSessionId,
recordTaskId,
cancellationToken);
return MapOutcome(outcome);
}
private async Task<ScriptExecutionOutcome> ExecuteWithRetryAsync(
string scriptMode,
string scriptPath,
string scriptContent,
int timeoutSeconds,
int retryAttempts,
int retryDelaySeconds,
string eventName,
IReadOnlyDictionary<string, string> environment,
LiveRoom? liveRoom,
RecordTask? recordTask,
string logCategory,
Guid? liveRoomId,
Guid? recordSessionId,
Guid? recordTaskId,
CancellationToken cancellationToken)
{
var outcome = await ExecuteAsync( var outcome = await ExecuteAsync(
scriptMode, scriptMode,
scriptPath, scriptPath,
@@ -202,7 +263,53 @@ public sealed class EventScriptService : IEventScriptService
recordTaskId, recordTaskId,
cancellationToken); cancellationToken);
return MapOutcome(outcome); if (outcome.Success || !IsRetryableFailure(outcome))
{
return outcome;
}
var maxRetryAttempts = Math.Clamp(retryAttempts, 0, 20);
var totalAttempts = 1;
for (var retryIndex = 1; retryIndex <= maxRetryAttempts; retryIndex++)
{
await _systemLogService.WriteAsync(
SystemLogLevel.Warning,
logCategory,
$"Event script retry {retryIndex} of {maxRetryAttempts} scheduled for {eventName}.",
BuildRetryAttemptDetail(eventName, retryIndex + 1, maxRetryAttempts + 1, retryDelaySeconds, outcome),
liveRoomId,
recordSessionId,
recordTaskId,
cancellationToken);
if (retryDelaySeconds > 0)
{
await Task.Delay(TimeSpan.FromSeconds(Math.Clamp(retryDelaySeconds, 0, 3600)), cancellationToken);
}
outcome = await ExecuteAsync(
scriptMode,
scriptPath,
scriptContent,
timeoutSeconds,
eventName,
environment,
logCategory,
liveRoomId,
recordSessionId,
recordTaskId,
cancellationToken);
totalAttempts++;
if (outcome.Success || !IsRetryableFailure(outcome))
{
return outcome;
}
}
await NotifyRetryExhaustedAsync(eventName, totalAttempts, outcome, liveRoom, recordTask, cancellationToken);
return outcome;
} }
private async Task<ScriptExecutionOutcome> ExecuteAsync( private async Task<ScriptExecutionOutcome> ExecuteAsync(
@@ -225,7 +332,9 @@ public sealed class EventScriptService : IEventScriptService
false, false,
$"Event script was not configured for {eventName}.", $"Event script was not configured for {eventName}.",
null, null,
null); null,
null,
ScriptFailureKind.MissingConfiguration);
await WriteOutcomeLogAsync( await WriteOutcomeLogAsync(
missingConfiguration, missingConfiguration,
@@ -244,7 +353,9 @@ public sealed class EventScriptService : IEventScriptService
false, false,
$"Event script was not found for {eventName}.", $"Event script was not found for {eventName}.",
execution.Detail, execution.Detail,
null); null,
execution.Detail,
ScriptFailureKind.MissingScript);
await WriteOutcomeLogAsync( await WriteOutcomeLogAsync(
missingScript, missingScript,
@@ -295,7 +406,9 @@ public sealed class EventScriptService : IEventScriptService
? $"Event script completed for {eventName}." ? $"Event script completed for {eventName}."
: $"Event script exited with code {process.ExitCode} for {eventName}.", : $"Event script exited with code {process.ExitCode} for {eventName}.",
execution.Detail, execution.Detail,
null); null,
execution.Detail,
process.ExitCode == 0 ? ScriptFailureKind.None : ScriptFailureKind.ExitCode);
outcomeLevel = process.ExitCode == 0 ? SystemLogLevel.Info : SystemLogLevel.Warning; outcomeLevel = process.ExitCode == 0 ? SystemLogLevel.Info : SystemLogLevel.Warning;
} }
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
@@ -305,7 +418,9 @@ public sealed class EventScriptService : IEventScriptService
false, false,
$"Event script timed out for {eventName}.", $"Event script timed out for {eventName}.",
execution.Detail, execution.Detail,
null); null,
execution.Detail,
ScriptFailureKind.Timeout);
outcomeLevel = SystemLogLevel.Warning; outcomeLevel = SystemLogLevel.Warning;
} }
catch (Exception ex) catch (Exception ex)
@@ -315,7 +430,9 @@ public sealed class EventScriptService : IEventScriptService
false, false,
$"Event script failed for {eventName}.", $"Event script failed for {eventName}.",
ex.ToString(), ex.ToString(),
null); null,
execution.Detail,
ScriptFailureKind.Exception);
outcomeLevel = SystemLogLevel.Warning; outcomeLevel = SystemLogLevel.Warning;
} }
finally finally
@@ -344,6 +461,31 @@ public sealed class EventScriptService : IEventScriptService
return outcome; return outcome;
} }
private async Task NotifyRetryExhaustedAsync(
string eventName,
int totalAttempts,
ScriptExecutionOutcome outcome,
LiveRoom? liveRoom,
RecordTask? recordTask,
CancellationToken cancellationToken)
{
var detail = BuildRetryFailureNotificationDetail(eventName, totalAttempts, outcome);
await _emailNotificationService.SendExceptionAsync(
"EventScript",
"Event script failed after retries.",
detail,
liveRoom,
recordTask,
cancellationToken);
await _webhookNotificationService.SendExceptionAsync(
"EventScript",
"Event script failed after retries.",
detail,
liveRoom,
recordTask,
cancellationToken);
}
private static EventScriptExecution? CreateExecution(string scriptMode, string scriptPath, string scriptContent) private static EventScriptExecution? CreateExecution(string scriptMode, string scriptPath, string scriptContent)
{ {
if (string.Equals(scriptMode, EventScriptSourceModes.Inline, StringComparison.OrdinalIgnoreCase)) if (string.Equals(scriptMode, EventScriptSourceModes.Inline, StringComparison.OrdinalIgnoreCase))
@@ -582,6 +724,71 @@ public sealed class EventScriptService : IEventScriptService
cancellationToken); cancellationToken);
} }
private static string BuildRetryAttemptDetail(
string eventName,
int nextAttempt,
int totalAttempts,
int retryDelaySeconds,
ScriptExecutionOutcome outcome)
{
var builder = new StringBuilder();
builder.AppendLine($"Event: {eventName}");
builder.AppendLine($"Next attempt: {nextAttempt}/{totalAttempts}");
builder.AppendLine($"Retry delay: {Math.Clamp(retryDelaySeconds, 0, 3600)} second(s)");
builder.AppendLine($"Last result: {outcome.Message}");
if (!string.IsNullOrWhiteSpace(outcome.ExecutionTarget))
{
builder.AppendLine($"Script: {outcome.ExecutionTarget}");
}
if (!string.IsNullOrWhiteSpace(outcome.Detail) &&
!string.Equals(outcome.Detail, outcome.ExecutionTarget, StringComparison.Ordinal))
{
builder.AppendLine();
builder.AppendLine("Detail:");
builder.AppendLine(outcome.Detail);
}
return TruncateSystemLogDetail(builder.ToString().Trim());
}
private static string BuildRetryFailureNotificationDetail(
string eventName,
int totalAttempts,
ScriptExecutionOutcome outcome)
{
var builder = new StringBuilder();
builder.AppendLine($"Event: {eventName}");
builder.AppendLine($"Attempts: {totalAttempts}");
builder.AppendLine($"Last result: {outcome.Message}");
if (!string.IsNullOrWhiteSpace(outcome.ExecutionTarget))
{
builder.AppendLine($"Script: {outcome.ExecutionTarget}");
}
if (!string.IsNullOrWhiteSpace(outcome.Detail) &&
!string.Equals(outcome.Detail, outcome.ExecutionTarget, StringComparison.Ordinal))
{
builder.AppendLine();
builder.AppendLine("Detail:");
builder.AppendLine(outcome.Detail);
}
if (!string.IsNullOrWhiteSpace(outcome.CustomLogOutput))
{
builder.AppendLine();
builder.AppendLine("Custom log output:");
builder.AppendLine(outcome.CustomLogOutput);
}
return TruncateSystemLogDetail(builder.ToString().Trim());
}
private static bool IsRetryableFailure(ScriptExecutionOutcome outcome) =>
outcome.FailureKind is ScriptFailureKind.ExitCode or ScriptFailureKind.Timeout or ScriptFailureKind.Exception;
private static string CreateScriptLogPath() private static string CreateScriptLogPath()
{ {
return Path.Combine( return Path.Combine(
@@ -628,11 +835,23 @@ public sealed class EventScriptService : IEventScriptService
} }
} }
private enum ScriptFailureKind
{
None,
MissingConfiguration,
MissingScript,
ExitCode,
Timeout,
Exception
}
private sealed record EventScriptExecution(ProcessStartInfo? StartInfo, string Detail, bool IsMissing = false); private sealed record EventScriptExecution(ProcessStartInfo? StartInfo, string Detail, bool IsMissing = false);
private sealed record ScriptExecutionOutcome( private sealed record ScriptExecutionOutcome(
bool Success, bool Success,
string Message, string Message,
string? Detail, string? Detail,
string? CustomLogOutput); string? CustomLogOutput,
string? ExecutionTarget,
ScriptFailureKind FailureKind);
} }
@@ -1,4 +1,5 @@
using System.Collections.Concurrent; using System.Collections.Concurrent;
using System.Text;
using LiveRecorder.Application.Common; using LiveRecorder.Application.Common;
using LiveRecorder.Application.Abstractions.Logging; using LiveRecorder.Application.Abstractions.Logging;
using LiveRecorder.Application.Abstractions.Notifications; using LiveRecorder.Application.Abstractions.Notifications;
@@ -360,6 +361,9 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
dbContext, dbContext,
ffmpegService, ffmpegService,
logService, logService,
emailNotificationService,
webhookNotificationService,
liveRoom,
liveRoom.Id, liveRoom.Id,
pauseCheck.Message, pauseCheck.Message,
cancellationToken); cancellationToken);
@@ -589,6 +593,9 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
LiveRecorderDbContext dbContext, LiveRecorderDbContext dbContext,
IFfmpegService ffmpegService, IFfmpegService ffmpegService,
ISystemLogService logService, ISystemLogService logService,
IEmailNotificationService emailNotificationService,
IWebhookNotificationService webhookNotificationService,
Domain.Entities.LiveRoom liveRoom,
Guid liveRoomId, Guid liveRoomId,
string detail, string detail,
CancellationToken cancellationToken) CancellationToken cancellationToken)
@@ -600,8 +607,13 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
item.Status == RecordSessionStatus.Stopping)) item.Status == RecordSessionStatus.Stopping))
.ToListAsync(cancellationToken); .ToListAsync(cancellationToken);
var affectedSessionCount = 0;
var forcedStopCount = 0;
foreach (var activeSession in activeSessions.OrderBy(static item => item.CreatedAt)) foreach (var activeSession in activeSessions.OrderBy(static item => item.CreatedAt))
{ {
affectedSessionCount++;
await logService.WriteAsync( await logService.WriteAsync(
SystemLogLevel.Warning, SystemLogLevel.Warning,
"Storage", "Storage",
@@ -628,12 +640,32 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
liveRoomId, liveRoomId,
activeSession.Id, activeSession.Id,
cancellationToken: cancellationToken); cancellationToken: cancellationToken);
forcedStopCount++;
await ffmpegService.KillAndWaitAsync(activeSession.Id, OfflineForcedStopTimeout, cancellationToken); await ffmpegService.KillAndWaitAsync(activeSession.Id, OfflineForcedStopTimeout, cancellationToken);
} }
} }
await ffmpegService.TryReconcileInactiveSessionAsync(activeSession.Id, cancellationToken); await ffmpegService.TryReconcileInactiveSessionAsync(activeSession.Id, cancellationToken);
} }
if (affectedSessionCount <= 0)
{
return;
}
var notificationDetail = BuildLowStorageNotificationDetail(liveRoom, affectedSessionCount, forcedStopCount, detail);
await emailNotificationService.SendExceptionAsync(
"StorageGuard",
"Low storage paused active recording sessions.",
notificationDetail,
liveRoom,
cancellationToken: cancellationToken);
await webhookNotificationService.SendExceptionAsync(
"StorageGuard",
"Low storage paused active recording sessions.",
notificationDetail,
liveRoom,
cancellationToken: cancellationToken);
} }
private static async Task<IReadOnlyList<Guid>> ReconcileStaleActiveSessionsAsync( private static async Task<IReadOnlyList<Guid>> ReconcileStaleActiveSessionsAsync(
@@ -699,6 +731,39 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
IsTransientPollingException(exception.InnerException, cancellationToken); IsTransientPollingException(exception.InnerException, cancellationToken);
} }
private static string BuildLowStorageNotificationDetail(
Domain.Entities.LiveRoom liveRoom,
int affectedSessionCount,
int forcedStopCount,
string storageDetail)
{
var builder = new StringBuilder();
builder.AppendLine($"Platform: {liveRoom.Platform}");
builder.AppendLine($"Room ID: {liveRoom.RoomId}");
if (!string.IsNullOrWhiteSpace(liveRoom.AnchorName))
{
builder.AppendLine($"Anchor: {liveRoom.AnchorName}");
}
if (!string.IsNullOrWhiteSpace(liveRoom.Title))
{
builder.AppendLine($"Title: {liveRoom.Title}");
}
builder.AppendLine($"Affected sessions: {affectedSessionCount}");
builder.AppendLine($"Forced stop attempts: {forcedStopCount}");
if (!string.IsNullOrWhiteSpace(storageDetail))
{
builder.AppendLine();
builder.AppendLine("Storage detail:");
builder.AppendLine(storageDetail);
}
return builder.ToString().Trim();
}
private static string BuildPollingFailureDetail(Exception exception) private static string BuildPollingFailureDetail(Exception exception)
{ {
var root = exception.GetBaseException(); var root = exception.GetBaseException();