feat: add storage and script failure notifications
This commit is contained in:
@@ -61,9 +61,17 @@
|
||||
--el-border-color-light: var(--border-subtle);
|
||||
--el-border-radius-base: var(--radius-sm);
|
||||
--el-bg-color: transparent;
|
||||
--el-bg-color-page: var(--bg-base);
|
||||
--el-bg-color-overlay: var(--surface);
|
||||
--el-fill-color: var(--surface-muted);
|
||||
--el-fill-color-blank: var(--surface);
|
||||
--el-fill-color-light: var(--surface-muted);
|
||||
--el-fill-color-lighter: var(--surface-strong);
|
||||
--el-fill-color-dark: var(--surface-raised);
|
||||
--el-fill-color-darker: var(--surface-strong);
|
||||
--el-disabled-bg-color: var(--surface-muted);
|
||||
--el-disabled-text-color: var(--text-soft);
|
||||
--el-text-color-placeholder: var(--text-soft);
|
||||
--el-mask-color: rgba(15, 23, 42, 0.54);
|
||||
}
|
||||
|
||||
@@ -123,9 +131,17 @@ html[data-theme="dark"] {
|
||||
--el-text-color-secondary: var(--text-muted);
|
||||
--el-border-color: var(--border-base);
|
||||
--el-border-color-light: var(--border-subtle);
|
||||
--el-bg-color-page: var(--bg-base);
|
||||
--el-bg-color-overlay: var(--surface);
|
||||
--el-fill-color: var(--surface-muted);
|
||||
--el-fill-color-blank: var(--surface);
|
||||
--el-fill-color-light: var(--surface-muted);
|
||||
--el-fill-color-lighter: var(--surface-strong);
|
||||
--el-fill-color-dark: var(--surface-raised);
|
||||
--el-fill-color-darker: var(--surface-strong);
|
||||
--el-disabled-bg-color: var(--surface-muted);
|
||||
--el-disabled-text-color: var(--text-soft);
|
||||
--el-text-color-placeholder: var(--text-soft);
|
||||
--el-mask-color: rgba(3, 7, 14, 0.72);
|
||||
}
|
||||
|
||||
@@ -255,6 +271,7 @@ select:focus-visible {
|
||||
.surface-card {
|
||||
position: relative;
|
||||
overflow: hidden;
|
||||
--el-card-bg-color: transparent;
|
||||
border-radius: var(--radius-md);
|
||||
border: 1px solid var(--border-subtle);
|
||||
background:
|
||||
@@ -573,9 +590,33 @@ html[data-theme="dark"] .stat-card {
|
||||
|
||||
.el-card {
|
||||
--el-card-border-color: transparent;
|
||||
--el-card-bg-color: transparent;
|
||||
background: transparent;
|
||||
}
|
||||
|
||||
.el-tabs--border-card {
|
||||
border-color: var(--border-subtle);
|
||||
background: var(--surface);
|
||||
}
|
||||
|
||||
.el-tabs--border-card > .el-tabs__content {
|
||||
background: transparent;
|
||||
color: var(--text-primary);
|
||||
}
|
||||
|
||||
.el-tabs--border-card > .el-tabs__header {
|
||||
background: transparent;
|
||||
}
|
||||
|
||||
.el-tabs--border-card > .el-tabs__header .el-tabs__item {
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
|
||||
.el-tabs--border-card > .el-tabs__header .el-tabs__item.is-active {
|
||||
color: var(--text-primary);
|
||||
background: var(--surface-raised);
|
||||
}
|
||||
|
||||
.el-button {
|
||||
min-height: var(--control-height);
|
||||
padding: 0 15px;
|
||||
@@ -802,6 +843,17 @@ html[data-theme="dark"] .stat-card {
|
||||
box-shadow: var(--shadow-float);
|
||||
}
|
||||
|
||||
.el-message-box,
|
||||
.el-popover.el-popper,
|
||||
.el-select__popper.el-popper,
|
||||
.el-picker__popper.el-popper,
|
||||
.el-dropdown__popper.el-popper .el-dropdown-menu {
|
||||
border-color: var(--border-subtle);
|
||||
background: linear-gradient(180deg, var(--surface-raised), var(--surface));
|
||||
color: var(--text-primary);
|
||||
box-shadow: var(--shadow-float);
|
||||
}
|
||||
|
||||
.el-dialog__header {
|
||||
margin: 0;
|
||||
padding: 22px 24px 10px;
|
||||
|
||||
@@ -458,6 +458,8 @@ export interface SystemSettings {
|
||||
segmentCompletedScriptPath: string;
|
||||
segmentCompletedScriptContent: string;
|
||||
eventScriptTimeoutSeconds: number;
|
||||
eventScriptRetryAttempts: number;
|
||||
eventScriptRetryDelaySeconds: number;
|
||||
enableEmailNotification: boolean;
|
||||
emailSmtpHost: string;
|
||||
emailSmtpPort: number;
|
||||
|
||||
@@ -168,6 +168,8 @@ const form = reactive<SettingsFormModel>({
|
||||
segmentCompletedScriptPath: "",
|
||||
segmentCompletedScriptContent: "",
|
||||
eventScriptTimeoutSeconds: 60,
|
||||
eventScriptRetryAttempts: 3,
|
||||
eventScriptRetryDelaySeconds: 10,
|
||||
enableEmailNotification: false,
|
||||
emailSmtpHost: "",
|
||||
emailSmtpPort: 587,
|
||||
@@ -1480,16 +1482,26 @@ watch(
|
||||
|
||||
<el-form label-position="top">
|
||||
<el-row :gutter="16">
|
||||
<el-col :span="8">
|
||||
<el-col :span="6">
|
||||
<el-form-item label="启用事件脚本">
|
||||
<el-switch v-model="form.enableEventScripts" />
|
||||
</el-form-item>
|
||||
</el-col>
|
||||
<el-col :span="8">
|
||||
<el-col :span="6">
|
||||
<el-form-item label="脚本超时(秒)">
|
||||
<el-input-number v-model="form.eventScriptTimeoutSeconds" :min="1" :max="3600" />
|
||||
</el-form-item>
|
||||
</el-col>
|
||||
<el-col :span="5">
|
||||
<el-form-item label="重试次数">
|
||||
<el-input-number v-model="form.eventScriptRetryAttempts" :min="0" :max="20" />
|
||||
</el-form-item>
|
||||
</el-col>
|
||||
<el-col :span="6">
|
||||
<el-form-item label="重试间隔(秒)">
|
||||
<el-input-number v-model="form.eventScriptRetryDelaySeconds" :min="0" :max="3600" />
|
||||
</el-form-item>
|
||||
</el-col>
|
||||
</el-row>
|
||||
</el-form>
|
||||
|
||||
@@ -1676,6 +1688,10 @@ watch(
|
||||
The official Docker image includes <code>curl</code> and <code>jq</code> by default. If you run on a host machine or a custom image, rely on the commands available in that environment.
|
||||
</div>
|
||||
|
||||
<div class="event-script-help__intro">
|
||||
Set <code>Retry attempts</code> to <code>0</code> to disable automatic retries. Retry exhaustion failures are sent through the existing exception notification channel.
|
||||
</div>
|
||||
|
||||
<div class="event-script-example">
|
||||
<div class="event-script-example__label">Environment variable examples</div>
|
||||
<div class="event-script-example__grid">
|
||||
@@ -1699,7 +1715,8 @@ watch(
|
||||
|
||||
<el-card class="surface-card settings-card settings-grid__full" shadow="never">
|
||||
<h3 class="section-title">Webhook 閫氱煡</h3>
|
||||
<p class="section-subtitle">Send fixed JSON POST payloads with custom headers. Currently only live-started and exception events are sent.</p>
|
||||
<h3 class="section-title">Webhook 通知</h3>
|
||||
<p class="section-subtitle">Send fixed JSON POST payloads with custom headers. Exception notifications also cover low-storage stop events and event-script failures after retries are exhausted.</p>
|
||||
|
||||
<el-form label-position="top">
|
||||
<el-row :gutter="16">
|
||||
@@ -1772,7 +1789,8 @@ watch(
|
||||
|
||||
<el-card class="surface-card settings-card settings-grid__full" shadow="never">
|
||||
<h3 class="section-title">閭欢閫氱煡</h3>
|
||||
<p class="section-subtitle">Configure SMTP plus HTML templates for live-started and exception alerts. Tests use the current unsaved form values directly.</p>
|
||||
<h3 class="section-title">邮件通知</h3>
|
||||
<p class="section-subtitle">Configure SMTP plus HTML templates for live-started and exception alerts. Exception notifications also cover low-storage stop events and event-script failures after retries are exhausted.</p>
|
||||
|
||||
<el-form label-position="top">
|
||||
<el-row :gutter="16">
|
||||
|
||||
@@ -177,6 +177,10 @@ public sealed class SystemSettingsDto
|
||||
|
||||
public int EventScriptTimeoutSeconds { get; set; } = 60;
|
||||
|
||||
public int EventScriptRetryAttempts { get; set; } = 3;
|
||||
|
||||
public int EventScriptRetryDelaySeconds { get; set; } = 10;
|
||||
|
||||
public bool EnableRetentionCleanup { get; set; } = false;
|
||||
|
||||
public int RetentionDays { get; set; } = 30;
|
||||
@@ -446,6 +450,10 @@ public sealed class UpdateSystemSettingsRequest
|
||||
|
||||
public int EventScriptTimeoutSeconds { get; set; } = 60;
|
||||
|
||||
public int EventScriptRetryAttempts { get; set; } = 3;
|
||||
|
||||
public int EventScriptRetryDelaySeconds { get; set; } = 10;
|
||||
|
||||
public bool EnableRetentionCleanup { get; set; } = false;
|
||||
|
||||
public int RetentionDays { get; set; } = 30;
|
||||
|
||||
@@ -71,6 +71,8 @@ public sealed class SystemSettingsService : ISystemSettingsService
|
||||
private const string SegmentCompletedScriptPathKey = "event_scripts.segment_completed.path";
|
||||
private const string SegmentCompletedScriptContentKey = "event_scripts.segment_completed.content";
|
||||
private const string EventScriptTimeoutSecondsKey = "event_scripts.timeout_seconds";
|
||||
private const string EventScriptRetryAttemptsKey = "event_scripts.retry_attempts";
|
||||
private const string EventScriptRetryDelaySecondsKey = "event_scripts.retry_delay_seconds";
|
||||
private const string EnableRetentionCleanupKey = "retention.cleanup.enabled";
|
||||
private const string RetentionDaysKey = "retention.cleanup.days";
|
||||
private const string RetentionDeleteFilesKey = "retention.cleanup.delete_files";
|
||||
@@ -202,6 +204,8 @@ public sealed class SystemSettingsService : ISystemSettingsService
|
||||
SegmentCompletedScriptPath = GetValue(lookup, SegmentCompletedScriptPathKey, string.Empty),
|
||||
SegmentCompletedScriptContent = GetValue(lookup, SegmentCompletedScriptContentKey, string.Empty),
|
||||
EventScriptTimeoutSeconds = GetIntValue(lookup, EventScriptTimeoutSecondsKey, 60, 1, 3600),
|
||||
EventScriptRetryAttempts = GetIntValue(lookup, EventScriptRetryAttemptsKey, 3, 0, 20),
|
||||
EventScriptRetryDelaySeconds = GetIntValue(lookup, EventScriptRetryDelaySecondsKey, 10, 0, 3600),
|
||||
EnableRetentionCleanup = bool.TryParse(GetValue(lookup, EnableRetentionCleanupKey, "false"), out var enableRetentionCleanup) && enableRetentionCleanup,
|
||||
RetentionDays = GetIntValue(lookup, RetentionDaysKey, 30, 1, 3650),
|
||||
RetentionDeleteFiles = bool.TryParse(GetValue(lookup, RetentionDeleteFilesKey, "false"), out var retentionDeleteFiles) && retentionDeleteFiles,
|
||||
@@ -352,6 +356,8 @@ public sealed class SystemSettingsService : ISystemSettingsService
|
||||
await UpsertAsync(SegmentCompletedScriptPathKey, request.SegmentCompletedScriptPath.Trim(), now, cancellationToken);
|
||||
await UpsertAsync(SegmentCompletedScriptContentKey, request.SegmentCompletedScriptContent, now, cancellationToken);
|
||||
await UpsertAsync(EventScriptTimeoutSecondsKey, Math.Clamp(request.EventScriptTimeoutSeconds, 1, 3600).ToString(), now, cancellationToken);
|
||||
await UpsertAsync(EventScriptRetryAttemptsKey, Math.Clamp(request.EventScriptRetryAttempts, 0, 20).ToString(), now, cancellationToken);
|
||||
await UpsertAsync(EventScriptRetryDelaySecondsKey, Math.Clamp(request.EventScriptRetryDelaySeconds, 0, 3600).ToString(), now, cancellationToken);
|
||||
await UpsertAsync(EnableRetentionCleanupKey, request.EnableRetentionCleanup.ToString(), now, cancellationToken);
|
||||
await UpsertAsync(RetentionDaysKey, Math.Clamp(request.RetentionDays, 1, 3650).ToString(), now, cancellationToken);
|
||||
await UpsertAsync(RetentionDeleteFilesKey, request.RetentionDeleteFiles.ToString(), now, cancellationToken);
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
using System.Diagnostics;
|
||||
using System.Text;
|
||||
using LiveRecorder.Application.Abstractions.Logging;
|
||||
using LiveRecorder.Application.Abstractions.Notifications;
|
||||
using LiveRecorder.Application.Abstractions.Scripting;
|
||||
using LiveRecorder.Application.Abstractions.Settings;
|
||||
using LiveRecorder.Application.Common;
|
||||
@@ -18,15 +20,21 @@ public sealed class EventScriptService : IEventScriptService
|
||||
|
||||
private readonly ISystemSettingsService _settingsService;
|
||||
private readonly ISystemLogService _systemLogService;
|
||||
private readonly IEmailNotificationService _emailNotificationService;
|
||||
private readonly IWebhookNotificationService _webhookNotificationService;
|
||||
private readonly ILogger<EventScriptService> _logger;
|
||||
|
||||
public EventScriptService(
|
||||
ISystemSettingsService settingsService,
|
||||
ISystemLogService systemLogService,
|
||||
IEmailNotificationService emailNotificationService,
|
||||
IWebhookNotificationService webhookNotificationService,
|
||||
ILogger<EventScriptService> logger)
|
||||
{
|
||||
_settingsService = settingsService;
|
||||
_systemLogService = systemLogService;
|
||||
_emailNotificationService = emailNotificationService;
|
||||
_webhookNotificationService = webhookNotificationService;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
@@ -44,8 +52,12 @@ public sealed class EventScriptService : IEventScriptService
|
||||
settings.LiveStartedScriptPath,
|
||||
settings.LiveStartedScriptContent,
|
||||
settings.EventScriptTimeoutSeconds,
|
||||
settings.EventScriptRetryAttempts,
|
||||
settings.EventScriptRetryDelaySeconds,
|
||||
"live_started",
|
||||
environment,
|
||||
liveRoom,
|
||||
recordTask: null,
|
||||
"Script",
|
||||
liveRoom.Id,
|
||||
recordSessionId: null,
|
||||
@@ -67,8 +79,12 @@ public sealed class EventScriptService : IEventScriptService
|
||||
settings.LiveEndedScriptPath,
|
||||
settings.LiveEndedScriptContent,
|
||||
settings.EventScriptTimeoutSeconds,
|
||||
settings.EventScriptRetryAttempts,
|
||||
settings.EventScriptRetryDelaySeconds,
|
||||
"live_ended",
|
||||
environment,
|
||||
liveRoom,
|
||||
recordTask: null,
|
||||
"Script",
|
||||
liveRoom.Id,
|
||||
recordSessionId: null,
|
||||
@@ -105,8 +121,12 @@ public sealed class EventScriptService : IEventScriptService
|
||||
settings.SegmentCompletedScriptPath,
|
||||
settings.SegmentCompletedScriptContent,
|
||||
settings.EventScriptTimeoutSeconds,
|
||||
settings.EventScriptRetryAttempts,
|
||||
settings.EventScriptRetryDelaySeconds,
|
||||
"segment_completed",
|
||||
environment,
|
||||
liveRoom,
|
||||
recordTask,
|
||||
"Script",
|
||||
liveRoom?.Id ?? recordSession.LiveRoomId,
|
||||
recordSession.Id,
|
||||
@@ -176,8 +196,12 @@ public sealed class EventScriptService : IEventScriptService
|
||||
string scriptPath,
|
||||
string scriptContent,
|
||||
int timeoutSeconds,
|
||||
int retryAttempts,
|
||||
int retryDelaySeconds,
|
||||
string eventName,
|
||||
IReadOnlyDictionary<string, string> environment,
|
||||
LiveRoom? liveRoom,
|
||||
RecordTask? recordTask,
|
||||
string logCategory,
|
||||
Guid? liveRoomId,
|
||||
Guid? recordSessionId,
|
||||
@@ -189,6 +213,43 @@ public sealed class EventScriptService : IEventScriptService
|
||||
return null;
|
||||
}
|
||||
|
||||
var outcome = await ExecuteWithRetryAsync(
|
||||
scriptMode,
|
||||
scriptPath,
|
||||
scriptContent,
|
||||
timeoutSeconds,
|
||||
retryAttempts,
|
||||
retryDelaySeconds,
|
||||
eventName,
|
||||
environment,
|
||||
liveRoom,
|
||||
recordTask,
|
||||
logCategory,
|
||||
liveRoomId,
|
||||
recordSessionId,
|
||||
recordTaskId,
|
||||
cancellationToken);
|
||||
|
||||
return MapOutcome(outcome);
|
||||
}
|
||||
|
||||
private async Task<ScriptExecutionOutcome> ExecuteWithRetryAsync(
|
||||
string scriptMode,
|
||||
string scriptPath,
|
||||
string scriptContent,
|
||||
int timeoutSeconds,
|
||||
int retryAttempts,
|
||||
int retryDelaySeconds,
|
||||
string eventName,
|
||||
IReadOnlyDictionary<string, string> environment,
|
||||
LiveRoom? liveRoom,
|
||||
RecordTask? recordTask,
|
||||
string logCategory,
|
||||
Guid? liveRoomId,
|
||||
Guid? recordSessionId,
|
||||
Guid? recordTaskId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var outcome = await ExecuteAsync(
|
||||
scriptMode,
|
||||
scriptPath,
|
||||
@@ -202,7 +263,53 @@ public sealed class EventScriptService : IEventScriptService
|
||||
recordTaskId,
|
||||
cancellationToken);
|
||||
|
||||
return MapOutcome(outcome);
|
||||
if (outcome.Success || !IsRetryableFailure(outcome))
|
||||
{
|
||||
return outcome;
|
||||
}
|
||||
|
||||
var maxRetryAttempts = Math.Clamp(retryAttempts, 0, 20);
|
||||
var totalAttempts = 1;
|
||||
|
||||
for (var retryIndex = 1; retryIndex <= maxRetryAttempts; retryIndex++)
|
||||
{
|
||||
await _systemLogService.WriteAsync(
|
||||
SystemLogLevel.Warning,
|
||||
logCategory,
|
||||
$"Event script retry {retryIndex} of {maxRetryAttempts} scheduled for {eventName}.",
|
||||
BuildRetryAttemptDetail(eventName, retryIndex + 1, maxRetryAttempts + 1, retryDelaySeconds, outcome),
|
||||
liveRoomId,
|
||||
recordSessionId,
|
||||
recordTaskId,
|
||||
cancellationToken);
|
||||
|
||||
if (retryDelaySeconds > 0)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(Math.Clamp(retryDelaySeconds, 0, 3600)), cancellationToken);
|
||||
}
|
||||
|
||||
outcome = await ExecuteAsync(
|
||||
scriptMode,
|
||||
scriptPath,
|
||||
scriptContent,
|
||||
timeoutSeconds,
|
||||
eventName,
|
||||
environment,
|
||||
logCategory,
|
||||
liveRoomId,
|
||||
recordSessionId,
|
||||
recordTaskId,
|
||||
cancellationToken);
|
||||
totalAttempts++;
|
||||
|
||||
if (outcome.Success || !IsRetryableFailure(outcome))
|
||||
{
|
||||
return outcome;
|
||||
}
|
||||
}
|
||||
|
||||
await NotifyRetryExhaustedAsync(eventName, totalAttempts, outcome, liveRoom, recordTask, cancellationToken);
|
||||
return outcome;
|
||||
}
|
||||
|
||||
private async Task<ScriptExecutionOutcome> ExecuteAsync(
|
||||
@@ -225,7 +332,9 @@ public sealed class EventScriptService : IEventScriptService
|
||||
false,
|
||||
$"Event script was not configured for {eventName}.",
|
||||
null,
|
||||
null);
|
||||
null,
|
||||
null,
|
||||
ScriptFailureKind.MissingConfiguration);
|
||||
|
||||
await WriteOutcomeLogAsync(
|
||||
missingConfiguration,
|
||||
@@ -244,7 +353,9 @@ public sealed class EventScriptService : IEventScriptService
|
||||
false,
|
||||
$"Event script was not found for {eventName}.",
|
||||
execution.Detail,
|
||||
null);
|
||||
null,
|
||||
execution.Detail,
|
||||
ScriptFailureKind.MissingScript);
|
||||
|
||||
await WriteOutcomeLogAsync(
|
||||
missingScript,
|
||||
@@ -295,7 +406,9 @@ public sealed class EventScriptService : IEventScriptService
|
||||
? $"Event script completed for {eventName}."
|
||||
: $"Event script exited with code {process.ExitCode} for {eventName}.",
|
||||
execution.Detail,
|
||||
null);
|
||||
null,
|
||||
execution.Detail,
|
||||
process.ExitCode == 0 ? ScriptFailureKind.None : ScriptFailureKind.ExitCode);
|
||||
outcomeLevel = process.ExitCode == 0 ? SystemLogLevel.Info : SystemLogLevel.Warning;
|
||||
}
|
||||
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
|
||||
@@ -305,7 +418,9 @@ public sealed class EventScriptService : IEventScriptService
|
||||
false,
|
||||
$"Event script timed out for {eventName}.",
|
||||
execution.Detail,
|
||||
null);
|
||||
null,
|
||||
execution.Detail,
|
||||
ScriptFailureKind.Timeout);
|
||||
outcomeLevel = SystemLogLevel.Warning;
|
||||
}
|
||||
catch (Exception ex)
|
||||
@@ -315,7 +430,9 @@ public sealed class EventScriptService : IEventScriptService
|
||||
false,
|
||||
$"Event script failed for {eventName}.",
|
||||
ex.ToString(),
|
||||
null);
|
||||
null,
|
||||
execution.Detail,
|
||||
ScriptFailureKind.Exception);
|
||||
outcomeLevel = SystemLogLevel.Warning;
|
||||
}
|
||||
finally
|
||||
@@ -344,6 +461,31 @@ public sealed class EventScriptService : IEventScriptService
|
||||
return outcome;
|
||||
}
|
||||
|
||||
private async Task NotifyRetryExhaustedAsync(
|
||||
string eventName,
|
||||
int totalAttempts,
|
||||
ScriptExecutionOutcome outcome,
|
||||
LiveRoom? liveRoom,
|
||||
RecordTask? recordTask,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var detail = BuildRetryFailureNotificationDetail(eventName, totalAttempts, outcome);
|
||||
await _emailNotificationService.SendExceptionAsync(
|
||||
"EventScript",
|
||||
"Event script failed after retries.",
|
||||
detail,
|
||||
liveRoom,
|
||||
recordTask,
|
||||
cancellationToken);
|
||||
await _webhookNotificationService.SendExceptionAsync(
|
||||
"EventScript",
|
||||
"Event script failed after retries.",
|
||||
detail,
|
||||
liveRoom,
|
||||
recordTask,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private static EventScriptExecution? CreateExecution(string scriptMode, string scriptPath, string scriptContent)
|
||||
{
|
||||
if (string.Equals(scriptMode, EventScriptSourceModes.Inline, StringComparison.OrdinalIgnoreCase))
|
||||
@@ -582,6 +724,71 @@ public sealed class EventScriptService : IEventScriptService
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private static string BuildRetryAttemptDetail(
|
||||
string eventName,
|
||||
int nextAttempt,
|
||||
int totalAttempts,
|
||||
int retryDelaySeconds,
|
||||
ScriptExecutionOutcome outcome)
|
||||
{
|
||||
var builder = new StringBuilder();
|
||||
builder.AppendLine($"Event: {eventName}");
|
||||
builder.AppendLine($"Next attempt: {nextAttempt}/{totalAttempts}");
|
||||
builder.AppendLine($"Retry delay: {Math.Clamp(retryDelaySeconds, 0, 3600)} second(s)");
|
||||
builder.AppendLine($"Last result: {outcome.Message}");
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(outcome.ExecutionTarget))
|
||||
{
|
||||
builder.AppendLine($"Script: {outcome.ExecutionTarget}");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(outcome.Detail) &&
|
||||
!string.Equals(outcome.Detail, outcome.ExecutionTarget, StringComparison.Ordinal))
|
||||
{
|
||||
builder.AppendLine();
|
||||
builder.AppendLine("Detail:");
|
||||
builder.AppendLine(outcome.Detail);
|
||||
}
|
||||
|
||||
return TruncateSystemLogDetail(builder.ToString().Trim());
|
||||
}
|
||||
|
||||
private static string BuildRetryFailureNotificationDetail(
|
||||
string eventName,
|
||||
int totalAttempts,
|
||||
ScriptExecutionOutcome outcome)
|
||||
{
|
||||
var builder = new StringBuilder();
|
||||
builder.AppendLine($"Event: {eventName}");
|
||||
builder.AppendLine($"Attempts: {totalAttempts}");
|
||||
builder.AppendLine($"Last result: {outcome.Message}");
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(outcome.ExecutionTarget))
|
||||
{
|
||||
builder.AppendLine($"Script: {outcome.ExecutionTarget}");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(outcome.Detail) &&
|
||||
!string.Equals(outcome.Detail, outcome.ExecutionTarget, StringComparison.Ordinal))
|
||||
{
|
||||
builder.AppendLine();
|
||||
builder.AppendLine("Detail:");
|
||||
builder.AppendLine(outcome.Detail);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(outcome.CustomLogOutput))
|
||||
{
|
||||
builder.AppendLine();
|
||||
builder.AppendLine("Custom log output:");
|
||||
builder.AppendLine(outcome.CustomLogOutput);
|
||||
}
|
||||
|
||||
return TruncateSystemLogDetail(builder.ToString().Trim());
|
||||
}
|
||||
|
||||
private static bool IsRetryableFailure(ScriptExecutionOutcome outcome) =>
|
||||
outcome.FailureKind is ScriptFailureKind.ExitCode or ScriptFailureKind.Timeout or ScriptFailureKind.Exception;
|
||||
|
||||
private static string CreateScriptLogPath()
|
||||
{
|
||||
return Path.Combine(
|
||||
@@ -628,11 +835,23 @@ public sealed class EventScriptService : IEventScriptService
|
||||
}
|
||||
}
|
||||
|
||||
private enum ScriptFailureKind
|
||||
{
|
||||
None,
|
||||
MissingConfiguration,
|
||||
MissingScript,
|
||||
ExitCode,
|
||||
Timeout,
|
||||
Exception
|
||||
}
|
||||
|
||||
private sealed record EventScriptExecution(ProcessStartInfo? StartInfo, string Detail, bool IsMissing = false);
|
||||
|
||||
private sealed record ScriptExecutionOutcome(
|
||||
bool Success,
|
||||
string Message,
|
||||
string? Detail,
|
||||
string? CustomLogOutput);
|
||||
string? CustomLogOutput,
|
||||
string? ExecutionTarget,
|
||||
ScriptFailureKind FailureKind);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Text;
|
||||
using LiveRecorder.Application.Common;
|
||||
using LiveRecorder.Application.Abstractions.Logging;
|
||||
using LiveRecorder.Application.Abstractions.Notifications;
|
||||
@@ -360,6 +361,9 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
|
||||
dbContext,
|
||||
ffmpegService,
|
||||
logService,
|
||||
emailNotificationService,
|
||||
webhookNotificationService,
|
||||
liveRoom,
|
||||
liveRoom.Id,
|
||||
pauseCheck.Message,
|
||||
cancellationToken);
|
||||
@@ -589,6 +593,9 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
|
||||
LiveRecorderDbContext dbContext,
|
||||
IFfmpegService ffmpegService,
|
||||
ISystemLogService logService,
|
||||
IEmailNotificationService emailNotificationService,
|
||||
IWebhookNotificationService webhookNotificationService,
|
||||
Domain.Entities.LiveRoom liveRoom,
|
||||
Guid liveRoomId,
|
||||
string detail,
|
||||
CancellationToken cancellationToken)
|
||||
@@ -600,8 +607,13 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
|
||||
item.Status == RecordSessionStatus.Stopping))
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
var affectedSessionCount = 0;
|
||||
var forcedStopCount = 0;
|
||||
|
||||
foreach (var activeSession in activeSessions.OrderBy(static item => item.CreatedAt))
|
||||
{
|
||||
affectedSessionCount++;
|
||||
|
||||
await logService.WriteAsync(
|
||||
SystemLogLevel.Warning,
|
||||
"Storage",
|
||||
@@ -628,12 +640,32 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
|
||||
liveRoomId,
|
||||
activeSession.Id,
|
||||
cancellationToken: cancellationToken);
|
||||
forcedStopCount++;
|
||||
await ffmpegService.KillAndWaitAsync(activeSession.Id, OfflineForcedStopTimeout, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
await ffmpegService.TryReconcileInactiveSessionAsync(activeSession.Id, cancellationToken);
|
||||
}
|
||||
|
||||
if (affectedSessionCount <= 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var notificationDetail = BuildLowStorageNotificationDetail(liveRoom, affectedSessionCount, forcedStopCount, detail);
|
||||
await emailNotificationService.SendExceptionAsync(
|
||||
"StorageGuard",
|
||||
"Low storage paused active recording sessions.",
|
||||
notificationDetail,
|
||||
liveRoom,
|
||||
cancellationToken: cancellationToken);
|
||||
await webhookNotificationService.SendExceptionAsync(
|
||||
"StorageGuard",
|
||||
"Low storage paused active recording sessions.",
|
||||
notificationDetail,
|
||||
liveRoom,
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task<IReadOnlyList<Guid>> ReconcileStaleActiveSessionsAsync(
|
||||
@@ -699,6 +731,39 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
|
||||
IsTransientPollingException(exception.InnerException, cancellationToken);
|
||||
}
|
||||
|
||||
private static string BuildLowStorageNotificationDetail(
|
||||
Domain.Entities.LiveRoom liveRoom,
|
||||
int affectedSessionCount,
|
||||
int forcedStopCount,
|
||||
string storageDetail)
|
||||
{
|
||||
var builder = new StringBuilder();
|
||||
builder.AppendLine($"Platform: {liveRoom.Platform}");
|
||||
builder.AppendLine($"Room ID: {liveRoom.RoomId}");
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(liveRoom.AnchorName))
|
||||
{
|
||||
builder.AppendLine($"Anchor: {liveRoom.AnchorName}");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(liveRoom.Title))
|
||||
{
|
||||
builder.AppendLine($"Title: {liveRoom.Title}");
|
||||
}
|
||||
|
||||
builder.AppendLine($"Affected sessions: {affectedSessionCount}");
|
||||
builder.AppendLine($"Forced stop attempts: {forcedStopCount}");
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(storageDetail))
|
||||
{
|
||||
builder.AppendLine();
|
||||
builder.AppendLine("Storage detail:");
|
||||
builder.AppendLine(storageDetail);
|
||||
}
|
||||
|
||||
return builder.ToString().Trim();
|
||||
}
|
||||
|
||||
private static string BuildPollingFailureDetail(Exception exception)
|
||||
{
|
||||
var root = exception.GetBaseException();
|
||||
|
||||
Reference in New Issue
Block a user