feat: add storage and script failure notifications
This commit is contained in:
@@ -61,9 +61,17 @@
|
|||||||
--el-border-color-light: var(--border-subtle);
|
--el-border-color-light: var(--border-subtle);
|
||||||
--el-border-radius-base: var(--radius-sm);
|
--el-border-radius-base: var(--radius-sm);
|
||||||
--el-bg-color: transparent;
|
--el-bg-color: transparent;
|
||||||
|
--el-bg-color-page: var(--bg-base);
|
||||||
|
--el-bg-color-overlay: var(--surface);
|
||||||
|
--el-fill-color: var(--surface-muted);
|
||||||
--el-fill-color-blank: var(--surface);
|
--el-fill-color-blank: var(--surface);
|
||||||
--el-fill-color-light: var(--surface-muted);
|
--el-fill-color-light: var(--surface-muted);
|
||||||
--el-fill-color-lighter: var(--surface-strong);
|
--el-fill-color-lighter: var(--surface-strong);
|
||||||
|
--el-fill-color-dark: var(--surface-raised);
|
||||||
|
--el-fill-color-darker: var(--surface-strong);
|
||||||
|
--el-disabled-bg-color: var(--surface-muted);
|
||||||
|
--el-disabled-text-color: var(--text-soft);
|
||||||
|
--el-text-color-placeholder: var(--text-soft);
|
||||||
--el-mask-color: rgba(15, 23, 42, 0.54);
|
--el-mask-color: rgba(15, 23, 42, 0.54);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -123,9 +131,17 @@ html[data-theme="dark"] {
|
|||||||
--el-text-color-secondary: var(--text-muted);
|
--el-text-color-secondary: var(--text-muted);
|
||||||
--el-border-color: var(--border-base);
|
--el-border-color: var(--border-base);
|
||||||
--el-border-color-light: var(--border-subtle);
|
--el-border-color-light: var(--border-subtle);
|
||||||
|
--el-bg-color-page: var(--bg-base);
|
||||||
|
--el-bg-color-overlay: var(--surface);
|
||||||
|
--el-fill-color: var(--surface-muted);
|
||||||
--el-fill-color-blank: var(--surface);
|
--el-fill-color-blank: var(--surface);
|
||||||
--el-fill-color-light: var(--surface-muted);
|
--el-fill-color-light: var(--surface-muted);
|
||||||
--el-fill-color-lighter: var(--surface-strong);
|
--el-fill-color-lighter: var(--surface-strong);
|
||||||
|
--el-fill-color-dark: var(--surface-raised);
|
||||||
|
--el-fill-color-darker: var(--surface-strong);
|
||||||
|
--el-disabled-bg-color: var(--surface-muted);
|
||||||
|
--el-disabled-text-color: var(--text-soft);
|
||||||
|
--el-text-color-placeholder: var(--text-soft);
|
||||||
--el-mask-color: rgba(3, 7, 14, 0.72);
|
--el-mask-color: rgba(3, 7, 14, 0.72);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -255,6 +271,7 @@ select:focus-visible {
|
|||||||
.surface-card {
|
.surface-card {
|
||||||
position: relative;
|
position: relative;
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
|
--el-card-bg-color: transparent;
|
||||||
border-radius: var(--radius-md);
|
border-radius: var(--radius-md);
|
||||||
border: 1px solid var(--border-subtle);
|
border: 1px solid var(--border-subtle);
|
||||||
background:
|
background:
|
||||||
@@ -573,9 +590,33 @@ html[data-theme="dark"] .stat-card {
|
|||||||
|
|
||||||
.el-card {
|
.el-card {
|
||||||
--el-card-border-color: transparent;
|
--el-card-border-color: transparent;
|
||||||
|
--el-card-bg-color: transparent;
|
||||||
background: transparent;
|
background: transparent;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.el-tabs--border-card {
|
||||||
|
border-color: var(--border-subtle);
|
||||||
|
background: var(--surface);
|
||||||
|
}
|
||||||
|
|
||||||
|
.el-tabs--border-card > .el-tabs__content {
|
||||||
|
background: transparent;
|
||||||
|
color: var(--text-primary);
|
||||||
|
}
|
||||||
|
|
||||||
|
.el-tabs--border-card > .el-tabs__header {
|
||||||
|
background: transparent;
|
||||||
|
}
|
||||||
|
|
||||||
|
.el-tabs--border-card > .el-tabs__header .el-tabs__item {
|
||||||
|
color: var(--text-secondary);
|
||||||
|
}
|
||||||
|
|
||||||
|
.el-tabs--border-card > .el-tabs__header .el-tabs__item.is-active {
|
||||||
|
color: var(--text-primary);
|
||||||
|
background: var(--surface-raised);
|
||||||
|
}
|
||||||
|
|
||||||
.el-button {
|
.el-button {
|
||||||
min-height: var(--control-height);
|
min-height: var(--control-height);
|
||||||
padding: 0 15px;
|
padding: 0 15px;
|
||||||
@@ -802,6 +843,17 @@ html[data-theme="dark"] .stat-card {
|
|||||||
box-shadow: var(--shadow-float);
|
box-shadow: var(--shadow-float);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.el-message-box,
|
||||||
|
.el-popover.el-popper,
|
||||||
|
.el-select__popper.el-popper,
|
||||||
|
.el-picker__popper.el-popper,
|
||||||
|
.el-dropdown__popper.el-popper .el-dropdown-menu {
|
||||||
|
border-color: var(--border-subtle);
|
||||||
|
background: linear-gradient(180deg, var(--surface-raised), var(--surface));
|
||||||
|
color: var(--text-primary);
|
||||||
|
box-shadow: var(--shadow-float);
|
||||||
|
}
|
||||||
|
|
||||||
.el-dialog__header {
|
.el-dialog__header {
|
||||||
margin: 0;
|
margin: 0;
|
||||||
padding: 22px 24px 10px;
|
padding: 22px 24px 10px;
|
||||||
|
|||||||
@@ -458,6 +458,8 @@ export interface SystemSettings {
|
|||||||
segmentCompletedScriptPath: string;
|
segmentCompletedScriptPath: string;
|
||||||
segmentCompletedScriptContent: string;
|
segmentCompletedScriptContent: string;
|
||||||
eventScriptTimeoutSeconds: number;
|
eventScriptTimeoutSeconds: number;
|
||||||
|
eventScriptRetryAttempts: number;
|
||||||
|
eventScriptRetryDelaySeconds: number;
|
||||||
enableEmailNotification: boolean;
|
enableEmailNotification: boolean;
|
||||||
emailSmtpHost: string;
|
emailSmtpHost: string;
|
||||||
emailSmtpPort: number;
|
emailSmtpPort: number;
|
||||||
|
|||||||
@@ -168,6 +168,8 @@ const form = reactive<SettingsFormModel>({
|
|||||||
segmentCompletedScriptPath: "",
|
segmentCompletedScriptPath: "",
|
||||||
segmentCompletedScriptContent: "",
|
segmentCompletedScriptContent: "",
|
||||||
eventScriptTimeoutSeconds: 60,
|
eventScriptTimeoutSeconds: 60,
|
||||||
|
eventScriptRetryAttempts: 3,
|
||||||
|
eventScriptRetryDelaySeconds: 10,
|
||||||
enableEmailNotification: false,
|
enableEmailNotification: false,
|
||||||
emailSmtpHost: "",
|
emailSmtpHost: "",
|
||||||
emailSmtpPort: 587,
|
emailSmtpPort: 587,
|
||||||
@@ -1480,16 +1482,26 @@ watch(
|
|||||||
|
|
||||||
<el-form label-position="top">
|
<el-form label-position="top">
|
||||||
<el-row :gutter="16">
|
<el-row :gutter="16">
|
||||||
<el-col :span="8">
|
<el-col :span="6">
|
||||||
<el-form-item label="启用事件脚本">
|
<el-form-item label="启用事件脚本">
|
||||||
<el-switch v-model="form.enableEventScripts" />
|
<el-switch v-model="form.enableEventScripts" />
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
</el-col>
|
</el-col>
|
||||||
<el-col :span="8">
|
<el-col :span="6">
|
||||||
<el-form-item label="脚本超时(秒)">
|
<el-form-item label="脚本超时(秒)">
|
||||||
<el-input-number v-model="form.eventScriptTimeoutSeconds" :min="1" :max="3600" />
|
<el-input-number v-model="form.eventScriptTimeoutSeconds" :min="1" :max="3600" />
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
</el-col>
|
</el-col>
|
||||||
|
<el-col :span="5">
|
||||||
|
<el-form-item label="重试次数">
|
||||||
|
<el-input-number v-model="form.eventScriptRetryAttempts" :min="0" :max="20" />
|
||||||
|
</el-form-item>
|
||||||
|
</el-col>
|
||||||
|
<el-col :span="6">
|
||||||
|
<el-form-item label="重试间隔(秒)">
|
||||||
|
<el-input-number v-model="form.eventScriptRetryDelaySeconds" :min="0" :max="3600" />
|
||||||
|
</el-form-item>
|
||||||
|
</el-col>
|
||||||
</el-row>
|
</el-row>
|
||||||
</el-form>
|
</el-form>
|
||||||
|
|
||||||
@@ -1676,6 +1688,10 @@ watch(
|
|||||||
The official Docker image includes <code>curl</code> and <code>jq</code> by default. If you run on a host machine or a custom image, rely on the commands available in that environment.
|
The official Docker image includes <code>curl</code> and <code>jq</code> by default. If you run on a host machine or a custom image, rely on the commands available in that environment.
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="event-script-help__intro">
|
||||||
|
Set <code>Retry attempts</code> to <code>0</code> to disable automatic retries. Retry exhaustion failures are sent through the existing exception notification channel.
|
||||||
|
</div>
|
||||||
|
|
||||||
<div class="event-script-example">
|
<div class="event-script-example">
|
||||||
<div class="event-script-example__label">Environment variable examples</div>
|
<div class="event-script-example__label">Environment variable examples</div>
|
||||||
<div class="event-script-example__grid">
|
<div class="event-script-example__grid">
|
||||||
@@ -1699,7 +1715,8 @@ watch(
|
|||||||
|
|
||||||
<el-card class="surface-card settings-card settings-grid__full" shadow="never">
|
<el-card class="surface-card settings-card settings-grid__full" shadow="never">
|
||||||
<h3 class="section-title">Webhook 閫氱煡</h3>
|
<h3 class="section-title">Webhook 閫氱煡</h3>
|
||||||
<p class="section-subtitle">Send fixed JSON POST payloads with custom headers. Currently only live-started and exception events are sent.</p>
|
<h3 class="section-title">Webhook 通知</h3>
|
||||||
|
<p class="section-subtitle">Send fixed JSON POST payloads with custom headers. Exception notifications also cover low-storage stop events and event-script failures after retries are exhausted.</p>
|
||||||
|
|
||||||
<el-form label-position="top">
|
<el-form label-position="top">
|
||||||
<el-row :gutter="16">
|
<el-row :gutter="16">
|
||||||
@@ -1772,7 +1789,8 @@ watch(
|
|||||||
|
|
||||||
<el-card class="surface-card settings-card settings-grid__full" shadow="never">
|
<el-card class="surface-card settings-card settings-grid__full" shadow="never">
|
||||||
<h3 class="section-title">閭欢閫氱煡</h3>
|
<h3 class="section-title">閭欢閫氱煡</h3>
|
||||||
<p class="section-subtitle">Configure SMTP plus HTML templates for live-started and exception alerts. Tests use the current unsaved form values directly.</p>
|
<h3 class="section-title">邮件通知</h3>
|
||||||
|
<p class="section-subtitle">Configure SMTP plus HTML templates for live-started and exception alerts. Exception notifications also cover low-storage stop events and event-script failures after retries are exhausted.</p>
|
||||||
|
|
||||||
<el-form label-position="top">
|
<el-form label-position="top">
|
||||||
<el-row :gutter="16">
|
<el-row :gutter="16">
|
||||||
|
|||||||
@@ -177,6 +177,10 @@ public sealed class SystemSettingsDto
|
|||||||
|
|
||||||
public int EventScriptTimeoutSeconds { get; set; } = 60;
|
public int EventScriptTimeoutSeconds { get; set; } = 60;
|
||||||
|
|
||||||
|
public int EventScriptRetryAttempts { get; set; } = 3;
|
||||||
|
|
||||||
|
public int EventScriptRetryDelaySeconds { get; set; } = 10;
|
||||||
|
|
||||||
public bool EnableRetentionCleanup { get; set; } = false;
|
public bool EnableRetentionCleanup { get; set; } = false;
|
||||||
|
|
||||||
public int RetentionDays { get; set; } = 30;
|
public int RetentionDays { get; set; } = 30;
|
||||||
@@ -446,6 +450,10 @@ public sealed class UpdateSystemSettingsRequest
|
|||||||
|
|
||||||
public int EventScriptTimeoutSeconds { get; set; } = 60;
|
public int EventScriptTimeoutSeconds { get; set; } = 60;
|
||||||
|
|
||||||
|
public int EventScriptRetryAttempts { get; set; } = 3;
|
||||||
|
|
||||||
|
public int EventScriptRetryDelaySeconds { get; set; } = 10;
|
||||||
|
|
||||||
public bool EnableRetentionCleanup { get; set; } = false;
|
public bool EnableRetentionCleanup { get; set; } = false;
|
||||||
|
|
||||||
public int RetentionDays { get; set; } = 30;
|
public int RetentionDays { get; set; } = 30;
|
||||||
|
|||||||
@@ -71,6 +71,8 @@ public sealed class SystemSettingsService : ISystemSettingsService
|
|||||||
private const string SegmentCompletedScriptPathKey = "event_scripts.segment_completed.path";
|
private const string SegmentCompletedScriptPathKey = "event_scripts.segment_completed.path";
|
||||||
private const string SegmentCompletedScriptContentKey = "event_scripts.segment_completed.content";
|
private const string SegmentCompletedScriptContentKey = "event_scripts.segment_completed.content";
|
||||||
private const string EventScriptTimeoutSecondsKey = "event_scripts.timeout_seconds";
|
private const string EventScriptTimeoutSecondsKey = "event_scripts.timeout_seconds";
|
||||||
|
private const string EventScriptRetryAttemptsKey = "event_scripts.retry_attempts";
|
||||||
|
private const string EventScriptRetryDelaySecondsKey = "event_scripts.retry_delay_seconds";
|
||||||
private const string EnableRetentionCleanupKey = "retention.cleanup.enabled";
|
private const string EnableRetentionCleanupKey = "retention.cleanup.enabled";
|
||||||
private const string RetentionDaysKey = "retention.cleanup.days";
|
private const string RetentionDaysKey = "retention.cleanup.days";
|
||||||
private const string RetentionDeleteFilesKey = "retention.cleanup.delete_files";
|
private const string RetentionDeleteFilesKey = "retention.cleanup.delete_files";
|
||||||
@@ -202,6 +204,8 @@ public sealed class SystemSettingsService : ISystemSettingsService
|
|||||||
SegmentCompletedScriptPath = GetValue(lookup, SegmentCompletedScriptPathKey, string.Empty),
|
SegmentCompletedScriptPath = GetValue(lookup, SegmentCompletedScriptPathKey, string.Empty),
|
||||||
SegmentCompletedScriptContent = GetValue(lookup, SegmentCompletedScriptContentKey, string.Empty),
|
SegmentCompletedScriptContent = GetValue(lookup, SegmentCompletedScriptContentKey, string.Empty),
|
||||||
EventScriptTimeoutSeconds = GetIntValue(lookup, EventScriptTimeoutSecondsKey, 60, 1, 3600),
|
EventScriptTimeoutSeconds = GetIntValue(lookup, EventScriptTimeoutSecondsKey, 60, 1, 3600),
|
||||||
|
EventScriptRetryAttempts = GetIntValue(lookup, EventScriptRetryAttemptsKey, 3, 0, 20),
|
||||||
|
EventScriptRetryDelaySeconds = GetIntValue(lookup, EventScriptRetryDelaySecondsKey, 10, 0, 3600),
|
||||||
EnableRetentionCleanup = bool.TryParse(GetValue(lookup, EnableRetentionCleanupKey, "false"), out var enableRetentionCleanup) && enableRetentionCleanup,
|
EnableRetentionCleanup = bool.TryParse(GetValue(lookup, EnableRetentionCleanupKey, "false"), out var enableRetentionCleanup) && enableRetentionCleanup,
|
||||||
RetentionDays = GetIntValue(lookup, RetentionDaysKey, 30, 1, 3650),
|
RetentionDays = GetIntValue(lookup, RetentionDaysKey, 30, 1, 3650),
|
||||||
RetentionDeleteFiles = bool.TryParse(GetValue(lookup, RetentionDeleteFilesKey, "false"), out var retentionDeleteFiles) && retentionDeleteFiles,
|
RetentionDeleteFiles = bool.TryParse(GetValue(lookup, RetentionDeleteFilesKey, "false"), out var retentionDeleteFiles) && retentionDeleteFiles,
|
||||||
@@ -352,6 +356,8 @@ public sealed class SystemSettingsService : ISystemSettingsService
|
|||||||
await UpsertAsync(SegmentCompletedScriptPathKey, request.SegmentCompletedScriptPath.Trim(), now, cancellationToken);
|
await UpsertAsync(SegmentCompletedScriptPathKey, request.SegmentCompletedScriptPath.Trim(), now, cancellationToken);
|
||||||
await UpsertAsync(SegmentCompletedScriptContentKey, request.SegmentCompletedScriptContent, now, cancellationToken);
|
await UpsertAsync(SegmentCompletedScriptContentKey, request.SegmentCompletedScriptContent, now, cancellationToken);
|
||||||
await UpsertAsync(EventScriptTimeoutSecondsKey, Math.Clamp(request.EventScriptTimeoutSeconds, 1, 3600).ToString(), now, cancellationToken);
|
await UpsertAsync(EventScriptTimeoutSecondsKey, Math.Clamp(request.EventScriptTimeoutSeconds, 1, 3600).ToString(), now, cancellationToken);
|
||||||
|
await UpsertAsync(EventScriptRetryAttemptsKey, Math.Clamp(request.EventScriptRetryAttempts, 0, 20).ToString(), now, cancellationToken);
|
||||||
|
await UpsertAsync(EventScriptRetryDelaySecondsKey, Math.Clamp(request.EventScriptRetryDelaySeconds, 0, 3600).ToString(), now, cancellationToken);
|
||||||
await UpsertAsync(EnableRetentionCleanupKey, request.EnableRetentionCleanup.ToString(), now, cancellationToken);
|
await UpsertAsync(EnableRetentionCleanupKey, request.EnableRetentionCleanup.ToString(), now, cancellationToken);
|
||||||
await UpsertAsync(RetentionDaysKey, Math.Clamp(request.RetentionDays, 1, 3650).ToString(), now, cancellationToken);
|
await UpsertAsync(RetentionDaysKey, Math.Clamp(request.RetentionDays, 1, 3650).ToString(), now, cancellationToken);
|
||||||
await UpsertAsync(RetentionDeleteFilesKey, request.RetentionDeleteFiles.ToString(), now, cancellationToken);
|
await UpsertAsync(RetentionDeleteFilesKey, request.RetentionDeleteFiles.ToString(), now, cancellationToken);
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
|
using System.Text;
|
||||||
using LiveRecorder.Application.Abstractions.Logging;
|
using LiveRecorder.Application.Abstractions.Logging;
|
||||||
|
using LiveRecorder.Application.Abstractions.Notifications;
|
||||||
using LiveRecorder.Application.Abstractions.Scripting;
|
using LiveRecorder.Application.Abstractions.Scripting;
|
||||||
using LiveRecorder.Application.Abstractions.Settings;
|
using LiveRecorder.Application.Abstractions.Settings;
|
||||||
using LiveRecorder.Application.Common;
|
using LiveRecorder.Application.Common;
|
||||||
@@ -18,15 +20,21 @@ public sealed class EventScriptService : IEventScriptService
|
|||||||
|
|
||||||
private readonly ISystemSettingsService _settingsService;
|
private readonly ISystemSettingsService _settingsService;
|
||||||
private readonly ISystemLogService _systemLogService;
|
private readonly ISystemLogService _systemLogService;
|
||||||
|
private readonly IEmailNotificationService _emailNotificationService;
|
||||||
|
private readonly IWebhookNotificationService _webhookNotificationService;
|
||||||
private readonly ILogger<EventScriptService> _logger;
|
private readonly ILogger<EventScriptService> _logger;
|
||||||
|
|
||||||
public EventScriptService(
|
public EventScriptService(
|
||||||
ISystemSettingsService settingsService,
|
ISystemSettingsService settingsService,
|
||||||
ISystemLogService systemLogService,
|
ISystemLogService systemLogService,
|
||||||
|
IEmailNotificationService emailNotificationService,
|
||||||
|
IWebhookNotificationService webhookNotificationService,
|
||||||
ILogger<EventScriptService> logger)
|
ILogger<EventScriptService> logger)
|
||||||
{
|
{
|
||||||
_settingsService = settingsService;
|
_settingsService = settingsService;
|
||||||
_systemLogService = systemLogService;
|
_systemLogService = systemLogService;
|
||||||
|
_emailNotificationService = emailNotificationService;
|
||||||
|
_webhookNotificationService = webhookNotificationService;
|
||||||
_logger = logger;
|
_logger = logger;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -44,8 +52,12 @@ public sealed class EventScriptService : IEventScriptService
|
|||||||
settings.LiveStartedScriptPath,
|
settings.LiveStartedScriptPath,
|
||||||
settings.LiveStartedScriptContent,
|
settings.LiveStartedScriptContent,
|
||||||
settings.EventScriptTimeoutSeconds,
|
settings.EventScriptTimeoutSeconds,
|
||||||
|
settings.EventScriptRetryAttempts,
|
||||||
|
settings.EventScriptRetryDelaySeconds,
|
||||||
"live_started",
|
"live_started",
|
||||||
environment,
|
environment,
|
||||||
|
liveRoom,
|
||||||
|
recordTask: null,
|
||||||
"Script",
|
"Script",
|
||||||
liveRoom.Id,
|
liveRoom.Id,
|
||||||
recordSessionId: null,
|
recordSessionId: null,
|
||||||
@@ -67,8 +79,12 @@ public sealed class EventScriptService : IEventScriptService
|
|||||||
settings.LiveEndedScriptPath,
|
settings.LiveEndedScriptPath,
|
||||||
settings.LiveEndedScriptContent,
|
settings.LiveEndedScriptContent,
|
||||||
settings.EventScriptTimeoutSeconds,
|
settings.EventScriptTimeoutSeconds,
|
||||||
|
settings.EventScriptRetryAttempts,
|
||||||
|
settings.EventScriptRetryDelaySeconds,
|
||||||
"live_ended",
|
"live_ended",
|
||||||
environment,
|
environment,
|
||||||
|
liveRoom,
|
||||||
|
recordTask: null,
|
||||||
"Script",
|
"Script",
|
||||||
liveRoom.Id,
|
liveRoom.Id,
|
||||||
recordSessionId: null,
|
recordSessionId: null,
|
||||||
@@ -105,8 +121,12 @@ public sealed class EventScriptService : IEventScriptService
|
|||||||
settings.SegmentCompletedScriptPath,
|
settings.SegmentCompletedScriptPath,
|
||||||
settings.SegmentCompletedScriptContent,
|
settings.SegmentCompletedScriptContent,
|
||||||
settings.EventScriptTimeoutSeconds,
|
settings.EventScriptTimeoutSeconds,
|
||||||
|
settings.EventScriptRetryAttempts,
|
||||||
|
settings.EventScriptRetryDelaySeconds,
|
||||||
"segment_completed",
|
"segment_completed",
|
||||||
environment,
|
environment,
|
||||||
|
liveRoom,
|
||||||
|
recordTask,
|
||||||
"Script",
|
"Script",
|
||||||
liveRoom?.Id ?? recordSession.LiveRoomId,
|
liveRoom?.Id ?? recordSession.LiveRoomId,
|
||||||
recordSession.Id,
|
recordSession.Id,
|
||||||
@@ -176,8 +196,12 @@ public sealed class EventScriptService : IEventScriptService
|
|||||||
string scriptPath,
|
string scriptPath,
|
||||||
string scriptContent,
|
string scriptContent,
|
||||||
int timeoutSeconds,
|
int timeoutSeconds,
|
||||||
|
int retryAttempts,
|
||||||
|
int retryDelaySeconds,
|
||||||
string eventName,
|
string eventName,
|
||||||
IReadOnlyDictionary<string, string> environment,
|
IReadOnlyDictionary<string, string> environment,
|
||||||
|
LiveRoom? liveRoom,
|
||||||
|
RecordTask? recordTask,
|
||||||
string logCategory,
|
string logCategory,
|
||||||
Guid? liveRoomId,
|
Guid? liveRoomId,
|
||||||
Guid? recordSessionId,
|
Guid? recordSessionId,
|
||||||
@@ -189,6 +213,43 @@ public sealed class EventScriptService : IEventScriptService
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var outcome = await ExecuteWithRetryAsync(
|
||||||
|
scriptMode,
|
||||||
|
scriptPath,
|
||||||
|
scriptContent,
|
||||||
|
timeoutSeconds,
|
||||||
|
retryAttempts,
|
||||||
|
retryDelaySeconds,
|
||||||
|
eventName,
|
||||||
|
environment,
|
||||||
|
liveRoom,
|
||||||
|
recordTask,
|
||||||
|
logCategory,
|
||||||
|
liveRoomId,
|
||||||
|
recordSessionId,
|
||||||
|
recordTaskId,
|
||||||
|
cancellationToken);
|
||||||
|
|
||||||
|
return MapOutcome(outcome);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<ScriptExecutionOutcome> ExecuteWithRetryAsync(
|
||||||
|
string scriptMode,
|
||||||
|
string scriptPath,
|
||||||
|
string scriptContent,
|
||||||
|
int timeoutSeconds,
|
||||||
|
int retryAttempts,
|
||||||
|
int retryDelaySeconds,
|
||||||
|
string eventName,
|
||||||
|
IReadOnlyDictionary<string, string> environment,
|
||||||
|
LiveRoom? liveRoom,
|
||||||
|
RecordTask? recordTask,
|
||||||
|
string logCategory,
|
||||||
|
Guid? liveRoomId,
|
||||||
|
Guid? recordSessionId,
|
||||||
|
Guid? recordTaskId,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
var outcome = await ExecuteAsync(
|
var outcome = await ExecuteAsync(
|
||||||
scriptMode,
|
scriptMode,
|
||||||
scriptPath,
|
scriptPath,
|
||||||
@@ -202,7 +263,53 @@ public sealed class EventScriptService : IEventScriptService
|
|||||||
recordTaskId,
|
recordTaskId,
|
||||||
cancellationToken);
|
cancellationToken);
|
||||||
|
|
||||||
return MapOutcome(outcome);
|
if (outcome.Success || !IsRetryableFailure(outcome))
|
||||||
|
{
|
||||||
|
return outcome;
|
||||||
|
}
|
||||||
|
|
||||||
|
var maxRetryAttempts = Math.Clamp(retryAttempts, 0, 20);
|
||||||
|
var totalAttempts = 1;
|
||||||
|
|
||||||
|
for (var retryIndex = 1; retryIndex <= maxRetryAttempts; retryIndex++)
|
||||||
|
{
|
||||||
|
await _systemLogService.WriteAsync(
|
||||||
|
SystemLogLevel.Warning,
|
||||||
|
logCategory,
|
||||||
|
$"Event script retry {retryIndex} of {maxRetryAttempts} scheduled for {eventName}.",
|
||||||
|
BuildRetryAttemptDetail(eventName, retryIndex + 1, maxRetryAttempts + 1, retryDelaySeconds, outcome),
|
||||||
|
liveRoomId,
|
||||||
|
recordSessionId,
|
||||||
|
recordTaskId,
|
||||||
|
cancellationToken);
|
||||||
|
|
||||||
|
if (retryDelaySeconds > 0)
|
||||||
|
{
|
||||||
|
await Task.Delay(TimeSpan.FromSeconds(Math.Clamp(retryDelaySeconds, 0, 3600)), cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
outcome = await ExecuteAsync(
|
||||||
|
scriptMode,
|
||||||
|
scriptPath,
|
||||||
|
scriptContent,
|
||||||
|
timeoutSeconds,
|
||||||
|
eventName,
|
||||||
|
environment,
|
||||||
|
logCategory,
|
||||||
|
liveRoomId,
|
||||||
|
recordSessionId,
|
||||||
|
recordTaskId,
|
||||||
|
cancellationToken);
|
||||||
|
totalAttempts++;
|
||||||
|
|
||||||
|
if (outcome.Success || !IsRetryableFailure(outcome))
|
||||||
|
{
|
||||||
|
return outcome;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await NotifyRetryExhaustedAsync(eventName, totalAttempts, outcome, liveRoom, recordTask, cancellationToken);
|
||||||
|
return outcome;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task<ScriptExecutionOutcome> ExecuteAsync(
|
private async Task<ScriptExecutionOutcome> ExecuteAsync(
|
||||||
@@ -225,7 +332,9 @@ public sealed class EventScriptService : IEventScriptService
|
|||||||
false,
|
false,
|
||||||
$"Event script was not configured for {eventName}.",
|
$"Event script was not configured for {eventName}.",
|
||||||
null,
|
null,
|
||||||
null);
|
null,
|
||||||
|
null,
|
||||||
|
ScriptFailureKind.MissingConfiguration);
|
||||||
|
|
||||||
await WriteOutcomeLogAsync(
|
await WriteOutcomeLogAsync(
|
||||||
missingConfiguration,
|
missingConfiguration,
|
||||||
@@ -244,7 +353,9 @@ public sealed class EventScriptService : IEventScriptService
|
|||||||
false,
|
false,
|
||||||
$"Event script was not found for {eventName}.",
|
$"Event script was not found for {eventName}.",
|
||||||
execution.Detail,
|
execution.Detail,
|
||||||
null);
|
null,
|
||||||
|
execution.Detail,
|
||||||
|
ScriptFailureKind.MissingScript);
|
||||||
|
|
||||||
await WriteOutcomeLogAsync(
|
await WriteOutcomeLogAsync(
|
||||||
missingScript,
|
missingScript,
|
||||||
@@ -295,7 +406,9 @@ public sealed class EventScriptService : IEventScriptService
|
|||||||
? $"Event script completed for {eventName}."
|
? $"Event script completed for {eventName}."
|
||||||
: $"Event script exited with code {process.ExitCode} for {eventName}.",
|
: $"Event script exited with code {process.ExitCode} for {eventName}.",
|
||||||
execution.Detail,
|
execution.Detail,
|
||||||
null);
|
null,
|
||||||
|
execution.Detail,
|
||||||
|
process.ExitCode == 0 ? ScriptFailureKind.None : ScriptFailureKind.ExitCode);
|
||||||
outcomeLevel = process.ExitCode == 0 ? SystemLogLevel.Info : SystemLogLevel.Warning;
|
outcomeLevel = process.ExitCode == 0 ? SystemLogLevel.Info : SystemLogLevel.Warning;
|
||||||
}
|
}
|
||||||
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
|
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
|
||||||
@@ -305,7 +418,9 @@ public sealed class EventScriptService : IEventScriptService
|
|||||||
false,
|
false,
|
||||||
$"Event script timed out for {eventName}.",
|
$"Event script timed out for {eventName}.",
|
||||||
execution.Detail,
|
execution.Detail,
|
||||||
null);
|
null,
|
||||||
|
execution.Detail,
|
||||||
|
ScriptFailureKind.Timeout);
|
||||||
outcomeLevel = SystemLogLevel.Warning;
|
outcomeLevel = SystemLogLevel.Warning;
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
@@ -315,7 +430,9 @@ public sealed class EventScriptService : IEventScriptService
|
|||||||
false,
|
false,
|
||||||
$"Event script failed for {eventName}.",
|
$"Event script failed for {eventName}.",
|
||||||
ex.ToString(),
|
ex.ToString(),
|
||||||
null);
|
null,
|
||||||
|
execution.Detail,
|
||||||
|
ScriptFailureKind.Exception);
|
||||||
outcomeLevel = SystemLogLevel.Warning;
|
outcomeLevel = SystemLogLevel.Warning;
|
||||||
}
|
}
|
||||||
finally
|
finally
|
||||||
@@ -344,6 +461,31 @@ public sealed class EventScriptService : IEventScriptService
|
|||||||
return outcome;
|
return outcome;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async Task NotifyRetryExhaustedAsync(
|
||||||
|
string eventName,
|
||||||
|
int totalAttempts,
|
||||||
|
ScriptExecutionOutcome outcome,
|
||||||
|
LiveRoom? liveRoom,
|
||||||
|
RecordTask? recordTask,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var detail = BuildRetryFailureNotificationDetail(eventName, totalAttempts, outcome);
|
||||||
|
await _emailNotificationService.SendExceptionAsync(
|
||||||
|
"EventScript",
|
||||||
|
"Event script failed after retries.",
|
||||||
|
detail,
|
||||||
|
liveRoom,
|
||||||
|
recordTask,
|
||||||
|
cancellationToken);
|
||||||
|
await _webhookNotificationService.SendExceptionAsync(
|
||||||
|
"EventScript",
|
||||||
|
"Event script failed after retries.",
|
||||||
|
detail,
|
||||||
|
liveRoom,
|
||||||
|
recordTask,
|
||||||
|
cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
private static EventScriptExecution? CreateExecution(string scriptMode, string scriptPath, string scriptContent)
|
private static EventScriptExecution? CreateExecution(string scriptMode, string scriptPath, string scriptContent)
|
||||||
{
|
{
|
||||||
if (string.Equals(scriptMode, EventScriptSourceModes.Inline, StringComparison.OrdinalIgnoreCase))
|
if (string.Equals(scriptMode, EventScriptSourceModes.Inline, StringComparison.OrdinalIgnoreCase))
|
||||||
@@ -582,6 +724,71 @@ public sealed class EventScriptService : IEventScriptService
|
|||||||
cancellationToken);
|
cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static string BuildRetryAttemptDetail(
|
||||||
|
string eventName,
|
||||||
|
int nextAttempt,
|
||||||
|
int totalAttempts,
|
||||||
|
int retryDelaySeconds,
|
||||||
|
ScriptExecutionOutcome outcome)
|
||||||
|
{
|
||||||
|
var builder = new StringBuilder();
|
||||||
|
builder.AppendLine($"Event: {eventName}");
|
||||||
|
builder.AppendLine($"Next attempt: {nextAttempt}/{totalAttempts}");
|
||||||
|
builder.AppendLine($"Retry delay: {Math.Clamp(retryDelaySeconds, 0, 3600)} second(s)");
|
||||||
|
builder.AppendLine($"Last result: {outcome.Message}");
|
||||||
|
|
||||||
|
if (!string.IsNullOrWhiteSpace(outcome.ExecutionTarget))
|
||||||
|
{
|
||||||
|
builder.AppendLine($"Script: {outcome.ExecutionTarget}");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!string.IsNullOrWhiteSpace(outcome.Detail) &&
|
||||||
|
!string.Equals(outcome.Detail, outcome.ExecutionTarget, StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
builder.AppendLine();
|
||||||
|
builder.AppendLine("Detail:");
|
||||||
|
builder.AppendLine(outcome.Detail);
|
||||||
|
}
|
||||||
|
|
||||||
|
return TruncateSystemLogDetail(builder.ToString().Trim());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string BuildRetryFailureNotificationDetail(
|
||||||
|
string eventName,
|
||||||
|
int totalAttempts,
|
||||||
|
ScriptExecutionOutcome outcome)
|
||||||
|
{
|
||||||
|
var builder = new StringBuilder();
|
||||||
|
builder.AppendLine($"Event: {eventName}");
|
||||||
|
builder.AppendLine($"Attempts: {totalAttempts}");
|
||||||
|
builder.AppendLine($"Last result: {outcome.Message}");
|
||||||
|
|
||||||
|
if (!string.IsNullOrWhiteSpace(outcome.ExecutionTarget))
|
||||||
|
{
|
||||||
|
builder.AppendLine($"Script: {outcome.ExecutionTarget}");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!string.IsNullOrWhiteSpace(outcome.Detail) &&
|
||||||
|
!string.Equals(outcome.Detail, outcome.ExecutionTarget, StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
builder.AppendLine();
|
||||||
|
builder.AppendLine("Detail:");
|
||||||
|
builder.AppendLine(outcome.Detail);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!string.IsNullOrWhiteSpace(outcome.CustomLogOutput))
|
||||||
|
{
|
||||||
|
builder.AppendLine();
|
||||||
|
builder.AppendLine("Custom log output:");
|
||||||
|
builder.AppendLine(outcome.CustomLogOutput);
|
||||||
|
}
|
||||||
|
|
||||||
|
return TruncateSystemLogDetail(builder.ToString().Trim());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsRetryableFailure(ScriptExecutionOutcome outcome) =>
|
||||||
|
outcome.FailureKind is ScriptFailureKind.ExitCode or ScriptFailureKind.Timeout or ScriptFailureKind.Exception;
|
||||||
|
|
||||||
private static string CreateScriptLogPath()
|
private static string CreateScriptLogPath()
|
||||||
{
|
{
|
||||||
return Path.Combine(
|
return Path.Combine(
|
||||||
@@ -628,11 +835,23 @@ public sealed class EventScriptService : IEventScriptService
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private enum ScriptFailureKind
|
||||||
|
{
|
||||||
|
None,
|
||||||
|
MissingConfiguration,
|
||||||
|
MissingScript,
|
||||||
|
ExitCode,
|
||||||
|
Timeout,
|
||||||
|
Exception
|
||||||
|
}
|
||||||
|
|
||||||
private sealed record EventScriptExecution(ProcessStartInfo? StartInfo, string Detail, bool IsMissing = false);
|
private sealed record EventScriptExecution(ProcessStartInfo? StartInfo, string Detail, bool IsMissing = false);
|
||||||
|
|
||||||
private sealed record ScriptExecutionOutcome(
|
private sealed record ScriptExecutionOutcome(
|
||||||
bool Success,
|
bool Success,
|
||||||
string Message,
|
string Message,
|
||||||
string? Detail,
|
string? Detail,
|
||||||
string? CustomLogOutput);
|
string? CustomLogOutput,
|
||||||
|
string? ExecutionTarget,
|
||||||
|
ScriptFailureKind FailureKind);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
using System.Collections.Concurrent;
|
using System.Collections.Concurrent;
|
||||||
|
using System.Text;
|
||||||
using LiveRecorder.Application.Common;
|
using LiveRecorder.Application.Common;
|
||||||
using LiveRecorder.Application.Abstractions.Logging;
|
using LiveRecorder.Application.Abstractions.Logging;
|
||||||
using LiveRecorder.Application.Abstractions.Notifications;
|
using LiveRecorder.Application.Abstractions.Notifications;
|
||||||
@@ -360,6 +361,9 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
|
|||||||
dbContext,
|
dbContext,
|
||||||
ffmpegService,
|
ffmpegService,
|
||||||
logService,
|
logService,
|
||||||
|
emailNotificationService,
|
||||||
|
webhookNotificationService,
|
||||||
|
liveRoom,
|
||||||
liveRoom.Id,
|
liveRoom.Id,
|
||||||
pauseCheck.Message,
|
pauseCheck.Message,
|
||||||
cancellationToken);
|
cancellationToken);
|
||||||
@@ -589,6 +593,9 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
|
|||||||
LiveRecorderDbContext dbContext,
|
LiveRecorderDbContext dbContext,
|
||||||
IFfmpegService ffmpegService,
|
IFfmpegService ffmpegService,
|
||||||
ISystemLogService logService,
|
ISystemLogService logService,
|
||||||
|
IEmailNotificationService emailNotificationService,
|
||||||
|
IWebhookNotificationService webhookNotificationService,
|
||||||
|
Domain.Entities.LiveRoom liveRoom,
|
||||||
Guid liveRoomId,
|
Guid liveRoomId,
|
||||||
string detail,
|
string detail,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
@@ -600,8 +607,13 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
|
|||||||
item.Status == RecordSessionStatus.Stopping))
|
item.Status == RecordSessionStatus.Stopping))
|
||||||
.ToListAsync(cancellationToken);
|
.ToListAsync(cancellationToken);
|
||||||
|
|
||||||
|
var affectedSessionCount = 0;
|
||||||
|
var forcedStopCount = 0;
|
||||||
|
|
||||||
foreach (var activeSession in activeSessions.OrderBy(static item => item.CreatedAt))
|
foreach (var activeSession in activeSessions.OrderBy(static item => item.CreatedAt))
|
||||||
{
|
{
|
||||||
|
affectedSessionCount++;
|
||||||
|
|
||||||
await logService.WriteAsync(
|
await logService.WriteAsync(
|
||||||
SystemLogLevel.Warning,
|
SystemLogLevel.Warning,
|
||||||
"Storage",
|
"Storage",
|
||||||
@@ -628,12 +640,32 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
|
|||||||
liveRoomId,
|
liveRoomId,
|
||||||
activeSession.Id,
|
activeSession.Id,
|
||||||
cancellationToken: cancellationToken);
|
cancellationToken: cancellationToken);
|
||||||
|
forcedStopCount++;
|
||||||
await ffmpegService.KillAndWaitAsync(activeSession.Id, OfflineForcedStopTimeout, cancellationToken);
|
await ffmpegService.KillAndWaitAsync(activeSession.Id, OfflineForcedStopTimeout, cancellationToken);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
await ffmpegService.TryReconcileInactiveSessionAsync(activeSession.Id, cancellationToken);
|
await ffmpegService.TryReconcileInactiveSessionAsync(activeSession.Id, cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (affectedSessionCount <= 0)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var notificationDetail = BuildLowStorageNotificationDetail(liveRoom, affectedSessionCount, forcedStopCount, detail);
|
||||||
|
await emailNotificationService.SendExceptionAsync(
|
||||||
|
"StorageGuard",
|
||||||
|
"Low storage paused active recording sessions.",
|
||||||
|
notificationDetail,
|
||||||
|
liveRoom,
|
||||||
|
cancellationToken: cancellationToken);
|
||||||
|
await webhookNotificationService.SendExceptionAsync(
|
||||||
|
"StorageGuard",
|
||||||
|
"Low storage paused active recording sessions.",
|
||||||
|
notificationDetail,
|
||||||
|
liveRoom,
|
||||||
|
cancellationToken: cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static async Task<IReadOnlyList<Guid>> ReconcileStaleActiveSessionsAsync(
|
private static async Task<IReadOnlyList<Guid>> ReconcileStaleActiveSessionsAsync(
|
||||||
@@ -699,6 +731,39 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
|
|||||||
IsTransientPollingException(exception.InnerException, cancellationToken);
|
IsTransientPollingException(exception.InnerException, cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static string BuildLowStorageNotificationDetail(
|
||||||
|
Domain.Entities.LiveRoom liveRoom,
|
||||||
|
int affectedSessionCount,
|
||||||
|
int forcedStopCount,
|
||||||
|
string storageDetail)
|
||||||
|
{
|
||||||
|
var builder = new StringBuilder();
|
||||||
|
builder.AppendLine($"Platform: {liveRoom.Platform}");
|
||||||
|
builder.AppendLine($"Room ID: {liveRoom.RoomId}");
|
||||||
|
|
||||||
|
if (!string.IsNullOrWhiteSpace(liveRoom.AnchorName))
|
||||||
|
{
|
||||||
|
builder.AppendLine($"Anchor: {liveRoom.AnchorName}");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!string.IsNullOrWhiteSpace(liveRoom.Title))
|
||||||
|
{
|
||||||
|
builder.AppendLine($"Title: {liveRoom.Title}");
|
||||||
|
}
|
||||||
|
|
||||||
|
builder.AppendLine($"Affected sessions: {affectedSessionCount}");
|
||||||
|
builder.AppendLine($"Forced stop attempts: {forcedStopCount}");
|
||||||
|
|
||||||
|
if (!string.IsNullOrWhiteSpace(storageDetail))
|
||||||
|
{
|
||||||
|
builder.AppendLine();
|
||||||
|
builder.AppendLine("Storage detail:");
|
||||||
|
builder.AppendLine(storageDetail);
|
||||||
|
}
|
||||||
|
|
||||||
|
return builder.ToString().Trim();
|
||||||
|
}
|
||||||
|
|
||||||
private static string BuildPollingFailureDetail(Exception exception)
|
private static string BuildPollingFailureDetail(Exception exception)
|
||||||
{
|
{
|
||||||
var root = exception.GetBaseException();
|
var root = exception.GetBaseException();
|
||||||
|
|||||||
Reference in New Issue
Block a user