diff --git a/frontend/src/styles/main.css b/frontend/src/styles/main.css index e81e63b..afa2607 100644 --- a/frontend/src/styles/main.css +++ b/frontend/src/styles/main.css @@ -61,9 +61,17 @@ --el-border-color-light: var(--border-subtle); --el-border-radius-base: var(--radius-sm); --el-bg-color: transparent; + --el-bg-color-page: var(--bg-base); + --el-bg-color-overlay: var(--surface); + --el-fill-color: var(--surface-muted); --el-fill-color-blank: var(--surface); --el-fill-color-light: var(--surface-muted); --el-fill-color-lighter: var(--surface-strong); + --el-fill-color-dark: var(--surface-raised); + --el-fill-color-darker: var(--surface-strong); + --el-disabled-bg-color: var(--surface-muted); + --el-disabled-text-color: var(--text-soft); + --el-text-color-placeholder: var(--text-soft); --el-mask-color: rgba(15, 23, 42, 0.54); } @@ -123,9 +131,17 @@ html[data-theme="dark"] { --el-text-color-secondary: var(--text-muted); --el-border-color: var(--border-base); --el-border-color-light: var(--border-subtle); + --el-bg-color-page: var(--bg-base); + --el-bg-color-overlay: var(--surface); + --el-fill-color: var(--surface-muted); --el-fill-color-blank: var(--surface); --el-fill-color-light: var(--surface-muted); --el-fill-color-lighter: var(--surface-strong); + --el-fill-color-dark: var(--surface-raised); + --el-fill-color-darker: var(--surface-strong); + --el-disabled-bg-color: var(--surface-muted); + --el-disabled-text-color: var(--text-soft); + --el-text-color-placeholder: var(--text-soft); --el-mask-color: rgba(3, 7, 14, 0.72); } @@ -255,6 +271,7 @@ select:focus-visible { .surface-card { position: relative; overflow: hidden; + --el-card-bg-color: transparent; border-radius: var(--radius-md); border: 1px solid var(--border-subtle); background: @@ -573,9 +590,33 @@ html[data-theme="dark"] .stat-card { .el-card { --el-card-border-color: transparent; + --el-card-bg-color: transparent; background: transparent; } +.el-tabs--border-card { + border-color: var(--border-subtle); + background: var(--surface); +} + +.el-tabs--border-card > .el-tabs__content { + background: transparent; + color: var(--text-primary); +} + +.el-tabs--border-card > .el-tabs__header { + background: transparent; +} + +.el-tabs--border-card > .el-tabs__header .el-tabs__item { + color: var(--text-secondary); +} + +.el-tabs--border-card > .el-tabs__header .el-tabs__item.is-active { + color: var(--text-primary); + background: var(--surface-raised); +} + .el-button { min-height: var(--control-height); padding: 0 15px; @@ -802,6 +843,17 @@ html[data-theme="dark"] .stat-card { box-shadow: var(--shadow-float); } +.el-message-box, +.el-popover.el-popper, +.el-select__popper.el-popper, +.el-picker__popper.el-popper, +.el-dropdown__popper.el-popper .el-dropdown-menu { + border-color: var(--border-subtle); + background: linear-gradient(180deg, var(--surface-raised), var(--surface)); + color: var(--text-primary); + box-shadow: var(--shadow-float); +} + .el-dialog__header { margin: 0; padding: 22px 24px 10px; diff --git a/frontend/src/types.ts b/frontend/src/types.ts index e45abc2..7437ef1 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -458,6 +458,8 @@ export interface SystemSettings { segmentCompletedScriptPath: string; segmentCompletedScriptContent: string; eventScriptTimeoutSeconds: number; + eventScriptRetryAttempts: number; + eventScriptRetryDelaySeconds: number; enableEmailNotification: boolean; emailSmtpHost: string; emailSmtpPort: number; diff --git a/frontend/src/views/SettingsView.vue b/frontend/src/views/SettingsView.vue index eb3a95a..84d2317 100644 --- a/frontend/src/views/SettingsView.vue +++ b/frontend/src/views/SettingsView.vue @@ -168,6 +168,8 @@ const form = reactive({ segmentCompletedScriptPath: "", segmentCompletedScriptContent: "", eventScriptTimeoutSeconds: 60, + eventScriptRetryAttempts: 3, + eventScriptRetryDelaySeconds: 10, enableEmailNotification: false, emailSmtpHost: "", emailSmtpPort: 587, @@ -1480,16 +1482,26 @@ watch( - + - + + + + + + + + + + + @@ -1676,6 +1688,10 @@ watch( The official Docker image includes curl and jq by default. If you run on a host machine or a custom image, rely on the commands available in that environment. +
+ Set Retry attempts to 0 to disable automatic retries. Retry exhaustion failures are sent through the existing exception notification channel. +
+
Environment variable examples
@@ -1699,7 +1715,8 @@ watch(

Webhook 閫氱煡

-

Send fixed JSON POST payloads with custom headers. Currently only live-started and exception events are sent.

+

Webhook 通知

+

Send fixed JSON POST payloads with custom headers. Exception notifications also cover low-storage stop events and event-script failures after retries are exhausted.

@@ -1772,7 +1789,8 @@ watch(

閭欢閫氱煡

-

Configure SMTP plus HTML templates for live-started and exception alerts. Tests use the current unsaved form values directly.

+

邮件通知

+

Configure SMTP plus HTML templates for live-started and exception alerts. Exception notifications also cover low-storage stop events and event-script failures after retries are exhausted.

diff --git a/src/LiveRecorder.Application/Models/Settings/SettingsModels.cs b/src/LiveRecorder.Application/Models/Settings/SettingsModels.cs index 0df07ef..01cfbf0 100644 --- a/src/LiveRecorder.Application/Models/Settings/SettingsModels.cs +++ b/src/LiveRecorder.Application/Models/Settings/SettingsModels.cs @@ -177,6 +177,10 @@ public sealed class SystemSettingsDto public int EventScriptTimeoutSeconds { get; set; } = 60; + public int EventScriptRetryAttempts { get; set; } = 3; + + public int EventScriptRetryDelaySeconds { get; set; } = 10; + public bool EnableRetentionCleanup { get; set; } = false; public int RetentionDays { get; set; } = 30; @@ -446,6 +450,10 @@ public sealed class UpdateSystemSettingsRequest public int EventScriptTimeoutSeconds { get; set; } = 60; + public int EventScriptRetryAttempts { get; set; } = 3; + + public int EventScriptRetryDelaySeconds { get; set; } = 10; + public bool EnableRetentionCleanup { get; set; } = false; public int RetentionDays { get; set; } = 30; diff --git a/src/LiveRecorder.Application/Services/SystemSettingsService.cs b/src/LiveRecorder.Application/Services/SystemSettingsService.cs index 7b00c8c..f72a5a8 100644 --- a/src/LiveRecorder.Application/Services/SystemSettingsService.cs +++ b/src/LiveRecorder.Application/Services/SystemSettingsService.cs @@ -71,6 +71,8 @@ public sealed class SystemSettingsService : ISystemSettingsService private const string SegmentCompletedScriptPathKey = "event_scripts.segment_completed.path"; private const string SegmentCompletedScriptContentKey = "event_scripts.segment_completed.content"; private const string EventScriptTimeoutSecondsKey = "event_scripts.timeout_seconds"; + private const string EventScriptRetryAttemptsKey = "event_scripts.retry_attempts"; + private const string EventScriptRetryDelaySecondsKey = "event_scripts.retry_delay_seconds"; private const string EnableRetentionCleanupKey = "retention.cleanup.enabled"; private const string RetentionDaysKey = "retention.cleanup.days"; private const string RetentionDeleteFilesKey = "retention.cleanup.delete_files"; @@ -202,6 +204,8 @@ public sealed class SystemSettingsService : ISystemSettingsService SegmentCompletedScriptPath = GetValue(lookup, SegmentCompletedScriptPathKey, string.Empty), SegmentCompletedScriptContent = GetValue(lookup, SegmentCompletedScriptContentKey, string.Empty), EventScriptTimeoutSeconds = GetIntValue(lookup, EventScriptTimeoutSecondsKey, 60, 1, 3600), + EventScriptRetryAttempts = GetIntValue(lookup, EventScriptRetryAttemptsKey, 3, 0, 20), + EventScriptRetryDelaySeconds = GetIntValue(lookup, EventScriptRetryDelaySecondsKey, 10, 0, 3600), EnableRetentionCleanup = bool.TryParse(GetValue(lookup, EnableRetentionCleanupKey, "false"), out var enableRetentionCleanup) && enableRetentionCleanup, RetentionDays = GetIntValue(lookup, RetentionDaysKey, 30, 1, 3650), RetentionDeleteFiles = bool.TryParse(GetValue(lookup, RetentionDeleteFilesKey, "false"), out var retentionDeleteFiles) && retentionDeleteFiles, @@ -352,6 +356,8 @@ public sealed class SystemSettingsService : ISystemSettingsService await UpsertAsync(SegmentCompletedScriptPathKey, request.SegmentCompletedScriptPath.Trim(), now, cancellationToken); await UpsertAsync(SegmentCompletedScriptContentKey, request.SegmentCompletedScriptContent, now, cancellationToken); await UpsertAsync(EventScriptTimeoutSecondsKey, Math.Clamp(request.EventScriptTimeoutSeconds, 1, 3600).ToString(), now, cancellationToken); + await UpsertAsync(EventScriptRetryAttemptsKey, Math.Clamp(request.EventScriptRetryAttempts, 0, 20).ToString(), now, cancellationToken); + await UpsertAsync(EventScriptRetryDelaySecondsKey, Math.Clamp(request.EventScriptRetryDelaySeconds, 0, 3600).ToString(), now, cancellationToken); await UpsertAsync(EnableRetentionCleanupKey, request.EnableRetentionCleanup.ToString(), now, cancellationToken); await UpsertAsync(RetentionDaysKey, Math.Clamp(request.RetentionDays, 1, 3650).ToString(), now, cancellationToken); await UpsertAsync(RetentionDeleteFilesKey, request.RetentionDeleteFiles.ToString(), now, cancellationToken); diff --git a/src/LiveRecorder.Infrastructure/Services/EventScriptService.cs b/src/LiveRecorder.Infrastructure/Services/EventScriptService.cs index 203f3d5..77356c7 100644 --- a/src/LiveRecorder.Infrastructure/Services/EventScriptService.cs +++ b/src/LiveRecorder.Infrastructure/Services/EventScriptService.cs @@ -1,5 +1,7 @@ using System.Diagnostics; +using System.Text; using LiveRecorder.Application.Abstractions.Logging; +using LiveRecorder.Application.Abstractions.Notifications; using LiveRecorder.Application.Abstractions.Scripting; using LiveRecorder.Application.Abstractions.Settings; using LiveRecorder.Application.Common; @@ -18,15 +20,21 @@ public sealed class EventScriptService : IEventScriptService private readonly ISystemSettingsService _settingsService; private readonly ISystemLogService _systemLogService; + private readonly IEmailNotificationService _emailNotificationService; + private readonly IWebhookNotificationService _webhookNotificationService; private readonly ILogger _logger; public EventScriptService( ISystemSettingsService settingsService, ISystemLogService systemLogService, + IEmailNotificationService emailNotificationService, + IWebhookNotificationService webhookNotificationService, ILogger logger) { _settingsService = settingsService; _systemLogService = systemLogService; + _emailNotificationService = emailNotificationService; + _webhookNotificationService = webhookNotificationService; _logger = logger; } @@ -44,8 +52,12 @@ public sealed class EventScriptService : IEventScriptService settings.LiveStartedScriptPath, settings.LiveStartedScriptContent, settings.EventScriptTimeoutSeconds, + settings.EventScriptRetryAttempts, + settings.EventScriptRetryDelaySeconds, "live_started", environment, + liveRoom, + recordTask: null, "Script", liveRoom.Id, recordSessionId: null, @@ -67,8 +79,12 @@ public sealed class EventScriptService : IEventScriptService settings.LiveEndedScriptPath, settings.LiveEndedScriptContent, settings.EventScriptTimeoutSeconds, + settings.EventScriptRetryAttempts, + settings.EventScriptRetryDelaySeconds, "live_ended", environment, + liveRoom, + recordTask: null, "Script", liveRoom.Id, recordSessionId: null, @@ -105,8 +121,12 @@ public sealed class EventScriptService : IEventScriptService settings.SegmentCompletedScriptPath, settings.SegmentCompletedScriptContent, settings.EventScriptTimeoutSeconds, + settings.EventScriptRetryAttempts, + settings.EventScriptRetryDelaySeconds, "segment_completed", environment, + liveRoom, + recordTask, "Script", liveRoom?.Id ?? recordSession.LiveRoomId, recordSession.Id, @@ -176,8 +196,12 @@ public sealed class EventScriptService : IEventScriptService string scriptPath, string scriptContent, int timeoutSeconds, + int retryAttempts, + int retryDelaySeconds, string eventName, IReadOnlyDictionary environment, + LiveRoom? liveRoom, + RecordTask? recordTask, string logCategory, Guid? liveRoomId, Guid? recordSessionId, @@ -189,6 +213,43 @@ public sealed class EventScriptService : IEventScriptService return null; } + var outcome = await ExecuteWithRetryAsync( + scriptMode, + scriptPath, + scriptContent, + timeoutSeconds, + retryAttempts, + retryDelaySeconds, + eventName, + environment, + liveRoom, + recordTask, + logCategory, + liveRoomId, + recordSessionId, + recordTaskId, + cancellationToken); + + return MapOutcome(outcome); + } + + private async Task ExecuteWithRetryAsync( + string scriptMode, + string scriptPath, + string scriptContent, + int timeoutSeconds, + int retryAttempts, + int retryDelaySeconds, + string eventName, + IReadOnlyDictionary environment, + LiveRoom? liveRoom, + RecordTask? recordTask, + string logCategory, + Guid? liveRoomId, + Guid? recordSessionId, + Guid? recordTaskId, + CancellationToken cancellationToken) + { var outcome = await ExecuteAsync( scriptMode, scriptPath, @@ -202,7 +263,53 @@ public sealed class EventScriptService : IEventScriptService recordTaskId, cancellationToken); - return MapOutcome(outcome); + if (outcome.Success || !IsRetryableFailure(outcome)) + { + return outcome; + } + + var maxRetryAttempts = Math.Clamp(retryAttempts, 0, 20); + var totalAttempts = 1; + + for (var retryIndex = 1; retryIndex <= maxRetryAttempts; retryIndex++) + { + await _systemLogService.WriteAsync( + SystemLogLevel.Warning, + logCategory, + $"Event script retry {retryIndex} of {maxRetryAttempts} scheduled for {eventName}.", + BuildRetryAttemptDetail(eventName, retryIndex + 1, maxRetryAttempts + 1, retryDelaySeconds, outcome), + liveRoomId, + recordSessionId, + recordTaskId, + cancellationToken); + + if (retryDelaySeconds > 0) + { + await Task.Delay(TimeSpan.FromSeconds(Math.Clamp(retryDelaySeconds, 0, 3600)), cancellationToken); + } + + outcome = await ExecuteAsync( + scriptMode, + scriptPath, + scriptContent, + timeoutSeconds, + eventName, + environment, + logCategory, + liveRoomId, + recordSessionId, + recordTaskId, + cancellationToken); + totalAttempts++; + + if (outcome.Success || !IsRetryableFailure(outcome)) + { + return outcome; + } + } + + await NotifyRetryExhaustedAsync(eventName, totalAttempts, outcome, liveRoom, recordTask, cancellationToken); + return outcome; } private async Task ExecuteAsync( @@ -225,7 +332,9 @@ public sealed class EventScriptService : IEventScriptService false, $"Event script was not configured for {eventName}.", null, - null); + null, + null, + ScriptFailureKind.MissingConfiguration); await WriteOutcomeLogAsync( missingConfiguration, @@ -244,7 +353,9 @@ public sealed class EventScriptService : IEventScriptService false, $"Event script was not found for {eventName}.", execution.Detail, - null); + null, + execution.Detail, + ScriptFailureKind.MissingScript); await WriteOutcomeLogAsync( missingScript, @@ -295,7 +406,9 @@ public sealed class EventScriptService : IEventScriptService ? $"Event script completed for {eventName}." : $"Event script exited with code {process.ExitCode} for {eventName}.", execution.Detail, - null); + null, + execution.Detail, + process.ExitCode == 0 ? ScriptFailureKind.None : ScriptFailureKind.ExitCode); outcomeLevel = process.ExitCode == 0 ? SystemLogLevel.Info : SystemLogLevel.Warning; } catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) @@ -305,7 +418,9 @@ public sealed class EventScriptService : IEventScriptService false, $"Event script timed out for {eventName}.", execution.Detail, - null); + null, + execution.Detail, + ScriptFailureKind.Timeout); outcomeLevel = SystemLogLevel.Warning; } catch (Exception ex) @@ -315,7 +430,9 @@ public sealed class EventScriptService : IEventScriptService false, $"Event script failed for {eventName}.", ex.ToString(), - null); + null, + execution.Detail, + ScriptFailureKind.Exception); outcomeLevel = SystemLogLevel.Warning; } finally @@ -344,6 +461,31 @@ public sealed class EventScriptService : IEventScriptService return outcome; } + private async Task NotifyRetryExhaustedAsync( + string eventName, + int totalAttempts, + ScriptExecutionOutcome outcome, + LiveRoom? liveRoom, + RecordTask? recordTask, + CancellationToken cancellationToken) + { + var detail = BuildRetryFailureNotificationDetail(eventName, totalAttempts, outcome); + await _emailNotificationService.SendExceptionAsync( + "EventScript", + "Event script failed after retries.", + detail, + liveRoom, + recordTask, + cancellationToken); + await _webhookNotificationService.SendExceptionAsync( + "EventScript", + "Event script failed after retries.", + detail, + liveRoom, + recordTask, + cancellationToken); + } + private static EventScriptExecution? CreateExecution(string scriptMode, string scriptPath, string scriptContent) { if (string.Equals(scriptMode, EventScriptSourceModes.Inline, StringComparison.OrdinalIgnoreCase)) @@ -582,6 +724,71 @@ public sealed class EventScriptService : IEventScriptService cancellationToken); } + private static string BuildRetryAttemptDetail( + string eventName, + int nextAttempt, + int totalAttempts, + int retryDelaySeconds, + ScriptExecutionOutcome outcome) + { + var builder = new StringBuilder(); + builder.AppendLine($"Event: {eventName}"); + builder.AppendLine($"Next attempt: {nextAttempt}/{totalAttempts}"); + builder.AppendLine($"Retry delay: {Math.Clamp(retryDelaySeconds, 0, 3600)} second(s)"); + builder.AppendLine($"Last result: {outcome.Message}"); + + if (!string.IsNullOrWhiteSpace(outcome.ExecutionTarget)) + { + builder.AppendLine($"Script: {outcome.ExecutionTarget}"); + } + + if (!string.IsNullOrWhiteSpace(outcome.Detail) && + !string.Equals(outcome.Detail, outcome.ExecutionTarget, StringComparison.Ordinal)) + { + builder.AppendLine(); + builder.AppendLine("Detail:"); + builder.AppendLine(outcome.Detail); + } + + return TruncateSystemLogDetail(builder.ToString().Trim()); + } + + private static string BuildRetryFailureNotificationDetail( + string eventName, + int totalAttempts, + ScriptExecutionOutcome outcome) + { + var builder = new StringBuilder(); + builder.AppendLine($"Event: {eventName}"); + builder.AppendLine($"Attempts: {totalAttempts}"); + builder.AppendLine($"Last result: {outcome.Message}"); + + if (!string.IsNullOrWhiteSpace(outcome.ExecutionTarget)) + { + builder.AppendLine($"Script: {outcome.ExecutionTarget}"); + } + + if (!string.IsNullOrWhiteSpace(outcome.Detail) && + !string.Equals(outcome.Detail, outcome.ExecutionTarget, StringComparison.Ordinal)) + { + builder.AppendLine(); + builder.AppendLine("Detail:"); + builder.AppendLine(outcome.Detail); + } + + if (!string.IsNullOrWhiteSpace(outcome.CustomLogOutput)) + { + builder.AppendLine(); + builder.AppendLine("Custom log output:"); + builder.AppendLine(outcome.CustomLogOutput); + } + + return TruncateSystemLogDetail(builder.ToString().Trim()); + } + + private static bool IsRetryableFailure(ScriptExecutionOutcome outcome) => + outcome.FailureKind is ScriptFailureKind.ExitCode or ScriptFailureKind.Timeout or ScriptFailureKind.Exception; + private static string CreateScriptLogPath() { return Path.Combine( @@ -628,11 +835,23 @@ public sealed class EventScriptService : IEventScriptService } } + private enum ScriptFailureKind + { + None, + MissingConfiguration, + MissingScript, + ExitCode, + Timeout, + Exception + } + private sealed record EventScriptExecution(ProcessStartInfo? StartInfo, string Detail, bool IsMissing = false); private sealed record ScriptExecutionOutcome( bool Success, string Message, string? Detail, - string? CustomLogOutput); + string? CustomLogOutput, + string? ExecutionTarget, + ScriptFailureKind FailureKind); } diff --git a/src/LiveRecorder.Infrastructure/Services/LiveRoomPollingBackgroundService.cs b/src/LiveRecorder.Infrastructure/Services/LiveRoomPollingBackgroundService.cs index 538e8b0..08ed5be 100644 --- a/src/LiveRecorder.Infrastructure/Services/LiveRoomPollingBackgroundService.cs +++ b/src/LiveRecorder.Infrastructure/Services/LiveRoomPollingBackgroundService.cs @@ -1,4 +1,5 @@ using System.Collections.Concurrent; +using System.Text; using LiveRecorder.Application.Common; using LiveRecorder.Application.Abstractions.Logging; using LiveRecorder.Application.Abstractions.Notifications; @@ -360,6 +361,9 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR dbContext, ffmpegService, logService, + emailNotificationService, + webhookNotificationService, + liveRoom, liveRoom.Id, pauseCheck.Message, cancellationToken); @@ -589,6 +593,9 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR LiveRecorderDbContext dbContext, IFfmpegService ffmpegService, ISystemLogService logService, + IEmailNotificationService emailNotificationService, + IWebhookNotificationService webhookNotificationService, + Domain.Entities.LiveRoom liveRoom, Guid liveRoomId, string detail, CancellationToken cancellationToken) @@ -600,8 +607,13 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR item.Status == RecordSessionStatus.Stopping)) .ToListAsync(cancellationToken); + var affectedSessionCount = 0; + var forcedStopCount = 0; + foreach (var activeSession in activeSessions.OrderBy(static item => item.CreatedAt)) { + affectedSessionCount++; + await logService.WriteAsync( SystemLogLevel.Warning, "Storage", @@ -628,12 +640,32 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR liveRoomId, activeSession.Id, cancellationToken: cancellationToken); + forcedStopCount++; await ffmpegService.KillAndWaitAsync(activeSession.Id, OfflineForcedStopTimeout, cancellationToken); } } await ffmpegService.TryReconcileInactiveSessionAsync(activeSession.Id, cancellationToken); } + + if (affectedSessionCount <= 0) + { + return; + } + + var notificationDetail = BuildLowStorageNotificationDetail(liveRoom, affectedSessionCount, forcedStopCount, detail); + await emailNotificationService.SendExceptionAsync( + "StorageGuard", + "Low storage paused active recording sessions.", + notificationDetail, + liveRoom, + cancellationToken: cancellationToken); + await webhookNotificationService.SendExceptionAsync( + "StorageGuard", + "Low storage paused active recording sessions.", + notificationDetail, + liveRoom, + cancellationToken: cancellationToken); } private static async Task> ReconcileStaleActiveSessionsAsync( @@ -699,6 +731,39 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR IsTransientPollingException(exception.InnerException, cancellationToken); } + private static string BuildLowStorageNotificationDetail( + Domain.Entities.LiveRoom liveRoom, + int affectedSessionCount, + int forcedStopCount, + string storageDetail) + { + var builder = new StringBuilder(); + builder.AppendLine($"Platform: {liveRoom.Platform}"); + builder.AppendLine($"Room ID: {liveRoom.RoomId}"); + + if (!string.IsNullOrWhiteSpace(liveRoom.AnchorName)) + { + builder.AppendLine($"Anchor: {liveRoom.AnchorName}"); + } + + if (!string.IsNullOrWhiteSpace(liveRoom.Title)) + { + builder.AppendLine($"Title: {liveRoom.Title}"); + } + + builder.AppendLine($"Affected sessions: {affectedSessionCount}"); + builder.AppendLine($"Forced stop attempts: {forcedStopCount}"); + + if (!string.IsNullOrWhiteSpace(storageDetail)) + { + builder.AppendLine(); + builder.AppendLine("Storage detail:"); + builder.AppendLine(storageDetail); + } + + return builder.ToString().Trim(); + } + private static string BuildPollingFailureDetail(Exception exception) { var root = exception.GetBaseException();