feat: add storage and script failure notifications

2026-05-31 19:35:06 +08:00
parent fd5be2cc8e
commit b82110461a
7 changed files with 381 additions and 11 deletions
@@ -1,5 +1,7 @@
 using System.Diagnostics;
+using System.Text;
 using LiveRecorder.Application.Abstractions.Logging;
+using LiveRecorder.Application.Abstractions.Notifications;
 using LiveRecorder.Application.Abstractions.Scripting;
 using LiveRecorder.Application.Abstractions.Settings;
 using LiveRecorder.Application.Common;
@@ -18,15 +20,21 @@ public sealed class EventScriptService : IEventScriptService

    private readonly ISystemSettingsService _settingsService;
    private readonly ISystemLogService _systemLogService;
+    private readonly IEmailNotificationService _emailNotificationService;
+    private readonly IWebhookNotificationService _webhookNotificationService;
    private readonly ILogger<EventScriptService> _logger;

    public EventScriptService(
        ISystemSettingsService settingsService,
        ISystemLogService systemLogService,
+        IEmailNotificationService emailNotificationService,
+        IWebhookNotificationService webhookNotificationService,
        ILogger<EventScriptService> logger)
    {
        _settingsService = settingsService;
        _systemLogService = systemLogService;
+        _emailNotificationService = emailNotificationService;
+        _webhookNotificationService = webhookNotificationService;
        _logger = logger;
    }

@@ -44,8 +52,12 @@ public sealed class EventScriptService : IEventScriptService
            settings.LiveStartedScriptPath,
            settings.LiveStartedScriptContent,
            settings.EventScriptTimeoutSeconds,
+            settings.EventScriptRetryAttempts,
+            settings.EventScriptRetryDelaySeconds,
            "live_started",
            environment,
+            liveRoom,
+            recordTask: null,
            "Script",
            liveRoom.Id,
            recordSessionId: null,
@@ -67,8 +79,12 @@ public sealed class EventScriptService : IEventScriptService
            settings.LiveEndedScriptPath,
            settings.LiveEndedScriptContent,
            settings.EventScriptTimeoutSeconds,
+            settings.EventScriptRetryAttempts,
+            settings.EventScriptRetryDelaySeconds,
            "live_ended",
            environment,
+            liveRoom,
+            recordTask: null,
            "Script",
            liveRoom.Id,
            recordSessionId: null,
@@ -105,8 +121,12 @@ public sealed class EventScriptService : IEventScriptService
            settings.SegmentCompletedScriptPath,
            settings.SegmentCompletedScriptContent,
            settings.EventScriptTimeoutSeconds,
+            settings.EventScriptRetryAttempts,
+            settings.EventScriptRetryDelaySeconds,
            "segment_completed",
            environment,
+            liveRoom,
+            recordTask,
            "Script",
            liveRoom?.Id ?? recordSession.LiveRoomId,
            recordSession.Id,
@@ -176,8 +196,12 @@ public sealed class EventScriptService : IEventScriptService
        string scriptPath,
        string scriptContent,
        int timeoutSeconds,
+        int retryAttempts,
+        int retryDelaySeconds,
        string eventName,
        IReadOnlyDictionary<string, string> environment,
+        LiveRoom? liveRoom,
+        RecordTask? recordTask,
        string logCategory,
        Guid? liveRoomId,
        Guid? recordSessionId,
@@ -189,6 +213,43 @@ public sealed class EventScriptService : IEventScriptService
            return null;
        }

+        var outcome = await ExecuteWithRetryAsync(
+            scriptMode,
+            scriptPath,
+            scriptContent,
+            timeoutSeconds,
+            retryAttempts,
+            retryDelaySeconds,
+            eventName,
+            environment,
+            liveRoom,
+            recordTask,
+            logCategory,
+            liveRoomId,
+            recordSessionId,
+            recordTaskId,
+            cancellationToken);
+
+        return MapOutcome(outcome);
+    }
+
+    private async Task<ScriptExecutionOutcome> ExecuteWithRetryAsync(
+        string scriptMode,
+        string scriptPath,
+        string scriptContent,
+        int timeoutSeconds,
+        int retryAttempts,
+        int retryDelaySeconds,
+        string eventName,
+        IReadOnlyDictionary<string, string> environment,
+        LiveRoom? liveRoom,
+        RecordTask? recordTask,
+        string logCategory,
+        Guid? liveRoomId,
+        Guid? recordSessionId,
+        Guid? recordTaskId,
+        CancellationToken cancellationToken)
+    {
        var outcome = await ExecuteAsync(
            scriptMode,
            scriptPath,
@@ -202,7 +263,53 @@ public sealed class EventScriptService : IEventScriptService
            recordTaskId,
            cancellationToken);

-        return MapOutcome(outcome);
+        if (outcome.Success || !IsRetryableFailure(outcome))
+        {
+            return outcome;
+        }
+
+        var maxRetryAttempts = Math.Clamp(retryAttempts, 0, 20);
+        var totalAttempts = 1;
+
+        for (var retryIndex = 1; retryIndex <= maxRetryAttempts; retryIndex++)
+        {
+            await _systemLogService.WriteAsync(
+                SystemLogLevel.Warning,
+                logCategory,
+                $"Event script retry {retryIndex} of {maxRetryAttempts} scheduled for {eventName}.",
+                BuildRetryAttemptDetail(eventName, retryIndex + 1, maxRetryAttempts + 1, retryDelaySeconds, outcome),
+                liveRoomId,
+                recordSessionId,
+                recordTaskId,
+                cancellationToken);
+
+            if (retryDelaySeconds > 0)
+            {
+                await Task.Delay(TimeSpan.FromSeconds(Math.Clamp(retryDelaySeconds, 0, 3600)), cancellationToken);
+            }
+
+            outcome = await ExecuteAsync(
+                scriptMode,
+                scriptPath,
+                scriptContent,
+                timeoutSeconds,
+                eventName,
+                environment,
+                logCategory,
+                liveRoomId,
+                recordSessionId,
+                recordTaskId,
+                cancellationToken);
+            totalAttempts++;
+
+            if (outcome.Success || !IsRetryableFailure(outcome))
+            {
+                return outcome;
+            }
+        }
+
+        await NotifyRetryExhaustedAsync(eventName, totalAttempts, outcome, liveRoom, recordTask, cancellationToken);
+        return outcome;
    }

    private async Task<ScriptExecutionOutcome> ExecuteAsync(
@@ -225,7 +332,9 @@ public sealed class EventScriptService : IEventScriptService
                false,
                $"Event script was not configured for {eventName}.",
                null,
-                null);
+                null,
+                null,
+                ScriptFailureKind.MissingConfiguration);

            await WriteOutcomeLogAsync(
                missingConfiguration,
@@ -244,7 +353,9 @@ public sealed class EventScriptService : IEventScriptService
                false,
                $"Event script was not found for {eventName}.",
                execution.Detail,
-                null);
+                null,
+                execution.Detail,
+                ScriptFailureKind.MissingScript);

            await WriteOutcomeLogAsync(
                missingScript,
@@ -295,7 +406,9 @@ public sealed class EventScriptService : IEventScriptService
                    ? $"Event script completed for {eventName}."
                    : $"Event script exited with code {process.ExitCode} for {eventName}.",
                execution.Detail,
-                null);
+                null,
+                execution.Detail,
+                process.ExitCode == 0 ? ScriptFailureKind.None : ScriptFailureKind.ExitCode);
            outcomeLevel = process.ExitCode == 0 ? SystemLogLevel.Info : SystemLogLevel.Warning;
        }
        catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
@@ -305,7 +418,9 @@ public sealed class EventScriptService : IEventScriptService
                false,
                $"Event script timed out for {eventName}.",
                execution.Detail,
-                null);
+                null,
+                execution.Detail,
+                ScriptFailureKind.Timeout);
            outcomeLevel = SystemLogLevel.Warning;
        }
        catch (Exception ex)
@@ -315,7 +430,9 @@ public sealed class EventScriptService : IEventScriptService
                false,
                $"Event script failed for {eventName}.",
                ex.ToString(),
-                null);
+                null,
+                execution.Detail,
+                ScriptFailureKind.Exception);
            outcomeLevel = SystemLogLevel.Warning;
        }
        finally
@@ -344,6 +461,31 @@ public sealed class EventScriptService : IEventScriptService
        return outcome;
    }

+    private async Task NotifyRetryExhaustedAsync(
+        string eventName,
+        int totalAttempts,
+        ScriptExecutionOutcome outcome,
+        LiveRoom? liveRoom,
+        RecordTask? recordTask,
+        CancellationToken cancellationToken)
+    {
+        var detail = BuildRetryFailureNotificationDetail(eventName, totalAttempts, outcome);
+        await _emailNotificationService.SendExceptionAsync(
+            "EventScript",
+            "Event script failed after retries.",
+            detail,
+            liveRoom,
+            recordTask,
+            cancellationToken);
+        await _webhookNotificationService.SendExceptionAsync(
+            "EventScript",
+            "Event script failed after retries.",
+            detail,
+            liveRoom,
+            recordTask,
+            cancellationToken);
+    }
+
    private static EventScriptExecution? CreateExecution(string scriptMode, string scriptPath, string scriptContent)
    {
        if (string.Equals(scriptMode, EventScriptSourceModes.Inline, StringComparison.OrdinalIgnoreCase))
@@ -582,6 +724,71 @@ public sealed class EventScriptService : IEventScriptService
            cancellationToken);
    }

+    private static string BuildRetryAttemptDetail(
+        string eventName,
+        int nextAttempt,
+        int totalAttempts,
+        int retryDelaySeconds,
+        ScriptExecutionOutcome outcome)
+    {
+        var builder = new StringBuilder();
+        builder.AppendLine($"Event: {eventName}");
+        builder.AppendLine($"Next attempt: {nextAttempt}/{totalAttempts}");
+        builder.AppendLine($"Retry delay: {Math.Clamp(retryDelaySeconds, 0, 3600)} second(s)");
+        builder.AppendLine($"Last result: {outcome.Message}");
+
+        if (!string.IsNullOrWhiteSpace(outcome.ExecutionTarget))
+        {
+            builder.AppendLine($"Script: {outcome.ExecutionTarget}");
+        }
+
+        if (!string.IsNullOrWhiteSpace(outcome.Detail) &&
+            !string.Equals(outcome.Detail, outcome.ExecutionTarget, StringComparison.Ordinal))
+        {
+            builder.AppendLine();
+            builder.AppendLine("Detail:");
+            builder.AppendLine(outcome.Detail);
+        }
+
+        return TruncateSystemLogDetail(builder.ToString().Trim());
+    }
+
+    private static string BuildRetryFailureNotificationDetail(
+        string eventName,
+        int totalAttempts,
+        ScriptExecutionOutcome outcome)
+    {
+        var builder = new StringBuilder();
+        builder.AppendLine($"Event: {eventName}");
+        builder.AppendLine($"Attempts: {totalAttempts}");
+        builder.AppendLine($"Last result: {outcome.Message}");
+
+        if (!string.IsNullOrWhiteSpace(outcome.ExecutionTarget))
+        {
+            builder.AppendLine($"Script: {outcome.ExecutionTarget}");
+        }
+
+        if (!string.IsNullOrWhiteSpace(outcome.Detail) &&
+            !string.Equals(outcome.Detail, outcome.ExecutionTarget, StringComparison.Ordinal))
+        {
+            builder.AppendLine();
+            builder.AppendLine("Detail:");
+            builder.AppendLine(outcome.Detail);
+        }
+
+        if (!string.IsNullOrWhiteSpace(outcome.CustomLogOutput))
+        {
+            builder.AppendLine();
+            builder.AppendLine("Custom log output:");
+            builder.AppendLine(outcome.CustomLogOutput);
+        }
+
+        return TruncateSystemLogDetail(builder.ToString().Trim());
+    }
+
+    private static bool IsRetryableFailure(ScriptExecutionOutcome outcome) =>
+        outcome.FailureKind is ScriptFailureKind.ExitCode or ScriptFailureKind.Timeout or ScriptFailureKind.Exception;
+
    private static string CreateScriptLogPath()
    {
        return Path.Combine(
@@ -628,11 +835,23 @@ public sealed class EventScriptService : IEventScriptService
        }
    }

+    private enum ScriptFailureKind
+    {
+        None,
+        MissingConfiguration,
+        MissingScript,
+        ExitCode,
+        Timeout,
+        Exception
+    }
+
    private sealed record EventScriptExecution(ProcessStartInfo? StartInfo, string Detail, bool IsMissing = false);

    private sealed record ScriptExecutionOutcome(
        bool Success,
        string Message,
        string? Detail,
-        string? CustomLogOutput);
+        string? CustomLogOutput,
+        string? ExecutionTarget,
+        ScriptFailureKind FailureKind);
 }