feat: add recovery center and daily reviews

This commit is contained in:
2026-04-26 11:03:08 +08:00
parent 94f6e08dea
commit 79047b5488
33 changed files with 3127 additions and 40 deletions
@@ -11,6 +11,10 @@ namespace LiveRecorder.Infrastructure.Services;
public sealed class EventScriptService : IEventScriptService
{
private const string ScriptLogPathEnvironmentVariable = "LIVE_RECORDER_SCRIPT_LOG_PATH";
private const int SystemLogDetailMaxLength = 4000;
private const string TruncatedDetailSuffix = "... [truncated to fit system log detail limit]";
private readonly ISystemSettingsService _settingsService;
private readonly ISystemLogService _systemLogService;
private readonly ILogger<EventScriptService> _logger;
@@ -38,6 +42,7 @@ public sealed class EventScriptService : IEventScriptService
settings.EventScriptTimeoutSeconds,
"live_started",
environment,
"Script",
liveRoom.Id,
recordSessionId: null,
recordTaskId: null,
@@ -57,6 +62,7 @@ public sealed class EventScriptService : IEventScriptService
settings.EventScriptTimeoutSeconds,
"live_ended",
environment,
"Script",
liveRoom.Id,
recordSessionId: null,
recordTaskId: null,
@@ -93,12 +99,61 @@ public sealed class EventScriptService : IEventScriptService
settings.EventScriptTimeoutSeconds,
"segment_completed",
environment,
"Script",
liveRoom?.Id ?? recordSession.LiveRoomId,
recordSession.Id,
recordTask.Id,
cancellationToken);
}
public async Task<EventScriptTestResultDto> TestAsync(
TestEventScriptRequest request,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(request);
var eventName = NormalizeEventType(request.EventType);
if (eventName is null)
{
var result = new EventScriptTestResultDto
{
Success = false,
Message = "Unsupported event script test type.",
Detail = request.EventType
};
await _systemLogService.WriteAsync(
SystemLogLevel.Warning,
"ScriptTest",
result.Message,
result.Detail,
cancellationToken: cancellationToken);
return result;
}
var outcome = await ExecuteAsync(
request.ScriptMode,
request.ScriptPath,
request.ScriptContent,
request.TimeoutSeconds,
eventName,
BuildTestEnvironment(eventName, DateTimeOffset.UtcNow),
"ScriptTest",
liveRoomId: null,
recordSessionId: null,
recordTaskId: null,
cancellationToken);
return new EventScriptTestResultDto
{
Success = outcome.Success,
Message = outcome.Message,
Detail = outcome.Detail,
CustomLogOutput = outcome.CustomLogOutput
};
}
private async Task RunAsync(
bool enabled,
string scriptMode,
@@ -107,6 +162,7 @@ public sealed class EventScriptService : IEventScriptService
int timeoutSeconds,
string eventName,
IReadOnlyDictionary<string, string> environment,
string logCategory,
Guid? liveRoomId,
Guid? recordSessionId,
Guid? recordTaskId,
@@ -117,24 +173,70 @@ public sealed class EventScriptService : IEventScriptService
return;
}
await ExecuteAsync(
scriptMode,
scriptPath,
scriptContent,
timeoutSeconds,
eventName,
environment,
logCategory,
liveRoomId,
recordSessionId,
recordTaskId,
cancellationToken);
}
private async Task<ScriptExecutionOutcome> ExecuteAsync(
string scriptMode,
string scriptPath,
string scriptContent,
int timeoutSeconds,
string eventName,
IReadOnlyDictionary<string, string> environment,
string logCategory,
Guid? liveRoomId,
Guid? recordSessionId,
Guid? recordTaskId,
CancellationToken cancellationToken)
{
var execution = CreateExecution(scriptMode, scriptPath, scriptContent);
if (execution is null)
{
return;
}
var missingConfiguration = new ScriptExecutionOutcome(
false,
$"Event script was not configured for {eventName}.",
null,
null);
if (execution.IsMissing)
{
await _systemLogService.WriteAsync(
await WriteOutcomeLogAsync(
missingConfiguration,
logCategory,
SystemLogLevel.Warning,
"Script",
$"Event script was not found for {eventName}.",
execution.Detail,
liveRoomId,
recordSessionId,
recordTaskId,
cancellationToken);
return;
return missingConfiguration;
}
if (execution.IsMissing)
{
var missingScript = new ScriptExecutionOutcome(
false,
$"Event script was not found for {eventName}.",
execution.Detail,
null);
await WriteOutcomeLogAsync(
missingScript,
logCategory,
SystemLogLevel.Warning,
liveRoomId,
recordSessionId,
recordTaskId,
cancellationToken);
return missingScript;
}
var startInfo = execution.StartInfo!;
@@ -143,12 +245,25 @@ public sealed class EventScriptService : IEventScriptService
startInfo.Environment[pair.Key] = pair.Value;
}
var scriptLogPath = CreateScriptLogPath();
try
{
await File.WriteAllTextAsync(scriptLogPath, string.Empty, CancellationToken.None);
startInfo.Environment[ScriptLogPathEnvironmentVariable] = scriptLogPath;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to prepare event script log file for {EventName}", eventName);
}
using var process = new Process
{
StartInfo = startInfo,
EnableRaisingEvents = true
};
ScriptExecutionOutcome outcome;
SystemLogLevel outcomeLevel;
try
{
process.Start();
@@ -156,44 +271,59 @@ public sealed class EventScriptService : IEventScriptService
timeoutCts.CancelAfter(TimeSpan.FromSeconds(Math.Clamp(timeoutSeconds, 1, 3600)));
await process.WaitForExitAsync(timeoutCts.Token);
await _systemLogService.WriteAsync(
process.ExitCode == 0 ? SystemLogLevel.Info : SystemLogLevel.Warning,
"Script",
outcome = new ScriptExecutionOutcome(
process.ExitCode == 0,
process.ExitCode == 0
? $"Event script completed for {eventName}."
: $"Event script exited with code {process.ExitCode} for {eventName}.",
execution.Detail,
liveRoomId,
recordSessionId,
recordTaskId,
cancellationToken);
null);
outcomeLevel = process.ExitCode == 0 ? SystemLogLevel.Info : SystemLogLevel.Warning;
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
{
TryKill(process);
await _systemLogService.WriteAsync(
SystemLogLevel.Warning,
"Script",
outcome = new ScriptExecutionOutcome(
false,
$"Event script timed out for {eventName}.",
execution.Detail,
liveRoomId,
recordSessionId,
recordTaskId,
CancellationToken.None);
null);
outcomeLevel = SystemLogLevel.Warning;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Event script failed for {EventName}", eventName);
await _systemLogService.WriteAsync(
SystemLogLevel.Warning,
"Script",
outcome = new ScriptExecutionOutcome(
false,
$"Event script failed for {eventName}.",
ex.ToString(),
liveRoomId,
recordSessionId,
recordTaskId,
cancellationToken);
null);
outcomeLevel = SystemLogLevel.Warning;
}
finally
{
// Any script-provided text is persisted and surfaced separately from the execution outcome.
}
var customLogOutput = await TryWriteCustomLogOutputAsync(
eventName,
logCategory,
scriptLogPath,
liveRoomId,
recordSessionId,
recordTaskId);
TryDeleteFile(scriptLogPath);
outcome = outcome with { CustomLogOutput = customLogOutput };
await WriteOutcomeLogAsync(
outcome,
logCategory,
outcomeLevel,
liveRoomId,
recordSessionId,
recordTaskId,
cancellationToken);
return outcome;
}
private static EventScriptExecution? CreateExecution(string scriptMode, string scriptPath, string scriptContent)
@@ -237,6 +367,47 @@ public sealed class EventScriptService : IEventScriptService
};
}
private static IReadOnlyDictionary<string, string> BuildTestEnvironment(string eventName, DateTimeOffset occurredAt)
{
var environment = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["LIVE_RECORDER_EVENT"] = eventName,
["LIVE_RECORDER_PLATFORM"] = "Douyin",
["LIVE_RECORDER_LIVE_ROOM_ID"] = "6f73a2f2-1d4c-4e7a-a9b1-3d29d54ed901",
["LIVE_RECORDER_ROOM_ID"] = "676493068539",
["LIVE_RECORDER_TITLE"] = "Sample live title",
["LIVE_RECORDER_ANCHOR"] = "Sample anchor",
["LIVE_RECORDER_SOURCE_URL"] = "https://live.douyin.com/676493068539",
["LIVE_RECORDER_OCCURRED_AT_UTC"] = occurredAt.ToString("O")
};
if (string.Equals(eventName, "segment_completed", StringComparison.OrdinalIgnoreCase))
{
environment["LIVE_RECORDER_RECORD_SESSION_ID"] = "8e2e9c64-b8f6-4d15-b6cb-1d4ce0adab77";
environment["LIVE_RECORDER_RECORD_TASK_ID"] = "2a4810a2-7ef4-4a22-90d4-0211b90cc54c";
environment["LIVE_RECORDER_SEGMENT_INDEX"] = "1";
environment["LIVE_RECORDER_SEGMENT_FILE_PATH"] = "/app/records/Douyin/Sample Anchor/2026-04-25/203000_Sample live title__00001.mp4";
environment["LIVE_RECORDER_DANMAKU_FILE_PATH"] = "/app/records/Douyin/Sample Anchor/2026-04-25/203000_Sample live title__00001.xml";
environment["LIVE_RECORDER_DURATION_SECONDS"] = "2185.1";
environment["LIVE_RECORDER_FILE_SIZE_BYTES"] = "734003200";
environment["LIVE_RECORDER_TASK_STATUS"] = "Completed";
environment["LIVE_RECORDER_SESSION_STATUS"] = "Completed";
}
return environment;
}
private static string? NormalizeEventType(string? eventType)
{
return eventType?.Trim().ToLowerInvariant() switch
{
"live_started" => "live_started",
"live_ended" => "live_ended",
"segment_completed" => "segment_completed",
_ => null
};
}
private static ProcessStartInfo CreateStartInfo(string scriptPath)
{
var extension = Path.GetExtension(scriptPath);
@@ -330,6 +501,85 @@ public sealed class EventScriptService : IEventScriptService
return new FileInfo(normalizedPath).Length.ToString();
}
private async Task<string?> TryWriteCustomLogOutputAsync(
string eventName,
string logCategory,
string scriptLogPath,
Guid? liveRoomId,
Guid? recordSessionId,
Guid? recordTaskId)
{
if (string.IsNullOrWhiteSpace(scriptLogPath) || !File.Exists(scriptLogPath))
{
return null;
}
try
{
var output = await File.ReadAllTextAsync(scriptLogPath, CancellationToken.None);
var trimmedOutput = output.Trim();
if (string.IsNullOrWhiteSpace(trimmedOutput))
{
return null;
}
await _systemLogService.WriteAsync(
SystemLogLevel.Info,
logCategory,
$"Event script emitted custom log output for {eventName}.",
TruncateSystemLogDetail(trimmedOutput),
liveRoomId,
recordSessionId,
recordTaskId,
CancellationToken.None);
return trimmedOutput;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to collect event script custom log output for {EventName}", eventName);
return null;
}
}
private async Task WriteOutcomeLogAsync(
ScriptExecutionOutcome outcome,
string logCategory,
SystemLogLevel level,
Guid? liveRoomId,
Guid? recordSessionId,
Guid? recordTaskId,
CancellationToken cancellationToken)
{
await _systemLogService.WriteAsync(
level,
logCategory,
outcome.Message,
outcome.Detail,
liveRoomId,
recordSessionId,
recordTaskId,
cancellationToken);
}
private static string CreateScriptLogPath()
{
return Path.Combine(
Path.GetTempPath(),
$"live-recorder-script-log-{Guid.NewGuid():N}.txt");
}
private static string TruncateSystemLogDetail(string detail)
{
if (detail.Length <= SystemLogDetailMaxLength)
{
return detail;
}
var prefixLength = Math.Max(0, SystemLogDetailMaxLength - TruncatedDetailSuffix.Length);
return string.Concat(detail[..prefixLength], TruncatedDetailSuffix);
}
private static void TryKill(Process process)
{
try
@@ -344,5 +594,25 @@ public sealed class EventScriptService : IEventScriptService
}
}
private static void TryDeleteFile(string path)
{
try
{
if (!string.IsNullOrWhiteSpace(path) && File.Exists(path))
{
File.Delete(path);
}
}
catch
{
}
}
private sealed record EventScriptExecution(ProcessStartInfo? StartInfo, string Detail, bool IsMissing = false);
private sealed record ScriptExecutionOutcome(
bool Success,
string Message,
string? Detail,
string? CustomLogOutput);
}
@@ -649,6 +649,7 @@ public sealed partial class FfmpegService
var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
var settingsService = scope.ServiceProvider.GetRequiredService<ISystemSettingsService>();
var emailNotificationService = scope.ServiceProvider.GetRequiredService<IEmailNotificationService>();
var webhookNotificationService = scope.ServiceProvider.GetRequiredService<IWebhookNotificationService>();
var settings = await settingsService.GetAsync();
var session = await dbContext.RecordSessions
@@ -783,6 +784,12 @@ public sealed partial class FfmpegService
$"exitCode={process.ExitCode}; output={effectiveOutputPath}",
session.LiveRoom,
currentTask);
await webhookNotificationService.SendExceptionAsync(
"FFmpeg",
"Recording session exited abnormally.",
$"exitCode={process.ExitCode}; output={effectiveOutputPath}",
session.LiveRoom,
currentTask);
}
}
@@ -1,4 +1,5 @@
using System.Collections.Concurrent;
using LiveRecorder.Application.Common;
using LiveRecorder.Application.Abstractions.Logging;
using LiveRecorder.Application.Abstractions.Notifications;
using LiveRecorder.Application.Abstractions.Platforms;
@@ -95,6 +96,7 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
using var notificationScope = _serviceScopeFactory.CreateScope();
var logService = notificationScope.ServiceProvider.GetRequiredService<ISystemLogService>();
var emailNotificationService = notificationScope.ServiceProvider.GetRequiredService<IEmailNotificationService>();
var webhookNotificationService = notificationScope.ServiceProvider.GetRequiredService<IWebhookNotificationService>();
await logService.WriteAsync(
SystemLogLevel.Error,
"Scheduler",
@@ -108,6 +110,11 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
"Background live room polling failed.",
ex.ToString(),
cancellationToken: stoppingToken);
await webhookNotificationService.SendExceptionAsync(
"Scheduler",
"Background live room polling failed.",
ex.ToString(),
cancellationToken: stoppingToken);
}
}
catch (Exception notificationEx)
@@ -133,6 +140,7 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
var liveRoomStatusService = scope.ServiceProvider.GetRequiredService<LiveRoomStatusService>();
var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
var emailNotificationService = scope.ServiceProvider.GetRequiredService<IEmailNotificationService>();
var webhookNotificationService = scope.ServiceProvider.GetRequiredService<IWebhookNotificationService>();
var storageGuardService = scope.ServiceProvider.GetRequiredService<IStorageGuardService>();
var liveRoom = await dbContext.LiveRooms.FirstOrDefaultAsync(item => item.Id == liveRoomId, cancellationToken);
@@ -175,12 +183,26 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
if (!settings.AutoStartRecordingOnLive)
{
await UpdateAutoStartDecisionAsync(
dbContext,
liveRoom,
AutoStartDecisionCodes.SkippedDisabled,
"Auto-start skipped because automatic start is disabled.",
detail: null,
cancellationToken);
return;
}
var startCheck = storageGuardService.CheckCanStartOrResume(settings);
if (!startCheck.HasEnoughSpace)
{
await UpdateAutoStartDecisionAsync(
dbContext,
liveRoom,
AutoStartDecisionCodes.SkippedStorage,
"Auto-start skipped because storage is below threshold.",
startCheck.Message,
cancellationToken);
await logService.WriteAsync(
SystemLogLevel.Warning,
"Storage",
@@ -216,6 +238,13 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
if (hasRunningSession)
{
await UpdateAutoStartDecisionAsync(
dbContext,
liveRoom,
AutoStartDecisionCodes.SkippedActiveSession,
"Auto-start skipped because an active recording session already exists.",
detail: null,
cancellationToken);
await logService.WriteAsync(
SystemLogLevel.Info,
"Scheduler",
@@ -238,6 +267,7 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
{
LiveRoomId = liveRoom.Id
},
trackAutoStartDecision: true,
cancellationToken);
}
catch (Exception ex)
@@ -263,6 +293,12 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
ex.ToString(),
liveRoom,
cancellationToken: cancellationToken);
await webhookNotificationService.SendExceptionAsync(
"Scheduler",
"Background polling failed for a live room.",
ex.ToString(),
liveRoom,
cancellationToken: cancellationToken);
}
}
}
@@ -515,6 +551,22 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
return exception.InnerException is not null && IsSqliteStorageFullException(exception.InnerException);
}
private static async Task UpdateAutoStartDecisionAsync(
LiveRecorderDbContext dbContext,
Domain.Entities.LiveRoom liveRoom,
string code,
string summary,
string? detail,
CancellationToken cancellationToken)
{
liveRoom.SetLastAutoStartDecision(
Truncate(code, 64),
Truncate(summary, 256),
Truncate(detail, 2048),
DateTimeOffset.UtcNow);
await SaveChangesWithRetryAsync(dbContext, cancellationToken);
}
private static async Task SaveChangesWithRetryAsync(LiveRecorderDbContext dbContext, CancellationToken cancellationToken)
{
for (var attempt = 1; attempt <= 5; attempt++)
@@ -539,4 +591,15 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
exception.SqliteErrorCode is 5 or 6 ||
exception.Message.Contains("database is locked", StringComparison.OrdinalIgnoreCase) ||
exception.Message.Contains("database table is locked", StringComparison.OrdinalIgnoreCase);
private static string? Truncate(string? value, int maxLength)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
var trimmed = value.Trim();
return trimmed.Length <= maxLength ? trimmed : trimmed[..maxLength];
}
}
@@ -0,0 +1,437 @@
using LiveRecorder.Application.Abstractions.Recording;
using LiveRecorder.Application.Abstractions.Settings;
using LiveRecorder.Application.Abstractions.Storage;
using LiveRecorder.Application.Common;
using LiveRecorder.Application.Models.Recovery;
using LiveRecorder.Application.Models.RecordTasks;
using LiveRecorder.Application.Services;
using LiveRecorder.Domain.Entities;
using LiveRecorder.Domain.Enums;
using LiveRecorder.Infrastructure.Persistence;
using Microsoft.EntityFrameworkCore;
namespace LiveRecorder.Infrastructure.Services;
public sealed class RecoveryService
{
private readonly LiveRecorderDbContext _dbContext;
private readonly ISystemSettingsService _systemSettingsService;
private readonly IStorageGuardService _storageGuardService;
private readonly IFfmpegService _ffmpegService;
private readonly RecordService _recordService;
public RecoveryService(
LiveRecorderDbContext dbContext,
ISystemSettingsService systemSettingsService,
IStorageGuardService storageGuardService,
IFfmpegService ffmpegService,
RecordService recordService)
{
_dbContext = dbContext;
_systemSettingsService = systemSettingsService;
_storageGuardService = storageGuardService;
_ffmpegService = ffmpegService;
_recordService = recordService;
}
public async Task<RecoveryOverviewDto> GetOverviewAsync(CancellationToken cancellationToken = default)
{
var settings = await _systemSettingsService.GetAsync(cancellationToken);
var storage = _storageGuardService.CheckCanStartOrResume(settings);
var liveRooms = await ListRecoverableLiveRoomsAsync(cancellationToken);
var finalizations = await ListRecoverableFinalizationsAsync(cancellationToken);
return new RecoveryOverviewDto
{
Storage = new StorageGuardStatusDto
{
IsEnabled = storage.IsEnabled,
HasEnoughSpace = storage.HasEnoughSpace,
CheckedPath = storage.CheckedPath,
AvailableBytes = storage.AvailableBytes,
RequiredBytes = storage.RequiredBytes,
Message = storage.Message
},
LiveRooms = liveRooms,
Finalizations = finalizations
};
}
public async Task<RecoveryActionResultDto> RetryLiveRoomAsync(Guid liveRoomId, CancellationToken cancellationToken = default)
{
var room = await _dbContext.LiveRooms.FirstOrDefaultAsync(item => item.Id == liveRoomId, cancellationToken);
if (room is null)
{
return FailureResult("Live room was not found.");
}
if (!room.IsEnabled)
{
return FailureResult("Live room is disabled and cannot be retried.");
}
if (room.AvailabilityStatus != LiveRoomAvailabilityStatus.Live)
{
return FailureResult("Live room is not currently online.");
}
var hasActiveSession = await _dbContext.RecordSessions.AnyAsync(
item => item.LiveRoomId == room.Id &&
(item.Status == RecordSessionStatus.Starting ||
item.Status == RecordSessionStatus.Running ||
item.Status == RecordSessionStatus.Stopping),
cancellationToken);
if (hasActiveSession)
{
room.SetLastAutoStartDecision(
AutoStartDecisionCodes.SkippedActiveSession,
"Auto-start skipped because an active recording session already exists.",
null,
DateTimeOffset.UtcNow);
await _dbContext.SaveChangesAsync(cancellationToken);
return FailureResult("An active recording session already exists for this live room.");
}
try
{
var task = await _recordService.StartAsync(
new StartRecordTaskRequest
{
LiveRoomId = liveRoomId
},
trackAutoStartDecision: true,
cancellationToken);
var started = task.Status is RecordTaskStatus.Starting or RecordTaskStatus.Running;
return new RecoveryActionResultDto
{
RequestedCount = 1,
SuccessCount = started ? 1 : 0,
FailedCount = started ? 0 : 1,
Messages =
[
started
? $"Recording retry started for room {room.RoomId}."
: $"Recording retry did not start for room {room.RoomId}. Status={task.Status}; Error={task.ErrorMessage ?? "n/a"}"
]
};
}
catch (Exception ex)
{
return FailureResult($"Recording retry failed for room {room.RoomId}: {ex.Message}");
}
}
public async Task<RecoveryActionResultDto> RetryAllLiveRoomsAsync(CancellationToken cancellationToken = default)
{
var liveRooms = await ListRecoverableLiveRoomsAsync(cancellationToken);
if (liveRooms.Count == 0)
{
return new RecoveryActionResultDto
{
RequestedCount = 0,
SuccessCount = 0,
FailedCount = 0,
Messages = ["No live rooms currently require retry."]
};
}
var messages = new List<string>();
var successCount = 0;
foreach (var item in liveRooms)
{
var result = await RetryLiveRoomAsync(item.LiveRoomId, cancellationToken);
successCount += result.SuccessCount;
messages.AddRange(result.Messages);
}
return new RecoveryActionResultDto
{
RequestedCount = liveRooms.Count,
SuccessCount = successCount,
FailedCount = liveRooms.Count - successCount,
Messages = messages
};
}
public async Task<RecoveryActionResultDto> ResumeFinalizationAsync(Guid recordTaskId, CancellationToken cancellationToken = default)
{
var started = await _ffmpegService.StartManualFinalizeTaskAsync(recordTaskId, cancellationToken);
return new RecoveryActionResultDto
{
RequestedCount = 1,
SuccessCount = started ? 1 : 0,
FailedCount = started ? 0 : 1,
Messages =
[
started
? $"MP4 finalization resumed for task {recordTaskId}."
: $"MP4 finalization could not be resumed for task {recordTaskId}."
]
};
}
public async Task<RecoveryActionResultDto> ResumeAllFinalizationsAsync(CancellationToken cancellationToken = default)
{
var finalizations = await ListRecoverableFinalizationsAsync(cancellationToken);
if (finalizations.Count == 0)
{
return new RecoveryActionResultDto
{
RequestedCount = 0,
SuccessCount = 0,
FailedCount = 0,
Messages = ["No MP4 finalization tasks currently require recovery."]
};
}
var successCount = 0;
var messages = new List<string>();
foreach (var item in finalizations)
{
var started = await _ffmpegService.StartManualFinalizeTaskAsync(item.RecordTaskId, cancellationToken);
if (started)
{
successCount++;
}
messages.Add(
started
? $"MP4 finalization resumed for task {item.RecordTaskId}."
: $"MP4 finalization could not be resumed for task {item.RecordTaskId}.");
}
return new RecoveryActionResultDto
{
RequestedCount = finalizations.Count,
SuccessCount = successCount,
FailedCount = finalizations.Count - successCount,
Messages = messages
};
}
private async Task<IReadOnlyList<RecoverableLiveRoomDto>> ListRecoverableLiveRoomsAsync(CancellationToken cancellationToken)
{
var activeLiveRoomIds = await _dbContext.RecordSessions
.AsNoTracking()
.Where(item => item.Status == RecordSessionStatus.Starting ||
item.Status == RecordSessionStatus.Running ||
item.Status == RecordSessionStatus.Stopping)
.Select(item => item.LiveRoomId)
.Distinct()
.ToListAsync(cancellationToken);
var rooms = await _dbContext.LiveRooms
.AsNoTracking()
.Where(item => item.IsEnabled &&
item.AvailabilityStatus == LiveRoomAvailabilityStatus.Live &&
item.LastAutoStartDecisionCode != AutoStartDecisionCodes.Started)
.ToListAsync(cancellationToken);
return rooms
.Where(item => !activeLiveRoomIds.Contains(item.Id))
.OrderByDescending(item => item.LastAutoStartDecisionAt ?? item.LastCheckedAt ?? item.UpdatedAt)
.Select(item => new RecoverableLiveRoomDto
{
LiveRoomId = item.Id,
PlatformName = item.Platform.ToString(),
RoomId = item.RoomId,
Title = item.Title,
AnchorName = item.AnchorName,
LastAutoStartDecisionCode = item.LastAutoStartDecisionCode,
LastAutoStartDecisionSummary = item.LastAutoStartDecisionSummary,
LastAutoStartDecisionDetail = item.LastAutoStartDecisionDetail,
LastAutoStartDecisionAt = item.LastAutoStartDecisionAt,
LastCheckedAt = item.LastCheckedAt
})
.ToList();
}
private async Task<IReadOnlyList<RecoverableFinalizationDto>> ListRecoverableFinalizationsAsync(CancellationToken cancellationToken)
{
var tasks = await _dbContext.RecordTasks
.AsNoTracking()
.Include(item => item.LiveRoom)
.Include(item => item.RecordSession)
.Include(item => item.Result)
.Where(item => item.OutputFormat == RecordOutputFormat.Mp4 &&
item.RecordSession != null)
.ToListAsync(cancellationToken);
return tasks
.Where(IsRecoverableFinalization)
.OrderByDescending(item => item.UpdatedAt)
.Select(item => new RecoverableFinalizationDto
{
RecordTaskId = item.Id,
RecordSessionId = item.RecordSessionId,
LiveRoomId = item.LiveRoomId,
LiveRoomTitle = item.LiveRoom?.Title ?? item.LiveRoom?.AnchorName ?? item.LiveRoom?.RoomId ?? item.LiveRoomId.ToString(),
RoomId = item.LiveRoom?.RoomId ?? "-",
PlatformName = item.LiveRoom?.Platform.ToString() ?? "Unknown",
SegmentIndex = item.SegmentIndex,
Status = item.Status,
OutputFilePath = item.OutputFilePath,
Reason = BuildFinalizationReason(item),
CreatedAt = item.CreatedAt,
EndedAt = item.EndedAt
})
.ToList();
}
private static bool IsRecoverableFinalization(RecordTask recordTask)
{
if (recordTask.RecordSession is null ||
recordTask.OutputFormat != RecordOutputFormat.Mp4)
{
return false;
}
if (recordTask.Status == RecordTaskStatus.Processing)
{
return true;
}
if (recordTask.Status is RecordTaskStatus.Starting or RecordTaskStatus.Running or RecordTaskStatus.Stopping)
{
return false;
}
if (recordTask.RecordSession.Status is RecordSessionStatus.Starting or RecordSessionStatus.Running or RecordSessionStatus.Stopping)
{
return false;
}
if (string.IsNullOrWhiteSpace(recordTask.OutputFilePath))
{
return false;
}
var recorderOutputPath = ResolveManualFinalizeSourcePath(recordTask, recordTask.RecordSession);
if (!File.Exists(recorderOutputPath))
{
return false;
}
if (recordTask.Status == RecordTaskStatus.Completed)
{
var finalOutputPath = NormalizeAbsolutePath(recordTask.OutputFilePath);
return !HasUsableOutput(finalOutputPath, CalculateFileSize(finalOutputPath));
}
return true;
}
private static string BuildFinalizationReason(RecordTask recordTask)
{
if (recordTask.Status == RecordTaskStatus.Processing)
{
return string.IsNullOrWhiteSpace(recordTask.ErrorMessage)
? "MP4 finalization is queued or paused and can be resumed."
: recordTask.ErrorMessage!;
}
if (recordTask.Status == RecordTaskStatus.Completed)
{
return "The final MP4 output is missing, but the intermediate recording file is still available.";
}
return "Manual MP4 finalization can be retried from the intermediate recording file.";
}
private static string ResolveManualFinalizeSourcePath(RecordTask recordTask, RecordSession recordSession)
{
var resultPath = recordTask.Result?.FilePath;
if (!string.IsNullOrWhiteSpace(resultPath) &&
resultPath.EndsWith(".ts", StringComparison.OrdinalIgnoreCase))
{
var normalizedResultPath = NormalizeAbsolutePath(resultPath);
if (File.Exists(normalizedResultPath))
{
return normalizedResultPath;
}
}
return NormalizeAbsolutePath(
GetRecorderOutputPath(
recordTask.OutputFilePath ?? resultPath ?? string.Empty,
recordSession.OutputFormat,
recordSession.SaveMode));
}
private static string GetRecorderOutputPath(
string finalOutputPath,
RecordOutputFormat outputFormat,
RecordSaveMode saveMode)
{
if (outputFormat != RecordOutputFormat.Mp4)
{
return finalOutputPath;
}
if (saveMode == RecordSaveMode.SingleFile)
{
return Path.Combine(
Path.GetDirectoryName(finalOutputPath)!,
$"{Path.GetFileNameWithoutExtension(finalOutputPath)}.recording.ts");
}
return Path.ChangeExtension(finalOutputPath, ".ts");
}
private static long? CalculateFileSize(string? outputPath)
{
if (string.IsNullOrWhiteSpace(outputPath))
{
return null;
}
if (File.Exists(outputPath))
{
return new FileInfo(outputPath).Length;
}
if (Directory.Exists(outputPath))
{
return new DirectoryInfo(outputPath)
.EnumerateFiles("*", SearchOption.TopDirectoryOnly)
.Sum(static file => file.Length);
}
return null;
}
private static bool HasUsableOutput(string? outputPath, long? fileSize)
{
if (string.IsNullOrWhiteSpace(outputPath))
{
return false;
}
if (File.Exists(outputPath))
{
return fileSize.GetValueOrDefault() > 0;
}
if (Directory.Exists(outputPath))
{
return Directory.EnumerateFiles(outputPath, "*", SearchOption.TopDirectoryOnly).Any();
}
return false;
}
private static string NormalizeAbsolutePath(string path) =>
Path.IsPathRooted(path)
? path
: Path.GetFullPath(path, AppContext.BaseDirectory);
private static RecoveryActionResultDto FailureResult(string message) => new()
{
RequestedCount = 1,
SuccessCount = 0,
FailedCount = 1,
Messages = [message]
};
}
@@ -0,0 +1,76 @@
using LiveRecorder.Application.Abstractions.Logging;
using LiveRecorder.Application.Abstractions.Settings;
using LiveRecorder.Domain.Enums;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace LiveRecorder.Infrastructure.Services;
public sealed class RetentionCleanupBackgroundService : BackgroundService
{
private static readonly TimeSpan CleanupInterval = TimeSpan.FromHours(24);
private readonly IServiceScopeFactory _serviceScopeFactory;
private readonly ILogger<RetentionCleanupBackgroundService> _logger;
public RetentionCleanupBackgroundService(
IServiceScopeFactory serviceScopeFactory,
ILogger<RetentionCleanupBackgroundService> logger)
{
_serviceScopeFactory = serviceScopeFactory;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
try
{
using var scope = _serviceScopeFactory.CreateScope();
var settingsService = scope.ServiceProvider.GetRequiredService<ISystemSettingsService>();
var cleanupService = scope.ServiceProvider.GetRequiredService<RetentionCleanupService>();
var settings = await settingsService.GetAsync(stoppingToken);
if (settings.EnableRetentionCleanup)
{
await cleanupService.RunAsync(ignoreEnabledSetting: false, stoppingToken);
}
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception ex)
{
_logger.LogError(ex, "Retention cleanup background task failed");
try
{
using var scope = _serviceScopeFactory.CreateScope();
var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
await logService.WriteAsync(
SystemLogLevel.Error,
"Retention",
"Retention cleanup background task failed.",
ex.ToString(),
cancellationToken: CancellationToken.None);
}
catch (Exception logEx)
{
_logger.LogWarning(logEx, "Failed to persist retention cleanup background error log");
}
}
try
{
await Task.Delay(CleanupInterval, stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
}
}
}
@@ -0,0 +1,283 @@
using LiveRecorder.Application.Abstractions.Logging;
using LiveRecorder.Application.Abstractions.Settings;
using LiveRecorder.Application.Models.Settings;
using LiveRecorder.Domain.Entities;
using LiveRecorder.Domain.Enums;
using LiveRecorder.Infrastructure.Persistence;
using Microsoft.EntityFrameworkCore;
namespace LiveRecorder.Infrastructure.Services;
public sealed class RetentionCleanupService
{
private readonly LiveRecorderDbContext _dbContext;
private readonly ISystemSettingsService _systemSettingsService;
private readonly ISystemLogService _systemLogService;
public RetentionCleanupService(
LiveRecorderDbContext dbContext,
ISystemSettingsService systemSettingsService,
ISystemLogService systemLogService)
{
_dbContext = dbContext;
_systemSettingsService = systemSettingsService;
_systemLogService = systemLogService;
}
public async Task<RetentionCleanupResultDto> RunAsync(
bool ignoreEnabledSetting = false,
CancellationToken cancellationToken = default)
{
var settings = await _systemSettingsService.GetAsync(cancellationToken);
if (!settings.EnableRetentionCleanup && !ignoreEnabledSetting)
{
return CreateEmptyResult();
}
var warnings = new List<string>();
var deletedFilePaths = new List<string>();
var deletedDanmakuPaths = new List<string>();
var deletedTaskIds = new HashSet<Guid>();
var deletedSessionIds = new HashSet<Guid>();
var deletedResultIds = new HashSet<Guid>();
var deletedLogIds = new HashSet<Guid>();
var cutoff = DateTimeOffset.UtcNow.AddDays(-Math.Max(1, settings.RetentionDays));
var staleTasks = await _dbContext.RecordTasks
.Include(item => item.LiveRoom)
.Include(item => item.RecordSession)
.Include(item => item.Result)
.Where(item => item.CreatedAt < cutoff &&
item.Status != RecordTaskStatus.Starting &&
item.Status != RecordTaskStatus.Running &&
item.Status != RecordTaskStatus.Stopping &&
item.Status != RecordTaskStatus.Processing)
.ToListAsync(cancellationToken);
foreach (var task in staleTasks)
{
if (settings.RetentionDeleteFiles)
{
TryDeleteRecordOutput(task, warnings, deletedFilePaths, deletedDanmakuPaths);
}
var taskLogs = await _dbContext.SystemLogEntries
.Where(item => item.RecordTaskId == task.Id)
.ToListAsync(cancellationToken);
foreach (var log in taskLogs)
{
deletedLogIds.Add(log.Id);
}
if (task.Result is not null)
{
deletedResultIds.Add(task.Result.Id);
_dbContext.RecordResults.Remove(task.Result);
}
if (taskLogs.Count > 0)
{
_dbContext.SystemLogEntries.RemoveRange(taskLogs);
}
deletedTaskIds.Add(task.Id);
_dbContext.RecordTasks.Remove(task);
}
if (deletedTaskIds.Count > 0 || deletedResultIds.Count > 0 || deletedLogIds.Count > 0)
{
await _dbContext.SaveChangesAsync(cancellationToken);
}
var staleSessions = await _dbContext.RecordSessions
.Include(item => item.RecordTasks)
.Where(item => item.CreatedAt < cutoff &&
item.Status != RecordSessionStatus.Starting &&
item.Status != RecordSessionStatus.Running &&
item.Status != RecordSessionStatus.Stopping)
.ToListAsync(cancellationToken);
foreach (var session in staleSessions.Where(static item => item.RecordTasks.Count == 0))
{
var sessionLogs = await _dbContext.SystemLogEntries
.Where(item => item.RecordSessionId == session.Id)
.ToListAsync(cancellationToken);
foreach (var log in sessionLogs)
{
deletedLogIds.Add(log.Id);
}
if (sessionLogs.Count > 0)
{
_dbContext.SystemLogEntries.RemoveRange(sessionLogs);
}
deletedSessionIds.Add(session.Id);
_dbContext.RecordSessions.Remove(session);
}
var staleGlobalLogs = await _dbContext.SystemLogEntries
.Where(item => item.CreatedAt < cutoff)
.ToListAsync(cancellationToken);
foreach (var log in staleGlobalLogs)
{
deletedLogIds.Add(log.Id);
}
if (staleGlobalLogs.Count > 0)
{
_dbContext.SystemLogEntries.RemoveRange(staleGlobalLogs);
}
if (deletedSessionIds.Count > 0 || staleGlobalLogs.Count > 0)
{
await _dbContext.SaveChangesAsync(cancellationToken);
}
var result = new RetentionCleanupResultDto
{
DeletedSessionCount = deletedSessionIds.Count,
DeletedTaskCount = deletedTaskIds.Count,
DeletedResultCount = deletedResultIds.Count,
DeletedLogCount = deletedLogIds.Count,
DeletedFileCount = deletedFilePaths.Count,
DeletedDanmakuFileCount = deletedDanmakuPaths.Count,
Warnings = warnings
};
if (deletedSessionIds.Count > 0 ||
deletedTaskIds.Count > 0 ||
deletedResultIds.Count > 0 ||
deletedLogIds.Count > 0 ||
deletedFilePaths.Count > 0 ||
deletedDanmakuPaths.Count > 0 ||
warnings.Count > 0)
{
await _systemLogService.WriteAsync(
SystemLogLevel.Info,
"Retention",
"Retention cleanup completed.",
$"sessions={result.DeletedSessionCount}; tasks={result.DeletedTaskCount}; results={result.DeletedResultCount}; logs={result.DeletedLogCount}; video-files={result.DeletedFileCount}; danmaku-files={result.DeletedDanmakuFileCount}; warnings={warnings.Count}",
cancellationToken: cancellationToken);
}
return result;
}
private static RetentionCleanupResultDto CreateEmptyResult() => new()
{
DeletedSessionCount = 0,
DeletedTaskCount = 0,
DeletedResultCount = 0,
DeletedLogCount = 0,
DeletedFileCount = 0,
DeletedDanmakuFileCount = 0,
Warnings = []
};
private static void TryDeleteRecordOutput(
RecordTask recordTask,
List<string> warnings,
List<string> deletedFilePaths,
List<string> deletedDanmakuPaths)
{
var outputPath = recordTask.Result?.FilePath ?? recordTask.OutputFilePath;
if (!string.IsNullOrWhiteSpace(outputPath))
{
TryDeletePath(outputPath, warnings, deletedFilePaths, $"output for task {recordTask.Id}");
TryDeleteIntermediateRecordingArtifacts(outputPath, recordTask.OutputFormat, warnings, deletedFilePaths, recordTask.Id);
}
var danmakuPath = recordTask.Result?.DanmakuFilePath;
if (!string.IsNullOrWhiteSpace(danmakuPath))
{
TryDeletePath(danmakuPath, warnings, deletedDanmakuPaths, $"danmaku for task {recordTask.Id}");
}
}
private static void TryDeleteIntermediateRecordingArtifacts(
string finalOutputPath,
RecordOutputFormat outputFormat,
List<string> warnings,
List<string> deletedFilePaths,
Guid recordTaskId)
{
if (outputFormat != RecordOutputFormat.Mp4)
{
return;
}
var absoluteFinalPath = Path.IsPathRooted(finalOutputPath)
? finalOutputPath
: Path.GetFullPath(finalOutputPath, AppContext.BaseDirectory);
var intermediateCandidates = new[]
{
Path.ChangeExtension(absoluteFinalPath, ".ts"),
Path.Combine(
Path.GetDirectoryName(absoluteFinalPath) ?? string.Empty,
$"{Path.GetFileNameWithoutExtension(absoluteFinalPath)}.recording.ts")
};
foreach (var candidate in intermediateCandidates
.Where(static path => !string.IsNullOrWhiteSpace(path))
.Distinct(StringComparer.OrdinalIgnoreCase))
{
if (string.Equals(candidate, absoluteFinalPath, StringComparison.OrdinalIgnoreCase) || !File.Exists(candidate))
{
continue;
}
TryDeletePath(candidate, warnings, deletedFilePaths, $"intermediate output for task {recordTaskId}");
}
}
private static void TryDeletePath(
string path,
List<string> warnings,
List<string> deletedPaths,
string label)
{
var absolutePath = Path.IsPathRooted(path)
? path
: Path.GetFullPath(path, AppContext.BaseDirectory);
try
{
if (IsUnsafeDeletionTarget(absolutePath))
{
warnings.Add($"Skipped deleting suspicious path: {absolutePath}");
return;
}
if (File.Exists(absolutePath))
{
File.Delete(absolutePath);
deletedPaths.Add(absolutePath);
return;
}
if (Directory.Exists(absolutePath))
{
Directory.Delete(absolutePath, true);
deletedPaths.Add(absolutePath);
return;
}
warnings.Add($"Path not found for {label}: {absolutePath}");
}
catch (Exception ex)
{
warnings.Add($"Failed to delete {label}: {ex.Message}");
}
}
private static bool IsUnsafeDeletionTarget(string absolutePath)
{
var normalized = absolutePath.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
var root = Path.GetPathRoot(normalized)?.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
return string.IsNullOrWhiteSpace(normalized) ||
normalized.Length < 4 ||
string.Equals(normalized, root, StringComparison.OrdinalIgnoreCase);
}
}
@@ -0,0 +1,337 @@
using System.Net.Http.Json;
using System.Text;
using System.Text.Json;
using LiveRecorder.Application.Abstractions.Logging;
using LiveRecorder.Application.Abstractions.Notifications;
using LiveRecorder.Application.Abstractions.Settings;
using LiveRecorder.Application.Models.Settings;
using LiveRecorder.Domain.Entities;
using Microsoft.Extensions.Logging;
namespace LiveRecorder.Infrastructure.Services;
public sealed class WebhookNotificationService : IWebhookNotificationService
{
private const string AppName = "LiveRecorder";
private readonly IHttpClientFactory _httpClientFactory;
private readonly ISystemSettingsService _systemSettingsService;
private readonly ISystemLogService _systemLogService;
private readonly ILogger<WebhookNotificationService> _logger;
public WebhookNotificationService(
IHttpClientFactory httpClientFactory,
ISystemSettingsService systemSettingsService,
ISystemLogService systemLogService,
ILogger<WebhookNotificationService> logger)
{
_httpClientFactory = httpClientFactory;
_systemSettingsService = systemSettingsService;
_systemLogService = systemLogService;
_logger = logger;
}
public async Task SendLiveStartedAsync(LiveRoom liveRoom, CancellationToken cancellationToken = default)
{
var settings = await _systemSettingsService.GetAsync(cancellationToken);
if (!settings.EnableWebhookNotification || !settings.NotifyWebhookOnLiveStarted)
{
return;
}
var payload = BuildPayload(
"live_started",
$"Live started: {liveRoom.AnchorName ?? liveRoom.RoomId}",
liveRoom.SourceUrl,
liveRoom,
recordTask: null);
await SendConfiguredWebhookAsync(
settings,
payload,
"Webhook notification sent for live_started.",
"Webhook notification failed for live_started.",
liveRoom.Id,
recordSessionId: null,
recordTaskId: null,
cancellationToken);
}
public async Task SendExceptionAsync(
string source,
string summary,
string? detail = null,
LiveRoom? liveRoom = null,
RecordTask? recordTask = null,
CancellationToken cancellationToken = default)
{
var settings = await _systemSettingsService.GetAsync(cancellationToken);
if (!settings.EnableWebhookNotification || !settings.NotifyWebhookOnException)
{
return;
}
var payload = BuildPayload(
"exception",
summary,
detail,
liveRoom,
recordTask,
source);
await SendConfiguredWebhookAsync(
settings,
payload,
"Webhook notification sent for exception.",
"Webhook notification failed for exception.",
liveRoom?.Id,
recordSessionId: recordTask?.RecordSessionId,
recordTaskId: recordTask?.Id,
cancellationToken);
}
public async Task<WebhookTestResultDto> SendTestAsync(
SendTestWebhookRequest request,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(request);
var settings = new SystemSettingsDto
{
EnableWebhookNotification = true,
WebhookUrl = request.WebhookUrl.Trim(),
WebhookHeaders = request.WebhookHeaders,
WebhookTimeoutSeconds = request.WebhookTimeoutSeconds,
NotifyWebhookOnLiveStarted = true,
NotifyWebhookOnException = true
};
var payload = BuildPayload(
"live_started",
"Webhook test from LiveRecorder.",
"This is a sample webhook payload generated from the settings test action.",
new LiveRoom(
Domain.Enums.LivePlatformType.Douyin,
"https://live.douyin.com/676493068539",
"676493068539",
"https://live.douyin.com/676493068539",
DateTimeOffset.UtcNow),
recordTask: null);
try
{
var result = await SendInternalAsync(settings, payload, cancellationToken);
await _systemLogService.WriteAsync(
result.Success ? Domain.Enums.SystemLogLevel.Info : Domain.Enums.SystemLogLevel.Warning,
"Webhook",
result.Success
? "Webhook test completed successfully."
: "Webhook test failed.",
result.Detail,
cancellationToken: cancellationToken);
return new WebhookTestResultDto
{
Success = result.Success,
Message = result.Success
? "Webhook test completed successfully."
: "Webhook test failed.",
Detail = result.Detail
};
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Webhook test failed");
await _systemLogService.WriteAsync(
Domain.Enums.SystemLogLevel.Warning,
"Webhook",
"Webhook test failed.",
ex.ToString(),
cancellationToken: cancellationToken);
return new WebhookTestResultDto
{
Success = false,
Message = "Webhook test failed.",
Detail = ex.Message
};
}
}
private async Task SendConfiguredWebhookAsync(
SystemSettingsDto settings,
object payload,
string successMessage,
string failureMessage,
Guid? liveRoomId,
Guid? recordSessionId,
Guid? recordTaskId,
CancellationToken cancellationToken)
{
try
{
var result = await SendInternalAsync(settings, payload, cancellationToken);
if (!result.Success)
{
await _systemLogService.WriteAsync(
Domain.Enums.SystemLogLevel.Warning,
"Webhook",
failureMessage,
result.Detail,
liveRoomId,
recordSessionId,
recordTaskId,
cancellationToken);
return;
}
await _systemLogService.WriteAsync(
Domain.Enums.SystemLogLevel.Info,
"Webhook",
successMessage,
result.Detail,
liveRoomId,
recordSessionId,
recordTaskId,
cancellationToken);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "{FailureMessage}", failureMessage);
await _systemLogService.WriteAsync(
Domain.Enums.SystemLogLevel.Warning,
"Webhook",
failureMessage,
ex.ToString(),
liveRoomId,
recordSessionId,
recordTaskId,
cancellationToken);
}
}
private async Task<WebhookSendResult> SendInternalAsync(
SystemSettingsDto settings,
object payload,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(settings.WebhookUrl))
{
throw new InvalidOperationException("Webhook URL is required.");
}
var client = _httpClientFactory.CreateClient();
client.Timeout = TimeSpan.FromSeconds(Math.Clamp(settings.WebhookTimeoutSeconds, 1, 300));
using var request = new HttpRequestMessage(HttpMethod.Post, settings.WebhookUrl.Trim())
{
Content = JsonContent.Create(payload)
};
foreach (var header in ParseHeaders(settings.WebhookHeaders))
{
if (!request.Headers.TryAddWithoutValidation(header.Key, header.Value))
{
request.Content?.Headers.TryAddWithoutValidation(header.Key, header.Value);
}
}
using var response = await client.SendAsync(request, cancellationToken);
var responseBody = await response.Content.ReadAsStringAsync(cancellationToken);
var detail = BuildResponseDetail(settings.WebhookUrl, response, responseBody);
return new WebhookSendResult(response.IsSuccessStatusCode, detail);
}
private static object BuildPayload(
string eventType,
string summary,
string? detail,
LiveRoom? liveRoom,
RecordTask? recordTask,
string? source = null)
{
return new
{
appName = AppName,
eventType,
sentAtUtc = DateTimeOffset.UtcNow,
summary,
detail,
source,
liveRoom = liveRoom is null
? null
: new
{
id = liveRoom.Id,
platform = liveRoom.Platform.ToString(),
roomId = liveRoom.RoomId,
title = liveRoom.Title,
anchorName = liveRoom.AnchorName,
sourceUrl = liveRoom.SourceUrl
},
recordTask = recordTask is null
? null
: new
{
id = recordTask.Id,
recordSessionId = recordTask.RecordSessionId,
status = recordTask.Status.ToString(),
segmentIndex = recordTask.SegmentIndex,
outputFilePath = recordTask.OutputFilePath
}
};
}
private static IReadOnlyList<KeyValuePair<string, string>> ParseHeaders(string rawHeaders)
{
if (string.IsNullOrWhiteSpace(rawHeaders))
{
return [];
}
var results = new List<KeyValuePair<string, string>>();
var lines = rawHeaders
.Replace("\r\n", "\n", StringComparison.Ordinal)
.Replace('\r', '\n')
.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
foreach (var line in lines)
{
var separatorIndex = line.IndexOf(':');
if (separatorIndex <= 0)
{
throw new InvalidOperationException($"Invalid webhook header format: {line}");
}
var name = line[..separatorIndex].Trim();
var value = line[(separatorIndex + 1)..].Trim();
if (string.IsNullOrWhiteSpace(name))
{
throw new InvalidOperationException($"Invalid webhook header format: {line}");
}
results.Add(new KeyValuePair<string, string>(name, value));
}
return results;
}
private static string BuildResponseDetail(string webhookUrl, HttpResponseMessage response, string responseBody)
{
var builder = new StringBuilder();
builder.Append("url=").Append(webhookUrl.Trim());
builder.Append("; status=").Append((int)response.StatusCode);
builder.Append(' ').Append(response.ReasonPhrase);
var normalizedBody = responseBody.Trim();
if (!string.IsNullOrWhiteSpace(normalizedBody))
{
var truncatedBody = normalizedBody.Length <= 1000 ? normalizedBody : normalizedBody[..1000];
builder.Append("; body=").Append(truncatedBody);
}
return builder.ToString();
}
private sealed record WebhookSendResult(bool Success, string Detail);
}