feat: add recovery center and daily reviews
This commit is contained in:
@@ -11,6 +11,10 @@ namespace LiveRecorder.Infrastructure.Services;
|
||||
|
||||
public sealed class EventScriptService : IEventScriptService
|
||||
{
|
||||
private const string ScriptLogPathEnvironmentVariable = "LIVE_RECORDER_SCRIPT_LOG_PATH";
|
||||
private const int SystemLogDetailMaxLength = 4000;
|
||||
private const string TruncatedDetailSuffix = "... [truncated to fit system log detail limit]";
|
||||
|
||||
private readonly ISystemSettingsService _settingsService;
|
||||
private readonly ISystemLogService _systemLogService;
|
||||
private readonly ILogger<EventScriptService> _logger;
|
||||
@@ -38,6 +42,7 @@ public sealed class EventScriptService : IEventScriptService
|
||||
settings.EventScriptTimeoutSeconds,
|
||||
"live_started",
|
||||
environment,
|
||||
"Script",
|
||||
liveRoom.Id,
|
||||
recordSessionId: null,
|
||||
recordTaskId: null,
|
||||
@@ -57,6 +62,7 @@ public sealed class EventScriptService : IEventScriptService
|
||||
settings.EventScriptTimeoutSeconds,
|
||||
"live_ended",
|
||||
environment,
|
||||
"Script",
|
||||
liveRoom.Id,
|
||||
recordSessionId: null,
|
||||
recordTaskId: null,
|
||||
@@ -93,12 +99,61 @@ public sealed class EventScriptService : IEventScriptService
|
||||
settings.EventScriptTimeoutSeconds,
|
||||
"segment_completed",
|
||||
environment,
|
||||
"Script",
|
||||
liveRoom?.Id ?? recordSession.LiveRoomId,
|
||||
recordSession.Id,
|
||||
recordTask.Id,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<EventScriptTestResultDto> TestAsync(
|
||||
TestEventScriptRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
var eventName = NormalizeEventType(request.EventType);
|
||||
if (eventName is null)
|
||||
{
|
||||
var result = new EventScriptTestResultDto
|
||||
{
|
||||
Success = false,
|
||||
Message = "Unsupported event script test type.",
|
||||
Detail = request.EventType
|
||||
};
|
||||
|
||||
await _systemLogService.WriteAsync(
|
||||
SystemLogLevel.Warning,
|
||||
"ScriptTest",
|
||||
result.Message,
|
||||
result.Detail,
|
||||
cancellationToken: cancellationToken);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
var outcome = await ExecuteAsync(
|
||||
request.ScriptMode,
|
||||
request.ScriptPath,
|
||||
request.ScriptContent,
|
||||
request.TimeoutSeconds,
|
||||
eventName,
|
||||
BuildTestEnvironment(eventName, DateTimeOffset.UtcNow),
|
||||
"ScriptTest",
|
||||
liveRoomId: null,
|
||||
recordSessionId: null,
|
||||
recordTaskId: null,
|
||||
cancellationToken);
|
||||
|
||||
return new EventScriptTestResultDto
|
||||
{
|
||||
Success = outcome.Success,
|
||||
Message = outcome.Message,
|
||||
Detail = outcome.Detail,
|
||||
CustomLogOutput = outcome.CustomLogOutput
|
||||
};
|
||||
}
|
||||
|
||||
private async Task RunAsync(
|
||||
bool enabled,
|
||||
string scriptMode,
|
||||
@@ -107,6 +162,7 @@ public sealed class EventScriptService : IEventScriptService
|
||||
int timeoutSeconds,
|
||||
string eventName,
|
||||
IReadOnlyDictionary<string, string> environment,
|
||||
string logCategory,
|
||||
Guid? liveRoomId,
|
||||
Guid? recordSessionId,
|
||||
Guid? recordTaskId,
|
||||
@@ -117,24 +173,70 @@ public sealed class EventScriptService : IEventScriptService
|
||||
return;
|
||||
}
|
||||
|
||||
await ExecuteAsync(
|
||||
scriptMode,
|
||||
scriptPath,
|
||||
scriptContent,
|
||||
timeoutSeconds,
|
||||
eventName,
|
||||
environment,
|
||||
logCategory,
|
||||
liveRoomId,
|
||||
recordSessionId,
|
||||
recordTaskId,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<ScriptExecutionOutcome> ExecuteAsync(
|
||||
string scriptMode,
|
||||
string scriptPath,
|
||||
string scriptContent,
|
||||
int timeoutSeconds,
|
||||
string eventName,
|
||||
IReadOnlyDictionary<string, string> environment,
|
||||
string logCategory,
|
||||
Guid? liveRoomId,
|
||||
Guid? recordSessionId,
|
||||
Guid? recordTaskId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var execution = CreateExecution(scriptMode, scriptPath, scriptContent);
|
||||
if (execution is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
var missingConfiguration = new ScriptExecutionOutcome(
|
||||
false,
|
||||
$"Event script was not configured for {eventName}.",
|
||||
null,
|
||||
null);
|
||||
|
||||
if (execution.IsMissing)
|
||||
{
|
||||
await _systemLogService.WriteAsync(
|
||||
await WriteOutcomeLogAsync(
|
||||
missingConfiguration,
|
||||
logCategory,
|
||||
SystemLogLevel.Warning,
|
||||
"Script",
|
||||
$"Event script was not found for {eventName}.",
|
||||
execution.Detail,
|
||||
liveRoomId,
|
||||
recordSessionId,
|
||||
recordTaskId,
|
||||
cancellationToken);
|
||||
return;
|
||||
return missingConfiguration;
|
||||
}
|
||||
|
||||
if (execution.IsMissing)
|
||||
{
|
||||
var missingScript = new ScriptExecutionOutcome(
|
||||
false,
|
||||
$"Event script was not found for {eventName}.",
|
||||
execution.Detail,
|
||||
null);
|
||||
|
||||
await WriteOutcomeLogAsync(
|
||||
missingScript,
|
||||
logCategory,
|
||||
SystemLogLevel.Warning,
|
||||
liveRoomId,
|
||||
recordSessionId,
|
||||
recordTaskId,
|
||||
cancellationToken);
|
||||
return missingScript;
|
||||
}
|
||||
|
||||
var startInfo = execution.StartInfo!;
|
||||
@@ -143,12 +245,25 @@ public sealed class EventScriptService : IEventScriptService
|
||||
startInfo.Environment[pair.Key] = pair.Value;
|
||||
}
|
||||
|
||||
var scriptLogPath = CreateScriptLogPath();
|
||||
try
|
||||
{
|
||||
await File.WriteAllTextAsync(scriptLogPath, string.Empty, CancellationToken.None);
|
||||
startInfo.Environment[ScriptLogPathEnvironmentVariable] = scriptLogPath;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to prepare event script log file for {EventName}", eventName);
|
||||
}
|
||||
|
||||
using var process = new Process
|
||||
{
|
||||
StartInfo = startInfo,
|
||||
EnableRaisingEvents = true
|
||||
};
|
||||
|
||||
ScriptExecutionOutcome outcome;
|
||||
SystemLogLevel outcomeLevel;
|
||||
try
|
||||
{
|
||||
process.Start();
|
||||
@@ -156,44 +271,59 @@ public sealed class EventScriptService : IEventScriptService
|
||||
timeoutCts.CancelAfter(TimeSpan.FromSeconds(Math.Clamp(timeoutSeconds, 1, 3600)));
|
||||
await process.WaitForExitAsync(timeoutCts.Token);
|
||||
|
||||
await _systemLogService.WriteAsync(
|
||||
process.ExitCode == 0 ? SystemLogLevel.Info : SystemLogLevel.Warning,
|
||||
"Script",
|
||||
outcome = new ScriptExecutionOutcome(
|
||||
process.ExitCode == 0,
|
||||
process.ExitCode == 0
|
||||
? $"Event script completed for {eventName}."
|
||||
: $"Event script exited with code {process.ExitCode} for {eventName}.",
|
||||
execution.Detail,
|
||||
liveRoomId,
|
||||
recordSessionId,
|
||||
recordTaskId,
|
||||
cancellationToken);
|
||||
null);
|
||||
outcomeLevel = process.ExitCode == 0 ? SystemLogLevel.Info : SystemLogLevel.Warning;
|
||||
}
|
||||
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
TryKill(process);
|
||||
await _systemLogService.WriteAsync(
|
||||
SystemLogLevel.Warning,
|
||||
"Script",
|
||||
outcome = new ScriptExecutionOutcome(
|
||||
false,
|
||||
$"Event script timed out for {eventName}.",
|
||||
execution.Detail,
|
||||
liveRoomId,
|
||||
recordSessionId,
|
||||
recordTaskId,
|
||||
CancellationToken.None);
|
||||
null);
|
||||
outcomeLevel = SystemLogLevel.Warning;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Event script failed for {EventName}", eventName);
|
||||
await _systemLogService.WriteAsync(
|
||||
SystemLogLevel.Warning,
|
||||
"Script",
|
||||
outcome = new ScriptExecutionOutcome(
|
||||
false,
|
||||
$"Event script failed for {eventName}.",
|
||||
ex.ToString(),
|
||||
liveRoomId,
|
||||
recordSessionId,
|
||||
recordTaskId,
|
||||
cancellationToken);
|
||||
null);
|
||||
outcomeLevel = SystemLogLevel.Warning;
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Any script-provided text is persisted and surfaced separately from the execution outcome.
|
||||
}
|
||||
|
||||
var customLogOutput = await TryWriteCustomLogOutputAsync(
|
||||
eventName,
|
||||
logCategory,
|
||||
scriptLogPath,
|
||||
liveRoomId,
|
||||
recordSessionId,
|
||||
recordTaskId);
|
||||
TryDeleteFile(scriptLogPath);
|
||||
|
||||
outcome = outcome with { CustomLogOutput = customLogOutput };
|
||||
await WriteOutcomeLogAsync(
|
||||
outcome,
|
||||
logCategory,
|
||||
outcomeLevel,
|
||||
liveRoomId,
|
||||
recordSessionId,
|
||||
recordTaskId,
|
||||
cancellationToken);
|
||||
return outcome;
|
||||
}
|
||||
|
||||
private static EventScriptExecution? CreateExecution(string scriptMode, string scriptPath, string scriptContent)
|
||||
@@ -237,6 +367,47 @@ public sealed class EventScriptService : IEventScriptService
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, string> BuildTestEnvironment(string eventName, DateTimeOffset occurredAt)
|
||||
{
|
||||
var environment = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["LIVE_RECORDER_EVENT"] = eventName,
|
||||
["LIVE_RECORDER_PLATFORM"] = "Douyin",
|
||||
["LIVE_RECORDER_LIVE_ROOM_ID"] = "6f73a2f2-1d4c-4e7a-a9b1-3d29d54ed901",
|
||||
["LIVE_RECORDER_ROOM_ID"] = "676493068539",
|
||||
["LIVE_RECORDER_TITLE"] = "Sample live title",
|
||||
["LIVE_RECORDER_ANCHOR"] = "Sample anchor",
|
||||
["LIVE_RECORDER_SOURCE_URL"] = "https://live.douyin.com/676493068539",
|
||||
["LIVE_RECORDER_OCCURRED_AT_UTC"] = occurredAt.ToString("O")
|
||||
};
|
||||
|
||||
if (string.Equals(eventName, "segment_completed", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
environment["LIVE_RECORDER_RECORD_SESSION_ID"] = "8e2e9c64-b8f6-4d15-b6cb-1d4ce0adab77";
|
||||
environment["LIVE_RECORDER_RECORD_TASK_ID"] = "2a4810a2-7ef4-4a22-90d4-0211b90cc54c";
|
||||
environment["LIVE_RECORDER_SEGMENT_INDEX"] = "1";
|
||||
environment["LIVE_RECORDER_SEGMENT_FILE_PATH"] = "/app/records/Douyin/Sample Anchor/2026-04-25/203000_Sample live title__00001.mp4";
|
||||
environment["LIVE_RECORDER_DANMAKU_FILE_PATH"] = "/app/records/Douyin/Sample Anchor/2026-04-25/203000_Sample live title__00001.xml";
|
||||
environment["LIVE_RECORDER_DURATION_SECONDS"] = "2185.1";
|
||||
environment["LIVE_RECORDER_FILE_SIZE_BYTES"] = "734003200";
|
||||
environment["LIVE_RECORDER_TASK_STATUS"] = "Completed";
|
||||
environment["LIVE_RECORDER_SESSION_STATUS"] = "Completed";
|
||||
}
|
||||
|
||||
return environment;
|
||||
}
|
||||
|
||||
private static string? NormalizeEventType(string? eventType)
|
||||
{
|
||||
return eventType?.Trim().ToLowerInvariant() switch
|
||||
{
|
||||
"live_started" => "live_started",
|
||||
"live_ended" => "live_ended",
|
||||
"segment_completed" => "segment_completed",
|
||||
_ => null
|
||||
};
|
||||
}
|
||||
|
||||
private static ProcessStartInfo CreateStartInfo(string scriptPath)
|
||||
{
|
||||
var extension = Path.GetExtension(scriptPath);
|
||||
@@ -330,6 +501,85 @@ public sealed class EventScriptService : IEventScriptService
|
||||
return new FileInfo(normalizedPath).Length.ToString();
|
||||
}
|
||||
|
||||
private async Task<string?> TryWriteCustomLogOutputAsync(
|
||||
string eventName,
|
||||
string logCategory,
|
||||
string scriptLogPath,
|
||||
Guid? liveRoomId,
|
||||
Guid? recordSessionId,
|
||||
Guid? recordTaskId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(scriptLogPath) || !File.Exists(scriptLogPath))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var output = await File.ReadAllTextAsync(scriptLogPath, CancellationToken.None);
|
||||
var trimmedOutput = output.Trim();
|
||||
if (string.IsNullOrWhiteSpace(trimmedOutput))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
await _systemLogService.WriteAsync(
|
||||
SystemLogLevel.Info,
|
||||
logCategory,
|
||||
$"Event script emitted custom log output for {eventName}.",
|
||||
TruncateSystemLogDetail(trimmedOutput),
|
||||
liveRoomId,
|
||||
recordSessionId,
|
||||
recordTaskId,
|
||||
CancellationToken.None);
|
||||
|
||||
return trimmedOutput;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to collect event script custom log output for {EventName}", eventName);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task WriteOutcomeLogAsync(
|
||||
ScriptExecutionOutcome outcome,
|
||||
string logCategory,
|
||||
SystemLogLevel level,
|
||||
Guid? liveRoomId,
|
||||
Guid? recordSessionId,
|
||||
Guid? recordTaskId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await _systemLogService.WriteAsync(
|
||||
level,
|
||||
logCategory,
|
||||
outcome.Message,
|
||||
outcome.Detail,
|
||||
liveRoomId,
|
||||
recordSessionId,
|
||||
recordTaskId,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private static string CreateScriptLogPath()
|
||||
{
|
||||
return Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
$"live-recorder-script-log-{Guid.NewGuid():N}.txt");
|
||||
}
|
||||
|
||||
private static string TruncateSystemLogDetail(string detail)
|
||||
{
|
||||
if (detail.Length <= SystemLogDetailMaxLength)
|
||||
{
|
||||
return detail;
|
||||
}
|
||||
|
||||
var prefixLength = Math.Max(0, SystemLogDetailMaxLength - TruncatedDetailSuffix.Length);
|
||||
return string.Concat(detail[..prefixLength], TruncatedDetailSuffix);
|
||||
}
|
||||
|
||||
private static void TryKill(Process process)
|
||||
{
|
||||
try
|
||||
@@ -344,5 +594,25 @@ public sealed class EventScriptService : IEventScriptService
|
||||
}
|
||||
}
|
||||
|
||||
private static void TryDeleteFile(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(path) && File.Exists(path))
|
||||
{
|
||||
File.Delete(path);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record EventScriptExecution(ProcessStartInfo? StartInfo, string Detail, bool IsMissing = false);
|
||||
|
||||
private sealed record ScriptExecutionOutcome(
|
||||
bool Success,
|
||||
string Message,
|
||||
string? Detail,
|
||||
string? CustomLogOutput);
|
||||
}
|
||||
|
||||
@@ -649,6 +649,7 @@ public sealed partial class FfmpegService
|
||||
var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
|
||||
var settingsService = scope.ServiceProvider.GetRequiredService<ISystemSettingsService>();
|
||||
var emailNotificationService = scope.ServiceProvider.GetRequiredService<IEmailNotificationService>();
|
||||
var webhookNotificationService = scope.ServiceProvider.GetRequiredService<IWebhookNotificationService>();
|
||||
var settings = await settingsService.GetAsync();
|
||||
|
||||
var session = await dbContext.RecordSessions
|
||||
@@ -783,6 +784,12 @@ public sealed partial class FfmpegService
|
||||
$"exitCode={process.ExitCode}; output={effectiveOutputPath}",
|
||||
session.LiveRoom,
|
||||
currentTask);
|
||||
await webhookNotificationService.SendExceptionAsync(
|
||||
"FFmpeg",
|
||||
"Recording session exited abnormally.",
|
||||
$"exitCode={process.ExitCode}; output={effectiveOutputPath}",
|
||||
session.LiveRoom,
|
||||
currentTask);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System.Collections.Concurrent;
|
||||
using LiveRecorder.Application.Common;
|
||||
using LiveRecorder.Application.Abstractions.Logging;
|
||||
using LiveRecorder.Application.Abstractions.Notifications;
|
||||
using LiveRecorder.Application.Abstractions.Platforms;
|
||||
@@ -95,6 +96,7 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
using var notificationScope = _serviceScopeFactory.CreateScope();
|
||||
var logService = notificationScope.ServiceProvider.GetRequiredService<ISystemLogService>();
|
||||
var emailNotificationService = notificationScope.ServiceProvider.GetRequiredService<IEmailNotificationService>();
|
||||
var webhookNotificationService = notificationScope.ServiceProvider.GetRequiredService<IWebhookNotificationService>();
|
||||
await logService.WriteAsync(
|
||||
SystemLogLevel.Error,
|
||||
"Scheduler",
|
||||
@@ -108,6 +110,11 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
"Background live room polling failed.",
|
||||
ex.ToString(),
|
||||
cancellationToken: stoppingToken);
|
||||
await webhookNotificationService.SendExceptionAsync(
|
||||
"Scheduler",
|
||||
"Background live room polling failed.",
|
||||
ex.ToString(),
|
||||
cancellationToken: stoppingToken);
|
||||
}
|
||||
}
|
||||
catch (Exception notificationEx)
|
||||
@@ -133,6 +140,7 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
var liveRoomStatusService = scope.ServiceProvider.GetRequiredService<LiveRoomStatusService>();
|
||||
var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
|
||||
var emailNotificationService = scope.ServiceProvider.GetRequiredService<IEmailNotificationService>();
|
||||
var webhookNotificationService = scope.ServiceProvider.GetRequiredService<IWebhookNotificationService>();
|
||||
var storageGuardService = scope.ServiceProvider.GetRequiredService<IStorageGuardService>();
|
||||
|
||||
var liveRoom = await dbContext.LiveRooms.FirstOrDefaultAsync(item => item.Id == liveRoomId, cancellationToken);
|
||||
@@ -175,12 +183,26 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
|
||||
if (!settings.AutoStartRecordingOnLive)
|
||||
{
|
||||
await UpdateAutoStartDecisionAsync(
|
||||
dbContext,
|
||||
liveRoom,
|
||||
AutoStartDecisionCodes.SkippedDisabled,
|
||||
"Auto-start skipped because automatic start is disabled.",
|
||||
detail: null,
|
||||
cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
var startCheck = storageGuardService.CheckCanStartOrResume(settings);
|
||||
if (!startCheck.HasEnoughSpace)
|
||||
{
|
||||
await UpdateAutoStartDecisionAsync(
|
||||
dbContext,
|
||||
liveRoom,
|
||||
AutoStartDecisionCodes.SkippedStorage,
|
||||
"Auto-start skipped because storage is below threshold.",
|
||||
startCheck.Message,
|
||||
cancellationToken);
|
||||
await logService.WriteAsync(
|
||||
SystemLogLevel.Warning,
|
||||
"Storage",
|
||||
@@ -216,6 +238,13 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
|
||||
if (hasRunningSession)
|
||||
{
|
||||
await UpdateAutoStartDecisionAsync(
|
||||
dbContext,
|
||||
liveRoom,
|
||||
AutoStartDecisionCodes.SkippedActiveSession,
|
||||
"Auto-start skipped because an active recording session already exists.",
|
||||
detail: null,
|
||||
cancellationToken);
|
||||
await logService.WriteAsync(
|
||||
SystemLogLevel.Info,
|
||||
"Scheduler",
|
||||
@@ -238,6 +267,7 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
{
|
||||
LiveRoomId = liveRoom.Id
|
||||
},
|
||||
trackAutoStartDecision: true,
|
||||
cancellationToken);
|
||||
}
|
||||
catch (Exception ex)
|
||||
@@ -263,6 +293,12 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
ex.ToString(),
|
||||
liveRoom,
|
||||
cancellationToken: cancellationToken);
|
||||
await webhookNotificationService.SendExceptionAsync(
|
||||
"Scheduler",
|
||||
"Background polling failed for a live room.",
|
||||
ex.ToString(),
|
||||
liveRoom,
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -515,6 +551,22 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
return exception.InnerException is not null && IsSqliteStorageFullException(exception.InnerException);
|
||||
}
|
||||
|
||||
private static async Task UpdateAutoStartDecisionAsync(
|
||||
LiveRecorderDbContext dbContext,
|
||||
Domain.Entities.LiveRoom liveRoom,
|
||||
string code,
|
||||
string summary,
|
||||
string? detail,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
liveRoom.SetLastAutoStartDecision(
|
||||
Truncate(code, 64),
|
||||
Truncate(summary, 256),
|
||||
Truncate(detail, 2048),
|
||||
DateTimeOffset.UtcNow);
|
||||
await SaveChangesWithRetryAsync(dbContext, cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task SaveChangesWithRetryAsync(LiveRecorderDbContext dbContext, CancellationToken cancellationToken)
|
||||
{
|
||||
for (var attempt = 1; attempt <= 5; attempt++)
|
||||
@@ -539,4 +591,15 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
|
||||
exception.SqliteErrorCode is 5 or 6 ||
|
||||
exception.Message.Contains("database is locked", StringComparison.OrdinalIgnoreCase) ||
|
||||
exception.Message.Contains("database table is locked", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static string? Truncate(string? value, int maxLength)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var trimmed = value.Trim();
|
||||
return trimmed.Length <= maxLength ? trimmed : trimmed[..maxLength];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,437 @@
|
||||
using LiveRecorder.Application.Abstractions.Recording;
|
||||
using LiveRecorder.Application.Abstractions.Settings;
|
||||
using LiveRecorder.Application.Abstractions.Storage;
|
||||
using LiveRecorder.Application.Common;
|
||||
using LiveRecorder.Application.Models.Recovery;
|
||||
using LiveRecorder.Application.Models.RecordTasks;
|
||||
using LiveRecorder.Application.Services;
|
||||
using LiveRecorder.Domain.Entities;
|
||||
using LiveRecorder.Domain.Enums;
|
||||
using LiveRecorder.Infrastructure.Persistence;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace LiveRecorder.Infrastructure.Services;
|
||||
|
||||
public sealed class RecoveryService
|
||||
{
|
||||
private readonly LiveRecorderDbContext _dbContext;
|
||||
private readonly ISystemSettingsService _systemSettingsService;
|
||||
private readonly IStorageGuardService _storageGuardService;
|
||||
private readonly IFfmpegService _ffmpegService;
|
||||
private readonly RecordService _recordService;
|
||||
|
||||
public RecoveryService(
|
||||
LiveRecorderDbContext dbContext,
|
||||
ISystemSettingsService systemSettingsService,
|
||||
IStorageGuardService storageGuardService,
|
||||
IFfmpegService ffmpegService,
|
||||
RecordService recordService)
|
||||
{
|
||||
_dbContext = dbContext;
|
||||
_systemSettingsService = systemSettingsService;
|
||||
_storageGuardService = storageGuardService;
|
||||
_ffmpegService = ffmpegService;
|
||||
_recordService = recordService;
|
||||
}
|
||||
|
||||
public async Task<RecoveryOverviewDto> GetOverviewAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var settings = await _systemSettingsService.GetAsync(cancellationToken);
|
||||
var storage = _storageGuardService.CheckCanStartOrResume(settings);
|
||||
var liveRooms = await ListRecoverableLiveRoomsAsync(cancellationToken);
|
||||
var finalizations = await ListRecoverableFinalizationsAsync(cancellationToken);
|
||||
|
||||
return new RecoveryOverviewDto
|
||||
{
|
||||
Storage = new StorageGuardStatusDto
|
||||
{
|
||||
IsEnabled = storage.IsEnabled,
|
||||
HasEnoughSpace = storage.HasEnoughSpace,
|
||||
CheckedPath = storage.CheckedPath,
|
||||
AvailableBytes = storage.AvailableBytes,
|
||||
RequiredBytes = storage.RequiredBytes,
|
||||
Message = storage.Message
|
||||
},
|
||||
LiveRooms = liveRooms,
|
||||
Finalizations = finalizations
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<RecoveryActionResultDto> RetryLiveRoomAsync(Guid liveRoomId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var room = await _dbContext.LiveRooms.FirstOrDefaultAsync(item => item.Id == liveRoomId, cancellationToken);
|
||||
if (room is null)
|
||||
{
|
||||
return FailureResult("Live room was not found.");
|
||||
}
|
||||
|
||||
if (!room.IsEnabled)
|
||||
{
|
||||
return FailureResult("Live room is disabled and cannot be retried.");
|
||||
}
|
||||
|
||||
if (room.AvailabilityStatus != LiveRoomAvailabilityStatus.Live)
|
||||
{
|
||||
return FailureResult("Live room is not currently online.");
|
||||
}
|
||||
|
||||
var hasActiveSession = await _dbContext.RecordSessions.AnyAsync(
|
||||
item => item.LiveRoomId == room.Id &&
|
||||
(item.Status == RecordSessionStatus.Starting ||
|
||||
item.Status == RecordSessionStatus.Running ||
|
||||
item.Status == RecordSessionStatus.Stopping),
|
||||
cancellationToken);
|
||||
|
||||
if (hasActiveSession)
|
||||
{
|
||||
room.SetLastAutoStartDecision(
|
||||
AutoStartDecisionCodes.SkippedActiveSession,
|
||||
"Auto-start skipped because an active recording session already exists.",
|
||||
null,
|
||||
DateTimeOffset.UtcNow);
|
||||
await _dbContext.SaveChangesAsync(cancellationToken);
|
||||
return FailureResult("An active recording session already exists for this live room.");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var task = await _recordService.StartAsync(
|
||||
new StartRecordTaskRequest
|
||||
{
|
||||
LiveRoomId = liveRoomId
|
||||
},
|
||||
trackAutoStartDecision: true,
|
||||
cancellationToken);
|
||||
|
||||
var started = task.Status is RecordTaskStatus.Starting or RecordTaskStatus.Running;
|
||||
return new RecoveryActionResultDto
|
||||
{
|
||||
RequestedCount = 1,
|
||||
SuccessCount = started ? 1 : 0,
|
||||
FailedCount = started ? 0 : 1,
|
||||
Messages =
|
||||
[
|
||||
started
|
||||
? $"Recording retry started for room {room.RoomId}."
|
||||
: $"Recording retry did not start for room {room.RoomId}. Status={task.Status}; Error={task.ErrorMessage ?? "n/a"}"
|
||||
]
|
||||
};
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return FailureResult($"Recording retry failed for room {room.RoomId}: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<RecoveryActionResultDto> RetryAllLiveRoomsAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var liveRooms = await ListRecoverableLiveRoomsAsync(cancellationToken);
|
||||
if (liveRooms.Count == 0)
|
||||
{
|
||||
return new RecoveryActionResultDto
|
||||
{
|
||||
RequestedCount = 0,
|
||||
SuccessCount = 0,
|
||||
FailedCount = 0,
|
||||
Messages = ["No live rooms currently require retry."]
|
||||
};
|
||||
}
|
||||
|
||||
var messages = new List<string>();
|
||||
var successCount = 0;
|
||||
foreach (var item in liveRooms)
|
||||
{
|
||||
var result = await RetryLiveRoomAsync(item.LiveRoomId, cancellationToken);
|
||||
successCount += result.SuccessCount;
|
||||
messages.AddRange(result.Messages);
|
||||
}
|
||||
|
||||
return new RecoveryActionResultDto
|
||||
{
|
||||
RequestedCount = liveRooms.Count,
|
||||
SuccessCount = successCount,
|
||||
FailedCount = liveRooms.Count - successCount,
|
||||
Messages = messages
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<RecoveryActionResultDto> ResumeFinalizationAsync(Guid recordTaskId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var started = await _ffmpegService.StartManualFinalizeTaskAsync(recordTaskId, cancellationToken);
|
||||
return new RecoveryActionResultDto
|
||||
{
|
||||
RequestedCount = 1,
|
||||
SuccessCount = started ? 1 : 0,
|
||||
FailedCount = started ? 0 : 1,
|
||||
Messages =
|
||||
[
|
||||
started
|
||||
? $"MP4 finalization resumed for task {recordTaskId}."
|
||||
: $"MP4 finalization could not be resumed for task {recordTaskId}."
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<RecoveryActionResultDto> ResumeAllFinalizationsAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var finalizations = await ListRecoverableFinalizationsAsync(cancellationToken);
|
||||
if (finalizations.Count == 0)
|
||||
{
|
||||
return new RecoveryActionResultDto
|
||||
{
|
||||
RequestedCount = 0,
|
||||
SuccessCount = 0,
|
||||
FailedCount = 0,
|
||||
Messages = ["No MP4 finalization tasks currently require recovery."]
|
||||
};
|
||||
}
|
||||
|
||||
var successCount = 0;
|
||||
var messages = new List<string>();
|
||||
foreach (var item in finalizations)
|
||||
{
|
||||
var started = await _ffmpegService.StartManualFinalizeTaskAsync(item.RecordTaskId, cancellationToken);
|
||||
if (started)
|
||||
{
|
||||
successCount++;
|
||||
}
|
||||
|
||||
messages.Add(
|
||||
started
|
||||
? $"MP4 finalization resumed for task {item.RecordTaskId}."
|
||||
: $"MP4 finalization could not be resumed for task {item.RecordTaskId}.");
|
||||
}
|
||||
|
||||
return new RecoveryActionResultDto
|
||||
{
|
||||
RequestedCount = finalizations.Count,
|
||||
SuccessCount = successCount,
|
||||
FailedCount = finalizations.Count - successCount,
|
||||
Messages = messages
|
||||
};
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<RecoverableLiveRoomDto>> ListRecoverableLiveRoomsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var activeLiveRoomIds = await _dbContext.RecordSessions
|
||||
.AsNoTracking()
|
||||
.Where(item => item.Status == RecordSessionStatus.Starting ||
|
||||
item.Status == RecordSessionStatus.Running ||
|
||||
item.Status == RecordSessionStatus.Stopping)
|
||||
.Select(item => item.LiveRoomId)
|
||||
.Distinct()
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
var rooms = await _dbContext.LiveRooms
|
||||
.AsNoTracking()
|
||||
.Where(item => item.IsEnabled &&
|
||||
item.AvailabilityStatus == LiveRoomAvailabilityStatus.Live &&
|
||||
item.LastAutoStartDecisionCode != AutoStartDecisionCodes.Started)
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
return rooms
|
||||
.Where(item => !activeLiveRoomIds.Contains(item.Id))
|
||||
.OrderByDescending(item => item.LastAutoStartDecisionAt ?? item.LastCheckedAt ?? item.UpdatedAt)
|
||||
.Select(item => new RecoverableLiveRoomDto
|
||||
{
|
||||
LiveRoomId = item.Id,
|
||||
PlatformName = item.Platform.ToString(),
|
||||
RoomId = item.RoomId,
|
||||
Title = item.Title,
|
||||
AnchorName = item.AnchorName,
|
||||
LastAutoStartDecisionCode = item.LastAutoStartDecisionCode,
|
||||
LastAutoStartDecisionSummary = item.LastAutoStartDecisionSummary,
|
||||
LastAutoStartDecisionDetail = item.LastAutoStartDecisionDetail,
|
||||
LastAutoStartDecisionAt = item.LastAutoStartDecisionAt,
|
||||
LastCheckedAt = item.LastCheckedAt
|
||||
})
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<RecoverableFinalizationDto>> ListRecoverableFinalizationsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var tasks = await _dbContext.RecordTasks
|
||||
.AsNoTracking()
|
||||
.Include(item => item.LiveRoom)
|
||||
.Include(item => item.RecordSession)
|
||||
.Include(item => item.Result)
|
||||
.Where(item => item.OutputFormat == RecordOutputFormat.Mp4 &&
|
||||
item.RecordSession != null)
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
return tasks
|
||||
.Where(IsRecoverableFinalization)
|
||||
.OrderByDescending(item => item.UpdatedAt)
|
||||
.Select(item => new RecoverableFinalizationDto
|
||||
{
|
||||
RecordTaskId = item.Id,
|
||||
RecordSessionId = item.RecordSessionId,
|
||||
LiveRoomId = item.LiveRoomId,
|
||||
LiveRoomTitle = item.LiveRoom?.Title ?? item.LiveRoom?.AnchorName ?? item.LiveRoom?.RoomId ?? item.LiveRoomId.ToString(),
|
||||
RoomId = item.LiveRoom?.RoomId ?? "-",
|
||||
PlatformName = item.LiveRoom?.Platform.ToString() ?? "Unknown",
|
||||
SegmentIndex = item.SegmentIndex,
|
||||
Status = item.Status,
|
||||
OutputFilePath = item.OutputFilePath,
|
||||
Reason = BuildFinalizationReason(item),
|
||||
CreatedAt = item.CreatedAt,
|
||||
EndedAt = item.EndedAt
|
||||
})
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static bool IsRecoverableFinalization(RecordTask recordTask)
|
||||
{
|
||||
if (recordTask.RecordSession is null ||
|
||||
recordTask.OutputFormat != RecordOutputFormat.Mp4)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (recordTask.Status == RecordTaskStatus.Processing)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (recordTask.Status is RecordTaskStatus.Starting or RecordTaskStatus.Running or RecordTaskStatus.Stopping)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (recordTask.RecordSession.Status is RecordSessionStatus.Starting or RecordSessionStatus.Running or RecordSessionStatus.Stopping)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(recordTask.OutputFilePath))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var recorderOutputPath = ResolveManualFinalizeSourcePath(recordTask, recordTask.RecordSession);
|
||||
if (!File.Exists(recorderOutputPath))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (recordTask.Status == RecordTaskStatus.Completed)
|
||||
{
|
||||
var finalOutputPath = NormalizeAbsolutePath(recordTask.OutputFilePath);
|
||||
return !HasUsableOutput(finalOutputPath, CalculateFileSize(finalOutputPath));
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private static string BuildFinalizationReason(RecordTask recordTask)
|
||||
{
|
||||
if (recordTask.Status == RecordTaskStatus.Processing)
|
||||
{
|
||||
return string.IsNullOrWhiteSpace(recordTask.ErrorMessage)
|
||||
? "MP4 finalization is queued or paused and can be resumed."
|
||||
: recordTask.ErrorMessage!;
|
||||
}
|
||||
|
||||
if (recordTask.Status == RecordTaskStatus.Completed)
|
||||
{
|
||||
return "The final MP4 output is missing, but the intermediate recording file is still available.";
|
||||
}
|
||||
|
||||
return "Manual MP4 finalization can be retried from the intermediate recording file.";
|
||||
}
|
||||
|
||||
private static string ResolveManualFinalizeSourcePath(RecordTask recordTask, RecordSession recordSession)
|
||||
{
|
||||
var resultPath = recordTask.Result?.FilePath;
|
||||
if (!string.IsNullOrWhiteSpace(resultPath) &&
|
||||
resultPath.EndsWith(".ts", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
var normalizedResultPath = NormalizeAbsolutePath(resultPath);
|
||||
if (File.Exists(normalizedResultPath))
|
||||
{
|
||||
return normalizedResultPath;
|
||||
}
|
||||
}
|
||||
|
||||
return NormalizeAbsolutePath(
|
||||
GetRecorderOutputPath(
|
||||
recordTask.OutputFilePath ?? resultPath ?? string.Empty,
|
||||
recordSession.OutputFormat,
|
||||
recordSession.SaveMode));
|
||||
}
|
||||
|
||||
private static string GetRecorderOutputPath(
|
||||
string finalOutputPath,
|
||||
RecordOutputFormat outputFormat,
|
||||
RecordSaveMode saveMode)
|
||||
{
|
||||
if (outputFormat != RecordOutputFormat.Mp4)
|
||||
{
|
||||
return finalOutputPath;
|
||||
}
|
||||
|
||||
if (saveMode == RecordSaveMode.SingleFile)
|
||||
{
|
||||
return Path.Combine(
|
||||
Path.GetDirectoryName(finalOutputPath)!,
|
||||
$"{Path.GetFileNameWithoutExtension(finalOutputPath)}.recording.ts");
|
||||
}
|
||||
|
||||
return Path.ChangeExtension(finalOutputPath, ".ts");
|
||||
}
|
||||
|
||||
private static long? CalculateFileSize(string? outputPath)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(outputPath))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (File.Exists(outputPath))
|
||||
{
|
||||
return new FileInfo(outputPath).Length;
|
||||
}
|
||||
|
||||
if (Directory.Exists(outputPath))
|
||||
{
|
||||
return new DirectoryInfo(outputPath)
|
||||
.EnumerateFiles("*", SearchOption.TopDirectoryOnly)
|
||||
.Sum(static file => file.Length);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool HasUsableOutput(string? outputPath, long? fileSize)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(outputPath))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (File.Exists(outputPath))
|
||||
{
|
||||
return fileSize.GetValueOrDefault() > 0;
|
||||
}
|
||||
|
||||
if (Directory.Exists(outputPath))
|
||||
{
|
||||
return Directory.EnumerateFiles(outputPath, "*", SearchOption.TopDirectoryOnly).Any();
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static string NormalizeAbsolutePath(string path) =>
|
||||
Path.IsPathRooted(path)
|
||||
? path
|
||||
: Path.GetFullPath(path, AppContext.BaseDirectory);
|
||||
|
||||
private static RecoveryActionResultDto FailureResult(string message) => new()
|
||||
{
|
||||
RequestedCount = 1,
|
||||
SuccessCount = 0,
|
||||
FailedCount = 1,
|
||||
Messages = [message]
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
using LiveRecorder.Application.Abstractions.Logging;
|
||||
using LiveRecorder.Application.Abstractions.Settings;
|
||||
using LiveRecorder.Domain.Enums;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace LiveRecorder.Infrastructure.Services;
|
||||
|
||||
public sealed class RetentionCleanupBackgroundService : BackgroundService
|
||||
{
|
||||
private static readonly TimeSpan CleanupInterval = TimeSpan.FromHours(24);
|
||||
|
||||
private readonly IServiceScopeFactory _serviceScopeFactory;
|
||||
private readonly ILogger<RetentionCleanupBackgroundService> _logger;
|
||||
|
||||
public RetentionCleanupBackgroundService(
|
||||
IServiceScopeFactory serviceScopeFactory,
|
||||
ILogger<RetentionCleanupBackgroundService> logger)
|
||||
{
|
||||
_serviceScopeFactory = serviceScopeFactory;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var scope = _serviceScopeFactory.CreateScope();
|
||||
var settingsService = scope.ServiceProvider.GetRequiredService<ISystemSettingsService>();
|
||||
var cleanupService = scope.ServiceProvider.GetRequiredService<RetentionCleanupService>();
|
||||
var settings = await settingsService.GetAsync(stoppingToken);
|
||||
|
||||
if (settings.EnableRetentionCleanup)
|
||||
{
|
||||
await cleanupService.RunAsync(ignoreEnabledSetting: false, stoppingToken);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Retention cleanup background task failed");
|
||||
|
||||
try
|
||||
{
|
||||
using var scope = _serviceScopeFactory.CreateScope();
|
||||
var logService = scope.ServiceProvider.GetRequiredService<ISystemLogService>();
|
||||
await logService.WriteAsync(
|
||||
SystemLogLevel.Error,
|
||||
"Retention",
|
||||
"Retention cleanup background task failed.",
|
||||
ex.ToString(),
|
||||
cancellationToken: CancellationToken.None);
|
||||
}
|
||||
catch (Exception logEx)
|
||||
{
|
||||
_logger.LogWarning(logEx, "Failed to persist retention cleanup background error log");
|
||||
}
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(CleanupInterval, stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
using LiveRecorder.Application.Abstractions.Logging;
|
||||
using LiveRecorder.Application.Abstractions.Settings;
|
||||
using LiveRecorder.Application.Models.Settings;
|
||||
using LiveRecorder.Domain.Entities;
|
||||
using LiveRecorder.Domain.Enums;
|
||||
using LiveRecorder.Infrastructure.Persistence;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace LiveRecorder.Infrastructure.Services;
|
||||
|
||||
public sealed class RetentionCleanupService
|
||||
{
|
||||
private readonly LiveRecorderDbContext _dbContext;
|
||||
private readonly ISystemSettingsService _systemSettingsService;
|
||||
private readonly ISystemLogService _systemLogService;
|
||||
|
||||
public RetentionCleanupService(
|
||||
LiveRecorderDbContext dbContext,
|
||||
ISystemSettingsService systemSettingsService,
|
||||
ISystemLogService systemLogService)
|
||||
{
|
||||
_dbContext = dbContext;
|
||||
_systemSettingsService = systemSettingsService;
|
||||
_systemLogService = systemLogService;
|
||||
}
|
||||
|
||||
public async Task<RetentionCleanupResultDto> RunAsync(
|
||||
bool ignoreEnabledSetting = false,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var settings = await _systemSettingsService.GetAsync(cancellationToken);
|
||||
if (!settings.EnableRetentionCleanup && !ignoreEnabledSetting)
|
||||
{
|
||||
return CreateEmptyResult();
|
||||
}
|
||||
|
||||
var warnings = new List<string>();
|
||||
var deletedFilePaths = new List<string>();
|
||||
var deletedDanmakuPaths = new List<string>();
|
||||
var deletedTaskIds = new HashSet<Guid>();
|
||||
var deletedSessionIds = new HashSet<Guid>();
|
||||
var deletedResultIds = new HashSet<Guid>();
|
||||
var deletedLogIds = new HashSet<Guid>();
|
||||
var cutoff = DateTimeOffset.UtcNow.AddDays(-Math.Max(1, settings.RetentionDays));
|
||||
|
||||
var staleTasks = await _dbContext.RecordTasks
|
||||
.Include(item => item.LiveRoom)
|
||||
.Include(item => item.RecordSession)
|
||||
.Include(item => item.Result)
|
||||
.Where(item => item.CreatedAt < cutoff &&
|
||||
item.Status != RecordTaskStatus.Starting &&
|
||||
item.Status != RecordTaskStatus.Running &&
|
||||
item.Status != RecordTaskStatus.Stopping &&
|
||||
item.Status != RecordTaskStatus.Processing)
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
foreach (var task in staleTasks)
|
||||
{
|
||||
if (settings.RetentionDeleteFiles)
|
||||
{
|
||||
TryDeleteRecordOutput(task, warnings, deletedFilePaths, deletedDanmakuPaths);
|
||||
}
|
||||
|
||||
var taskLogs = await _dbContext.SystemLogEntries
|
||||
.Where(item => item.RecordTaskId == task.Id)
|
||||
.ToListAsync(cancellationToken);
|
||||
foreach (var log in taskLogs)
|
||||
{
|
||||
deletedLogIds.Add(log.Id);
|
||||
}
|
||||
|
||||
if (task.Result is not null)
|
||||
{
|
||||
deletedResultIds.Add(task.Result.Id);
|
||||
_dbContext.RecordResults.Remove(task.Result);
|
||||
}
|
||||
|
||||
if (taskLogs.Count > 0)
|
||||
{
|
||||
_dbContext.SystemLogEntries.RemoveRange(taskLogs);
|
||||
}
|
||||
|
||||
deletedTaskIds.Add(task.Id);
|
||||
_dbContext.RecordTasks.Remove(task);
|
||||
}
|
||||
|
||||
if (deletedTaskIds.Count > 0 || deletedResultIds.Count > 0 || deletedLogIds.Count > 0)
|
||||
{
|
||||
await _dbContext.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
var staleSessions = await _dbContext.RecordSessions
|
||||
.Include(item => item.RecordTasks)
|
||||
.Where(item => item.CreatedAt < cutoff &&
|
||||
item.Status != RecordSessionStatus.Starting &&
|
||||
item.Status != RecordSessionStatus.Running &&
|
||||
item.Status != RecordSessionStatus.Stopping)
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
foreach (var session in staleSessions.Where(static item => item.RecordTasks.Count == 0))
|
||||
{
|
||||
var sessionLogs = await _dbContext.SystemLogEntries
|
||||
.Where(item => item.RecordSessionId == session.Id)
|
||||
.ToListAsync(cancellationToken);
|
||||
foreach (var log in sessionLogs)
|
||||
{
|
||||
deletedLogIds.Add(log.Id);
|
||||
}
|
||||
|
||||
if (sessionLogs.Count > 0)
|
||||
{
|
||||
_dbContext.SystemLogEntries.RemoveRange(sessionLogs);
|
||||
}
|
||||
|
||||
deletedSessionIds.Add(session.Id);
|
||||
_dbContext.RecordSessions.Remove(session);
|
||||
}
|
||||
|
||||
var staleGlobalLogs = await _dbContext.SystemLogEntries
|
||||
.Where(item => item.CreatedAt < cutoff)
|
||||
.ToListAsync(cancellationToken);
|
||||
foreach (var log in staleGlobalLogs)
|
||||
{
|
||||
deletedLogIds.Add(log.Id);
|
||||
}
|
||||
|
||||
if (staleGlobalLogs.Count > 0)
|
||||
{
|
||||
_dbContext.SystemLogEntries.RemoveRange(staleGlobalLogs);
|
||||
}
|
||||
|
||||
if (deletedSessionIds.Count > 0 || staleGlobalLogs.Count > 0)
|
||||
{
|
||||
await _dbContext.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
var result = new RetentionCleanupResultDto
|
||||
{
|
||||
DeletedSessionCount = deletedSessionIds.Count,
|
||||
DeletedTaskCount = deletedTaskIds.Count,
|
||||
DeletedResultCount = deletedResultIds.Count,
|
||||
DeletedLogCount = deletedLogIds.Count,
|
||||
DeletedFileCount = deletedFilePaths.Count,
|
||||
DeletedDanmakuFileCount = deletedDanmakuPaths.Count,
|
||||
Warnings = warnings
|
||||
};
|
||||
|
||||
if (deletedSessionIds.Count > 0 ||
|
||||
deletedTaskIds.Count > 0 ||
|
||||
deletedResultIds.Count > 0 ||
|
||||
deletedLogIds.Count > 0 ||
|
||||
deletedFilePaths.Count > 0 ||
|
||||
deletedDanmakuPaths.Count > 0 ||
|
||||
warnings.Count > 0)
|
||||
{
|
||||
await _systemLogService.WriteAsync(
|
||||
SystemLogLevel.Info,
|
||||
"Retention",
|
||||
"Retention cleanup completed.",
|
||||
$"sessions={result.DeletedSessionCount}; tasks={result.DeletedTaskCount}; results={result.DeletedResultCount}; logs={result.DeletedLogCount}; video-files={result.DeletedFileCount}; danmaku-files={result.DeletedDanmakuFileCount}; warnings={warnings.Count}",
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static RetentionCleanupResultDto CreateEmptyResult() => new()
|
||||
{
|
||||
DeletedSessionCount = 0,
|
||||
DeletedTaskCount = 0,
|
||||
DeletedResultCount = 0,
|
||||
DeletedLogCount = 0,
|
||||
DeletedFileCount = 0,
|
||||
DeletedDanmakuFileCount = 0,
|
||||
Warnings = []
|
||||
};
|
||||
|
||||
private static void TryDeleteRecordOutput(
|
||||
RecordTask recordTask,
|
||||
List<string> warnings,
|
||||
List<string> deletedFilePaths,
|
||||
List<string> deletedDanmakuPaths)
|
||||
{
|
||||
var outputPath = recordTask.Result?.FilePath ?? recordTask.OutputFilePath;
|
||||
if (!string.IsNullOrWhiteSpace(outputPath))
|
||||
{
|
||||
TryDeletePath(outputPath, warnings, deletedFilePaths, $"output for task {recordTask.Id}");
|
||||
TryDeleteIntermediateRecordingArtifacts(outputPath, recordTask.OutputFormat, warnings, deletedFilePaths, recordTask.Id);
|
||||
}
|
||||
|
||||
var danmakuPath = recordTask.Result?.DanmakuFilePath;
|
||||
if (!string.IsNullOrWhiteSpace(danmakuPath))
|
||||
{
|
||||
TryDeletePath(danmakuPath, warnings, deletedDanmakuPaths, $"danmaku for task {recordTask.Id}");
|
||||
}
|
||||
}
|
||||
|
||||
private static void TryDeleteIntermediateRecordingArtifacts(
|
||||
string finalOutputPath,
|
||||
RecordOutputFormat outputFormat,
|
||||
List<string> warnings,
|
||||
List<string> deletedFilePaths,
|
||||
Guid recordTaskId)
|
||||
{
|
||||
if (outputFormat != RecordOutputFormat.Mp4)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var absoluteFinalPath = Path.IsPathRooted(finalOutputPath)
|
||||
? finalOutputPath
|
||||
: Path.GetFullPath(finalOutputPath, AppContext.BaseDirectory);
|
||||
|
||||
var intermediateCandidates = new[]
|
||||
{
|
||||
Path.ChangeExtension(absoluteFinalPath, ".ts"),
|
||||
Path.Combine(
|
||||
Path.GetDirectoryName(absoluteFinalPath) ?? string.Empty,
|
||||
$"{Path.GetFileNameWithoutExtension(absoluteFinalPath)}.recording.ts")
|
||||
};
|
||||
|
||||
foreach (var candidate in intermediateCandidates
|
||||
.Where(static path => !string.IsNullOrWhiteSpace(path))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
if (string.Equals(candidate, absoluteFinalPath, StringComparison.OrdinalIgnoreCase) || !File.Exists(candidate))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
TryDeletePath(candidate, warnings, deletedFilePaths, $"intermediate output for task {recordTaskId}");
|
||||
}
|
||||
}
|
||||
|
||||
private static void TryDeletePath(
|
||||
string path,
|
||||
List<string> warnings,
|
||||
List<string> deletedPaths,
|
||||
string label)
|
||||
{
|
||||
var absolutePath = Path.IsPathRooted(path)
|
||||
? path
|
||||
: Path.GetFullPath(path, AppContext.BaseDirectory);
|
||||
|
||||
try
|
||||
{
|
||||
if (IsUnsafeDeletionTarget(absolutePath))
|
||||
{
|
||||
warnings.Add($"Skipped deleting suspicious path: {absolutePath}");
|
||||
return;
|
||||
}
|
||||
|
||||
if (File.Exists(absolutePath))
|
||||
{
|
||||
File.Delete(absolutePath);
|
||||
deletedPaths.Add(absolutePath);
|
||||
return;
|
||||
}
|
||||
|
||||
if (Directory.Exists(absolutePath))
|
||||
{
|
||||
Directory.Delete(absolutePath, true);
|
||||
deletedPaths.Add(absolutePath);
|
||||
return;
|
||||
}
|
||||
|
||||
warnings.Add($"Path not found for {label}: {absolutePath}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
warnings.Add($"Failed to delete {label}: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsUnsafeDeletionTarget(string absolutePath)
|
||||
{
|
||||
var normalized = absolutePath.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
|
||||
var root = Path.GetPathRoot(normalized)?.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
|
||||
return string.IsNullOrWhiteSpace(normalized) ||
|
||||
normalized.Length < 4 ||
|
||||
string.Equals(normalized, root, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,337 @@
|
||||
using System.Net.Http.Json;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using LiveRecorder.Application.Abstractions.Logging;
|
||||
using LiveRecorder.Application.Abstractions.Notifications;
|
||||
using LiveRecorder.Application.Abstractions.Settings;
|
||||
using LiveRecorder.Application.Models.Settings;
|
||||
using LiveRecorder.Domain.Entities;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace LiveRecorder.Infrastructure.Services;
|
||||
|
||||
public sealed class WebhookNotificationService : IWebhookNotificationService
|
||||
{
|
||||
private const string AppName = "LiveRecorder";
|
||||
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly ISystemSettingsService _systemSettingsService;
|
||||
private readonly ISystemLogService _systemLogService;
|
||||
private readonly ILogger<WebhookNotificationService> _logger;
|
||||
|
||||
public WebhookNotificationService(
|
||||
IHttpClientFactory httpClientFactory,
|
||||
ISystemSettingsService systemSettingsService,
|
||||
ISystemLogService systemLogService,
|
||||
ILogger<WebhookNotificationService> logger)
|
||||
{
|
||||
_httpClientFactory = httpClientFactory;
|
||||
_systemSettingsService = systemSettingsService;
|
||||
_systemLogService = systemLogService;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task SendLiveStartedAsync(LiveRoom liveRoom, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var settings = await _systemSettingsService.GetAsync(cancellationToken);
|
||||
if (!settings.EnableWebhookNotification || !settings.NotifyWebhookOnLiveStarted)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var payload = BuildPayload(
|
||||
"live_started",
|
||||
$"Live started: {liveRoom.AnchorName ?? liveRoom.RoomId}",
|
||||
liveRoom.SourceUrl,
|
||||
liveRoom,
|
||||
recordTask: null);
|
||||
|
||||
await SendConfiguredWebhookAsync(
|
||||
settings,
|
||||
payload,
|
||||
"Webhook notification sent for live_started.",
|
||||
"Webhook notification failed for live_started.",
|
||||
liveRoom.Id,
|
||||
recordSessionId: null,
|
||||
recordTaskId: null,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public async Task SendExceptionAsync(
|
||||
string source,
|
||||
string summary,
|
||||
string? detail = null,
|
||||
LiveRoom? liveRoom = null,
|
||||
RecordTask? recordTask = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var settings = await _systemSettingsService.GetAsync(cancellationToken);
|
||||
if (!settings.EnableWebhookNotification || !settings.NotifyWebhookOnException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var payload = BuildPayload(
|
||||
"exception",
|
||||
summary,
|
||||
detail,
|
||||
liveRoom,
|
||||
recordTask,
|
||||
source);
|
||||
|
||||
await SendConfiguredWebhookAsync(
|
||||
settings,
|
||||
payload,
|
||||
"Webhook notification sent for exception.",
|
||||
"Webhook notification failed for exception.",
|
||||
liveRoom?.Id,
|
||||
recordSessionId: recordTask?.RecordSessionId,
|
||||
recordTaskId: recordTask?.Id,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<WebhookTestResultDto> SendTestAsync(
|
||||
SendTestWebhookRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
var settings = new SystemSettingsDto
|
||||
{
|
||||
EnableWebhookNotification = true,
|
||||
WebhookUrl = request.WebhookUrl.Trim(),
|
||||
WebhookHeaders = request.WebhookHeaders,
|
||||
WebhookTimeoutSeconds = request.WebhookTimeoutSeconds,
|
||||
NotifyWebhookOnLiveStarted = true,
|
||||
NotifyWebhookOnException = true
|
||||
};
|
||||
|
||||
var payload = BuildPayload(
|
||||
"live_started",
|
||||
"Webhook test from LiveRecorder.",
|
||||
"This is a sample webhook payload generated from the settings test action.",
|
||||
new LiveRoom(
|
||||
Domain.Enums.LivePlatformType.Douyin,
|
||||
"https://live.douyin.com/676493068539",
|
||||
"676493068539",
|
||||
"https://live.douyin.com/676493068539",
|
||||
DateTimeOffset.UtcNow),
|
||||
recordTask: null);
|
||||
|
||||
try
|
||||
{
|
||||
var result = await SendInternalAsync(settings, payload, cancellationToken);
|
||||
await _systemLogService.WriteAsync(
|
||||
result.Success ? Domain.Enums.SystemLogLevel.Info : Domain.Enums.SystemLogLevel.Warning,
|
||||
"Webhook",
|
||||
result.Success
|
||||
? "Webhook test completed successfully."
|
||||
: "Webhook test failed.",
|
||||
result.Detail,
|
||||
cancellationToken: cancellationToken);
|
||||
|
||||
return new WebhookTestResultDto
|
||||
{
|
||||
Success = result.Success,
|
||||
Message = result.Success
|
||||
? "Webhook test completed successfully."
|
||||
: "Webhook test failed.",
|
||||
Detail = result.Detail
|
||||
};
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Webhook test failed");
|
||||
await _systemLogService.WriteAsync(
|
||||
Domain.Enums.SystemLogLevel.Warning,
|
||||
"Webhook",
|
||||
"Webhook test failed.",
|
||||
ex.ToString(),
|
||||
cancellationToken: cancellationToken);
|
||||
|
||||
return new WebhookTestResultDto
|
||||
{
|
||||
Success = false,
|
||||
Message = "Webhook test failed.",
|
||||
Detail = ex.Message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
private async Task SendConfiguredWebhookAsync(
|
||||
SystemSettingsDto settings,
|
||||
object payload,
|
||||
string successMessage,
|
||||
string failureMessage,
|
||||
Guid? liveRoomId,
|
||||
Guid? recordSessionId,
|
||||
Guid? recordTaskId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var result = await SendInternalAsync(settings, payload, cancellationToken);
|
||||
if (!result.Success)
|
||||
{
|
||||
await _systemLogService.WriteAsync(
|
||||
Domain.Enums.SystemLogLevel.Warning,
|
||||
"Webhook",
|
||||
failureMessage,
|
||||
result.Detail,
|
||||
liveRoomId,
|
||||
recordSessionId,
|
||||
recordTaskId,
|
||||
cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
await _systemLogService.WriteAsync(
|
||||
Domain.Enums.SystemLogLevel.Info,
|
||||
"Webhook",
|
||||
successMessage,
|
||||
result.Detail,
|
||||
liveRoomId,
|
||||
recordSessionId,
|
||||
recordTaskId,
|
||||
cancellationToken);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "{FailureMessage}", failureMessage);
|
||||
await _systemLogService.WriteAsync(
|
||||
Domain.Enums.SystemLogLevel.Warning,
|
||||
"Webhook",
|
||||
failureMessage,
|
||||
ex.ToString(),
|
||||
liveRoomId,
|
||||
recordSessionId,
|
||||
recordTaskId,
|
||||
cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<WebhookSendResult> SendInternalAsync(
|
||||
SystemSettingsDto settings,
|
||||
object payload,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(settings.WebhookUrl))
|
||||
{
|
||||
throw new InvalidOperationException("Webhook URL is required.");
|
||||
}
|
||||
|
||||
var client = _httpClientFactory.CreateClient();
|
||||
client.Timeout = TimeSpan.FromSeconds(Math.Clamp(settings.WebhookTimeoutSeconds, 1, 300));
|
||||
|
||||
using var request = new HttpRequestMessage(HttpMethod.Post, settings.WebhookUrl.Trim())
|
||||
{
|
||||
Content = JsonContent.Create(payload)
|
||||
};
|
||||
|
||||
foreach (var header in ParseHeaders(settings.WebhookHeaders))
|
||||
{
|
||||
if (!request.Headers.TryAddWithoutValidation(header.Key, header.Value))
|
||||
{
|
||||
request.Content?.Headers.TryAddWithoutValidation(header.Key, header.Value);
|
||||
}
|
||||
}
|
||||
|
||||
using var response = await client.SendAsync(request, cancellationToken);
|
||||
var responseBody = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
var detail = BuildResponseDetail(settings.WebhookUrl, response, responseBody);
|
||||
return new WebhookSendResult(response.IsSuccessStatusCode, detail);
|
||||
}
|
||||
|
||||
private static object BuildPayload(
|
||||
string eventType,
|
||||
string summary,
|
||||
string? detail,
|
||||
LiveRoom? liveRoom,
|
||||
RecordTask? recordTask,
|
||||
string? source = null)
|
||||
{
|
||||
return new
|
||||
{
|
||||
appName = AppName,
|
||||
eventType,
|
||||
sentAtUtc = DateTimeOffset.UtcNow,
|
||||
summary,
|
||||
detail,
|
||||
source,
|
||||
liveRoom = liveRoom is null
|
||||
? null
|
||||
: new
|
||||
{
|
||||
id = liveRoom.Id,
|
||||
platform = liveRoom.Platform.ToString(),
|
||||
roomId = liveRoom.RoomId,
|
||||
title = liveRoom.Title,
|
||||
anchorName = liveRoom.AnchorName,
|
||||
sourceUrl = liveRoom.SourceUrl
|
||||
},
|
||||
recordTask = recordTask is null
|
||||
? null
|
||||
: new
|
||||
{
|
||||
id = recordTask.Id,
|
||||
recordSessionId = recordTask.RecordSessionId,
|
||||
status = recordTask.Status.ToString(),
|
||||
segmentIndex = recordTask.SegmentIndex,
|
||||
outputFilePath = recordTask.OutputFilePath
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyList<KeyValuePair<string, string>> ParseHeaders(string rawHeaders)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(rawHeaders))
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var results = new List<KeyValuePair<string, string>>();
|
||||
var lines = rawHeaders
|
||||
.Replace("\r\n", "\n", StringComparison.Ordinal)
|
||||
.Replace('\r', '\n')
|
||||
.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
|
||||
foreach (var line in lines)
|
||||
{
|
||||
var separatorIndex = line.IndexOf(':');
|
||||
if (separatorIndex <= 0)
|
||||
{
|
||||
throw new InvalidOperationException($"Invalid webhook header format: {line}");
|
||||
}
|
||||
|
||||
var name = line[..separatorIndex].Trim();
|
||||
var value = line[(separatorIndex + 1)..].Trim();
|
||||
if (string.IsNullOrWhiteSpace(name))
|
||||
{
|
||||
throw new InvalidOperationException($"Invalid webhook header format: {line}");
|
||||
}
|
||||
|
||||
results.Add(new KeyValuePair<string, string>(name, value));
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private static string BuildResponseDetail(string webhookUrl, HttpResponseMessage response, string responseBody)
|
||||
{
|
||||
var builder = new StringBuilder();
|
||||
builder.Append("url=").Append(webhookUrl.Trim());
|
||||
builder.Append("; status=").Append((int)response.StatusCode);
|
||||
builder.Append(' ').Append(response.ReasonPhrase);
|
||||
|
||||
var normalizedBody = responseBody.Trim();
|
||||
if (!string.IsNullOrWhiteSpace(normalizedBody))
|
||||
{
|
||||
var truncatedBody = normalizedBody.Length <= 1000 ? normalizedBody : normalizedBody[..1000];
|
||||
builder.Append("; body=").Append(truncatedBody);
|
||||
}
|
||||
|
||||
return builder.ToString();
|
||||
}
|
||||
|
||||
private sealed record WebhookSendResult(bool Success, string Detail);
|
||||
}
|
||||
Reference in New Issue
Block a user