4 Commits
Author SHA1 Message Date
nanxunandClaude Opus 4.8 a062bf84bf fix: avoid danmaku event name collision in session detail replay
Alias the replay player's danmakuEvents to replayDanmakuEvents so it no longer shadows another binding in the session detail view.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-11 17:52:18 +08:00
nanxunandClaude Opus 4.8 df70b64956 build: target linux-arm64 and streamline the Docker image build
Pin the WebApi to RuntimeIdentifier=linux-arm64 (framework-dependent) and rework the Dockerfile for ARM64: copy the full context up front, add QEMU emulation workarounds, and drop debug symbols / cap parallelism during publish.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-11 17:52:18 +08:00
nanxunandClaude Opus 4.8 5196ffa0f0 feat: add database circuit breaker and health-readiness endpoint
Introduce a process-wide DatabaseCircuitBreaker that fails fast when Postgres is unavailable (e.g. disk full) instead of letting every request burn doomed EF Core retries. CircuitAwareExecutionStrategy derives from NpgsqlRetryingExecutionStrategy and records success/failure around the public Execute/ExecuteAsync seams; background workers skip work and back off while the circuit is open; the exception middleware maps an open circuit (and other DB outages) to 503. Adds /health (liveness) and /health/ready (readiness, reporting circuit state), plus unit tests for the open/half-open transitions and non-transient SQLSTATE detection.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-11 17:52:18 +08:00
nanxunandClaude Opus 4.8 6da0690fd3 feat: batch-delete recording segments whose files are missing
Add a segment-level cleanup alongside the existing session-level one. DeleteMissingFileTasksAsync scans all non-active record tasks, keeps those whose video file no longer exists on disk, and deletes them individually via DeleteTasksAsync (which also drops any session left empty). Exposed as POST /record-tasks/delete-missing-files and a new "清理无文件分片" action in the record tasks view.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-11 17:52:18 +08:00
18 changed files with 745 additions and 33 deletions
@@ -32,7 +32,7 @@ const router = useRouter();
const { isMobile } = useViewport();
// Danmaku replay dialog
const { danmakuEvents, loading: danmakuLoading, error: danmakuError, loadTaskDanmaku, clear: clearDanmaku } = useDanmakuPlayer();
const { danmakuEvents: replayDanmakuEvents, loading: danmakuLoading, error: danmakuError, loadTaskDanmaku, clear: clearDanmaku } = useDanmakuPlayer();
const danmakuDialogVisible = ref(false);
const danmakuDialogTitle = ref("");
const danmakuPreviewUrl = ref("");
@@ -611,7 +611,7 @@ onMounted(loadDetail);
<DanmakuPlayer
v-else-if="danmakuPreviewUrl"
:video-src="danmakuPreviewUrl"
:danmaku-events="danmakuEvents"
:danmaku-events="replayDanmakuEvents"
/>
<div v-else class="preview-empty">无法加载该分片的预览</div>
</el-dialog>
+46 -8
View File
@@ -42,7 +42,7 @@ const uploadingTaskId = ref<string | null>(null);
const triggeringSegmentCompletedTaskId = ref<string | null>(null);
const cleanupOperation = ref<CleanupOperation | null>(null);
const deleteDialogVisible = ref(false);
const deleteDialogMode = ref<"tasks" | "sessions" | "conditional-sessions" | "mixed" | "empty-sessions">("tasks");
const deleteDialogMode = ref<"tasks" | "sessions" | "conditional-sessions" | "mixed" | "empty-sessions" | "missing-file-tasks">("tasks");
const deleteDialogTaskIds = ref<string[]>([]);
const deleteDialogSessionIds = ref<string[]>([]);
@@ -174,6 +174,10 @@ const deleteDialogEyebrow = computed(() => {
return "空闲会话清理";
}
if (deleteDialogMode.value === "missing-file-tasks") {
return "无文件分片清理";
}
return "删除确认";
});
const deleteDialogTitle = computed(() => {
@@ -193,6 +197,10 @@ const deleteDialogTitle = computed(() => {
return "清理无分片会话";
}
if (deleteDialogMode.value === "missing-file-tasks") {
return "清理无文件分片";
}
return "删除分片任务";
});
const deleteDialogLead = computed(() => {
@@ -212,6 +220,10 @@ const deleteDialogLead = computed(() => {
return `将自动找出所有没有任何分片任务的录制会话并批量删除。你也可以选择同时删除可能残留的本地文件。`;
}
if (deleteDialogMode.value === "missing-file-tasks") {
return `将自动找出所有视频文件已丢失的分片任务并批量删除(不限会话,只删命中的分片本身)。删除后若某个会话下不再有任何分片,会话也会一并清理;你也可以选择同时清理残留的弹幕 XML 文件。`;
}
return `将删除 ${deleteDialogTaskCount.value} 个已选择分片任务。你可以只移除数据库记录,也可以同时清理本地视频和弹幕 XML 文件。`;
});
const deleteDialogNote = computed(() => {
@@ -227,6 +239,10 @@ const deleteDialogNote = computed(() => {
return "仅清理没有任何关联分片的空会话,不影响有录制产物的会话。";
}
if (deleteDialogMode.value === "missing-file-tasks") {
return "以分片为单位判定:仅当分片的视频文件在磁盘上不存在时才会删除。正在录制或处理中的分片会自动跳过,有视频文件的分片不受影响。";
}
return "记录加文件会尝试删除视频文件和对应弹幕 XML。文件不存在时不会阻断删除,但会返回警告信息。";
});
function isActiveStatus(status: number) {
@@ -572,6 +588,13 @@ function openDeleteEmptySessionsDialog() {
deleteDialogVisible.value = true;
}
function openDeleteMissingFileTasksDialog() {
deleteDialogMode.value = "missing-file-tasks";
deleteDialogSessionIds.value = [];
deleteDialogTaskIds.value = [];
deleteDialogVisible.value = true;
}
async function confirmConditionalDelete() {
conditionalDialogVisible.value = false;
deleteDialogMode.value = "conditional-sessions";
@@ -642,6 +665,7 @@ async function confirmDelete(deleteFiles: boolean) {
const deletingSessions = currentMode === "sessions";
const deletingConditionalSessions = currentMode === "conditional-sessions";
const deletingEmptySessions = currentMode === "empty-sessions";
const deletingMissingFileTasks = currentMode === "missing-file-tasks";
const deletingMixed = currentMode === "mixed";
const hasSelection = deletingMixed
? deleteDialogSessionIds.value.length > 0 || deleteDialogTaskIds.value.length > 0
@@ -649,7 +673,7 @@ async function confirmDelete(deleteFiles: boolean) {
? deleteDialogSessionIds.value.length > 0
: deleteDialogTaskIds.value.length > 0;
if (!deletingConditionalSessions && !deletingEmptySessions && !hasSelection) {
if (!deletingConditionalSessions && !deletingEmptySessions && !deletingMissingFileTasks && !hasSelection) {
closeDeleteDialog();
return;
}
@@ -685,6 +709,11 @@ async function confirmDelete(deleteFiles: boolean) {
deleteFiles
});
createdCleanupOperation = data;
} else if (deletingMissingFileTasks) {
const { data } = await apiClient.post<DeleteCompletedRecordTasksResult>("/record-tasks/delete-missing-files", {
deleteFiles
});
deletedTaskResult = data;
} else if (deletingMixed) {
if (deleteDialogTaskIds.value.length > 0) {
const { data } = await apiClient.post<DeleteCompletedRecordTasksResult>("/record-tasks/delete", {
@@ -723,7 +752,17 @@ async function confirmDelete(deleteFiles: boolean) {
deleteDialogVisible.value = false;
resetDeleteDialogState();
if (deleteDialogMode.value === "tasks") {
if (currentMode === "missing-file-tasks") {
ElMessage.success(
deletedTaskResult
? `已清理 ${deletedTaskResult.deletedTaskIds.length} 个无文件分片。`
: "未发现可清理的无文件分片。"
);
await loadSessions({ resetPanels: false, resetSelection: false });
return;
}
if (currentMode === "tasks") {
ElMessage.success(
deletedTaskResult ? `已删除 ${deletedTaskResult.deletedTaskIds.length} 个分片任务。` : "已删除分片任务。"
);
@@ -737,11 +776,7 @@ async function confirmDelete(deleteFiles: boolean) {
return;
}
ElMessage.success(
currentMode === "tasks"
? "已删除分片任务。"
: "后台清理任务已创建,页面会自动轮询进度。"
);
ElMessage.success("后台清理任务已创建,页面会自动轮询进度。");
} finally {
deleting.value = false;
}
@@ -931,6 +966,9 @@ onBeforeUnmount(() => {
<el-button plain :loading="deleting" @click="openDeleteEmptySessionsDialog">
清理无分片会话
</el-button>
<el-button plain :loading="deleting" @click="openDeleteMissingFileTasksDialog">
清理无文件分片
</el-button>
</div>
</div>
@@ -108,6 +108,11 @@ public sealed class DeleteCompletedRecordTasksRequest
public bool DeleteFiles { get; set; }
}
public sealed class DeleteMissingFileRecordTasksRequest
{
public bool DeleteFiles { get; set; }
}
public sealed class DeleteCompletedRecordTasksResultDto
{
public required IReadOnlyList<Guid> DeletedTaskIds { get; init; }
@@ -0,0 +1,15 @@
namespace LiveRecorder.Application.Models.Reports;
public sealed class HealthReadyResponse
{
public string Status { get; set; } = "ready";
public DateTimeOffset Timestamp { get; set; }
public DatabaseHealthStatus Database { get; set; } = new();
}
public sealed class DatabaseHealthStatus
{
public bool Reachable { get; set; }
public string? Reason { get; set; }
public int ConsecutiveFailures { get; set; }
}
@@ -504,6 +504,32 @@ public sealed class RecordService
};
}
public async Task<DeleteCompletedRecordTasksResultDto> DeleteMissingFileTasksAsync(
DeleteMissingFileRecordTasksRequest request,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(request);
var tasks = await _recordTaskRepository.ListAsync(null, cancellationToken);
var missingFileTaskIds = tasks
.Where(static task => !IsActiveTaskStatus(task.Status) && !HasExistingVideoFile(task))
.Select(static task => task.Id)
.ToArray();
if (missingFileTaskIds.Length == 0)
{
return CreateEmptyDeleteResult();
}
return await DeleteTasksAsync(
new DeleteCompletedRecordTasksRequest
{
TaskIds = missingFileTaskIds,
DeleteFiles = request.DeleteFiles
},
cancellationToken);
}
public async Task<RecordPreviewTicketDto> CreatePreviewTicketAsync(Guid id, string mediaBaseUrl, CancellationToken cancellationToken = default)
{
var previewTicket = await _recordMediaService.CreatePreviewTicketAsync(id, cancellationToken);
@@ -823,6 +849,24 @@ public sealed class RecordService
or RecordTaskStatus.Stopping
or RecordTaskStatus.Processing;
private static bool HasExistingVideoFile(RecordTask task)
{
var candidatePath = !string.IsNullOrWhiteSpace(task.Result?.FilePath)
? task.Result!.FilePath
: task.OutputFilePath;
if (string.IsNullOrWhiteSpace(candidatePath))
{
return false;
}
var resolvedPath = Path.IsPathRooted(candidatePath)
? candidatePath
: Path.GetFullPath(candidatePath, AppContext.BaseDirectory);
return File.Exists(resolvedPath);
}
private static bool IsActiveSessionStatus(RecordSessionStatus status) =>
status is RecordSessionStatus.Starting or RecordSessionStatus.Running or RecordSessionStatus.Stopping;
@@ -0,0 +1,103 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Storage;
using Npgsql.EntityFrameworkCore.PostgreSQL;
namespace LiveRecorder.Infrastructure.Persistence;
/// <summary>
/// Custom EF Core execution strategy that integrates with <see cref="DatabaseCircuitBreaker"/>.
///
/// When the circuit is open, operations fail immediately without retrying.
/// When a non-transient error occurs (e.g. disk_full), the operation does not retry.
/// After each failure, the circuit is recorded, and after each success, the circuit is reset.
/// </summary>
public sealed class CircuitAwareExecutionStrategy : NpgsqlRetryingExecutionStrategy
{
private static readonly TimeSpan DefaultMaxRetryDelay = TimeSpan.FromSeconds(15);
public CircuitAwareExecutionStrategy(
ExecutionStrategyDependencies dependencies,
int maxRetryCount,
TimeSpan maxRetryDelay)
: base(dependencies, maxRetryCount, maxRetryDelay, errorCodesToAdd: null)
{
}
public CircuitAwareExecutionStrategy(
ExecutionStrategyDependencies dependencies,
int maxRetryCount,
TimeSpan maxRetryDelay,
ICollection<string>? errorCodesToAdd)
: base(dependencies, maxRetryCount, maxRetryDelay, errorCodesToAdd)
{
}
protected override bool ShouldRetryOn(Exception? exception)
{
// Fast-fail: circuit is open
if (DatabaseCircuitBreaker.IsOpen)
{
return false;
}
// Fast-fail: non-transient errors (disk_full, out_of_memory, etc.)
if (DatabaseCircuitBreaker.IsNonTransient(exception))
{
return false;
}
// Delegate to default Npgsql retry logic for transient errors
return base.ShouldRetryOn(exception);
}
protected override void OnFirstExecution()
{
// If circuit is open, throw immediately before even attempting
if (DatabaseCircuitBreaker.IsOpen)
{
throw new DatabaseCircuitOpenException(
$"Database circuit breaker is open. Consecutive failures: {DatabaseCircuitBreaker.ConsecutiveFailures}. " +
$"Circuit opened at: {DatabaseCircuitBreaker.OpenedAt:O}.");
}
base.OnFirstExecution();
}
public override TResult Execute<TState, TResult>(
TState state,
Func<DbContext, TState, TResult> operation,
Func<DbContext, TState, ExecutionResult<TResult>>? verifySucceeded)
{
try
{
var result = base.Execute(state, operation, verifySucceeded);
DatabaseCircuitBreaker.RecordSuccess();
return result;
}
catch
{
DatabaseCircuitBreaker.RecordFailure();
throw;
}
}
public override async Task<TResult> ExecuteAsync<TState, TResult>(
TState state,
Func<DbContext, TState, CancellationToken, Task<TResult>> operation,
Func<DbContext, TState, CancellationToken, Task<ExecutionResult<TResult>>>? verifySucceeded,
CancellationToken cancellationToken)
{
try
{
var result = await base.ExecuteAsync(state, operation, verifySucceeded, cancellationToken)
.ConfigureAwait(false);
DatabaseCircuitBreaker.RecordSuccess();
return result;
}
catch
{
DatabaseCircuitBreaker.RecordFailure();
throw;
}
}
}
@@ -0,0 +1,121 @@
using Microsoft.Extensions.Logging;
namespace LiveRecorder.Infrastructure.Persistence;
/// <summary>
/// Thread-safe static circuit breaker for database operations.
/// When the database becomes unavailable (e.g. disk full), this prevents
/// every API request from wasting 45 seconds on doomed retries.
/// </summary>
public static class DatabaseCircuitBreaker
{
private static readonly object Lock = new();
/// <summary>Consecutive failures before the circuit opens.</summary>
private const int FailureThreshold = 5;
/// <summary>How long the circuit stays open before allowing a probe.</summary>
private static readonly TimeSpan BreakDuration = TimeSpan.FromSeconds(30);
/// <summary>Npgsql error codes that are NOT transient — retrying is futile.</summary>
private static readonly HashSet<string> NonTransientCodes = new(StringComparer.Ordinal)
{
"53100", // disk_full
"53200", // out_of_memory
"53300", // too_many_connections
"08006", // connection_failure (persistent)
"57P03", // cannot_connect_now
"42601", // syntax_error (bug, not transient)
"42501", // insufficient_privilege
"3D000", // invalid_catalog_name
"28P01", // invalid_password
};
private static int _consecutiveFailures;
private static DateTimeOffset _openedAt = DateTimeOffset.MinValue;
/// <summary>Whether the circuit is currently open (failing fast).</summary>
public static bool IsOpen
{
get
{
if (_consecutiveFailures < FailureThreshold)
{
return false;
}
if (DateTimeOffset.UtcNow - _openedAt > BreakDuration)
{
// Transition to half-open: allow one probe
lock (Lock)
{
if (_consecutiveFailures >= FailureThreshold &&
DateTimeOffset.UtcNow - _openedAt > BreakDuration)
{
// Reset to just below threshold so the next call probes
_consecutiveFailures = FailureThreshold - 1;
}
}
return false;
}
return true;
}
}
public static int ConsecutiveFailures => Volatile.Read(ref _consecutiveFailures);
public static DateTimeOffset OpenedAt => _openedAt;
/// <summary>Record a successful database operation.</summary>
public static void RecordSuccess()
{
lock (Lock)
{
_consecutiveFailures = 0;
}
}
/// <summary>Record a failed database operation.</summary>
public static void RecordFailure()
{
lock (Lock)
{
_consecutiveFailures++;
if (_consecutiveFailures >= FailureThreshold)
{
_openedAt = DateTimeOffset.UtcNow;
}
}
}
/// <summary>
/// Check whether a given exception is a non-transient database error
/// that should NOT be retried. Returns true if retrying would be futile.
/// </summary>
public static bool IsNonTransient(Exception? ex)
{
while (ex is not null)
{
if (ex is Npgsql.NpgsqlException npgEx && npgEx.SqlState is { Length: 5 } state)
{
return NonTransientCodes.Contains(state);
}
ex = ex.InnerException;
}
return false;
}
/// <summary>Log the current circuit state.</summary>
public static void LogState(ILogger logger)
{
logger.LogInformation(
"DatabaseCircuitBreaker state: Open={IsOpen}, ConsecutiveFailures={Failures}, OpenedAt={OpenedAt}",
IsOpen,
ConsecutiveFailures,
OpenedAt == DateTimeOffset.MinValue ? "never" : OpenedAt.ToString("O"));
}
}
@@ -0,0 +1,16 @@
namespace LiveRecorder.Infrastructure.Persistence;
/// <summary>
/// Thrown when a database operation is rejected because the circuit breaker is open.
/// This is a fast-fail — the request will not be retried.
/// </summary>
public sealed class DatabaseCircuitOpenException : InvalidOperationException
{
public DatabaseCircuitOpenException(string message) : base(message)
{
}
public DatabaseCircuitOpenException(string message, Exception innerException) : base(message, innerException)
{
}
}
@@ -1,3 +1,4 @@
using LiveRecorder.Infrastructure.Persistence;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
@@ -7,6 +8,8 @@ namespace LiveRecorder.Infrastructure.Services;
public sealed class CleanupOperationBackgroundService : BackgroundService
{
private static readonly TimeSpan IdleDelay = TimeSpan.FromSeconds(2);
private static readonly TimeSpan ErrorBaseDelay = TimeSpan.FromSeconds(5);
private static readonly TimeSpan ErrorMaxDelay = TimeSpan.FromMinutes(5);
private readonly IServiceScopeFactory _serviceScopeFactory;
private readonly ILogger<CleanupOperationBackgroundService> _logger;
@@ -20,6 +23,7 @@ public sealed class CleanupOperationBackgroundService : BackgroundService
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
// Startup: requeue interrupted operations
try
{
using var startupScope = _serviceScopeFactory.CreateScope();
@@ -31,17 +35,36 @@ public sealed class CleanupOperationBackgroundService : BackgroundService
_logger.LogWarning(ex, "Failed to requeue interrupted cleanup operations at startup");
}
var consecutiveErrors = 0;
while (!stoppingToken.IsCancellationRequested)
{
try
{
// Skip processing if the circuit is open — don't waste resources
if (DatabaseCircuitBreaker.IsOpen)
{
consecutiveErrors = await DelayWithBackoff(ErrorBaseDelay, ErrorMaxDelay, consecutiveErrors, stoppingToken);
continue;
}
using var scope = _serviceScopeFactory.CreateScope();
var coordinator = scope.ServiceProvider.GetRequiredService<CleanupOperationCoordinator>();
var processed = await coordinator.ProcessNextQueuedOperationAsync(stoppingToken);
if (processed)
{
consecutiveErrors = 0;
continue;
}
// No queued operations — idle delay
consecutiveErrors = 0;
await Task.Delay(IdleDelay, stoppingToken);
}
catch (DatabaseCircuitOpenException)
{
consecutiveErrors = await DelayWithBackoff(ErrorBaseDelay, ErrorMaxDelay, consecutiveErrors, stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
@@ -49,17 +72,47 @@ public sealed class CleanupOperationBackgroundService : BackgroundService
}
catch (Exception ex)
{
_logger.LogError(ex, "Cleanup operation background worker failed");
}
var isDatabaseError = DatabaseCircuitBreaker.IsNonTransient(ex) ||
ex is Npgsql.NpgsqlException ||
ex is Microsoft.EntityFrameworkCore.DbUpdateException;
try
{
await Task.Delay(IdleDelay, stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
if (isDatabaseError)
{
DatabaseCircuitBreaker.RecordFailure();
_logger.LogWarning(ex,
"Cleanup background worker: database error (#{ErrorCount}). Circuit state: Open={IsOpen}, Failures={Failures}",
consecutiveErrors + 1,
DatabaseCircuitBreaker.IsOpen,
DatabaseCircuitBreaker.ConsecutiveFailures);
}
else
{
_logger.LogError(ex, "Cleanup operation background worker failed");
}
consecutiveErrors = await DelayWithBackoff(ErrorBaseDelay, ErrorMaxDelay, consecutiveErrors, stoppingToken);
}
}
}
private static async Task<int> DelayWithBackoff(
TimeSpan baseDelay, TimeSpan maxDelay, int errorCount, CancellationToken cancellationToken)
{
errorCount++;
// Exponential backoff: 5s, 10s, 20s, 40s, 80s, 160s, capping at 5min
var factor = Math.Pow(2, Math.Min(errorCount - 1, 6));
var delay = TimeSpan.FromMilliseconds(
Math.Min(baseDelay.TotalMilliseconds * factor, maxDelay.TotalMilliseconds));
try
{
await Task.Delay(delay, cancellationToken);
}
catch (OperationCanceledException)
{
// Swallow — loop will exit on next iteration
}
return errorCount;
}
}
@@ -146,8 +146,20 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
{
break;
}
catch (DatabaseCircuitOpenException)
{
// Circuit is open — skip this iteration and wait
_logger.LogDebug("Polling loop skipped: database circuit breaker is open");
delay = TimeSpan.FromSeconds(30);
}
catch (Exception ex)
{
// Record database failures to the circuit breaker
if (IsTransientDatabaseException(ex) || DatabaseCircuitBreaker.IsNonTransient(ex))
{
DatabaseCircuitBreaker.RecordFailure();
}
_logger.LogError(ex, "Background live room polling failed");
try
@@ -1,3 +1,4 @@
using LiveRecorder.Infrastructure.Persistence;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
@@ -7,7 +8,8 @@ namespace LiveRecorder.Infrastructure.Services;
public sealed class RetentionCleanupBackgroundService : BackgroundService
{
private static readonly TimeSpan CleanupInterval = TimeSpan.FromHours(24);
private static readonly TimeSpan ErrorBaseDelay = TimeSpan.FromSeconds(10);
private static readonly TimeSpan ErrorMaxDelay = TimeSpan.FromMinutes(5);
private readonly IServiceScopeFactory _serviceScopeFactory;
private readonly ILogger<RetentionCleanupBackgroundService> _logger;
@@ -21,13 +23,31 @@ public sealed class RetentionCleanupBackgroundService : BackgroundService
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var consecutiveErrors = 0;
while (!stoppingToken.IsCancellationRequested)
{
try
{
// Skip if circuit is already open
if (DatabaseCircuitBreaker.IsOpen)
{
_logger.LogDebug("Retention cleanup skipped: database circuit breaker is open");
await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken);
continue;
}
using var scope = _serviceScopeFactory.CreateScope();
var cleanupService = scope.ServiceProvider.GetRequiredService<RetentionCleanupService>();
await cleanupService.TryEnqueueAsync(ignoreEnabledSetting: false, cancellationToken: stoppingToken);
consecutiveErrors = 0;
}
catch (DatabaseCircuitOpenException)
{
// Silent — circuit is already logged
await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken);
continue;
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
@@ -35,7 +55,38 @@ public sealed class RetentionCleanupBackgroundService : BackgroundService
}
catch (Exception ex)
{
_logger.LogError(ex, "Retention cleanup background task failed");
var isDatabaseError = DatabaseCircuitBreaker.IsNonTransient(ex) ||
ex is Npgsql.NpgsqlException ||
ex is Microsoft.EntityFrameworkCore.DbUpdateException;
consecutiveErrors++;
var delay = TimeSpan.FromMilliseconds(
Math.Min(ErrorBaseDelay.TotalMilliseconds * Math.Pow(2, Math.Min(consecutiveErrors - 1, 6)),
ErrorMaxDelay.TotalMilliseconds));
if (isDatabaseError)
{
DatabaseCircuitBreaker.RecordFailure();
_logger.LogWarning(ex,
"Retention cleanup: database error (#{ErrorCount}). Circuit: Open={IsOpen}, Failures={Failures}",
consecutiveErrors,
DatabaseCircuitBreaker.IsOpen,
DatabaseCircuitBreaker.ConsecutiveFailures);
}
else
{
_logger.LogError(ex, "Retention cleanup background task failed");
}
try
{
await Task.Delay(delay, stoppingToken);
}
catch (OperationCanceledException)
{
break;
}
continue;
}
try
@@ -61,6 +61,12 @@ public sealed class RecordTasksController : ControllerBase
CancellationToken cancellationToken) =>
Ok(await _recordService.DeleteTasksAsync(request, cancellationToken));
[HttpPost("delete-missing-files")]
public async Task<ActionResult<DeleteCompletedRecordTasksResultDto>> DeleteMissingFiles(
[FromBody] DeleteMissingFileRecordTasksRequest request,
CancellationToken cancellationToken) =>
Ok(await _recordService.DeleteMissingFileTasksAsync(request, cancellationToken));
[HttpPost("{id:guid}/preview-ticket")]
public async Task<ActionResult<RecordPreviewTicketDto>> CreatePreviewTicket(Guid id, CancellationToken cancellationToken)
{
+8 -9
View File
@@ -4,16 +4,15 @@ ARG DOTNET_RUNTIME_IMAGE=mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim
FROM ${DOTNET_SDK_IMAGE} AS build
WORKDIR /src
COPY ["LiveRecorder.sln", "./"]
COPY ["src/LiveRecorder.Domain/LiveRecorder.Domain.csproj", "src/LiveRecorder.Domain/"]
COPY ["src/LiveRecorder.Application/LiveRecorder.Application.csproj", "src/LiveRecorder.Application/"]
COPY ["src/LiveRecorder.Infrastructure/LiveRecorder.Infrastructure.csproj", "src/LiveRecorder.Infrastructure/"]
COPY ["src/LiveRecorder.WebApi/LiveRecorder.WebApi.csproj", "src/LiveRecorder.WebApi/"]
RUN dotnet restore "src/LiveRecorder.WebApi/LiveRecorder.WebApi.csproj"
COPY . .
RUN dotnet publish "src/LiveRecorder.WebApi/LiveRecorder.WebApi.csproj" -c Release -o /app/publish /p:UseAppHost=false
# Workaround for QEMU ARM64 emulation
ENV DOTNET_EnableWriteXorExecute=0
ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1
ENV DOTNET_GCConserveMemory=9
RUN dotnet restore "src/LiveRecorder.WebApi/LiveRecorder.WebApi.csproj" -p:RestoreUseStaticGraphEvaluation=true
RUN dotnet publish "src/LiveRecorder.WebApi/LiveRecorder.WebApi.csproj" -c Release -o /app/publish /p:UseAppHost=false /p:DebugType=None /p:DebugSymbols=false /maxcpucount:1
FROM ${DOTNET_RUNTIME_IMAGE} AS runtime
@@ -17,6 +17,8 @@
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<RuntimeIdentifier>linux-arm64</RuntimeIdentifier>
<SelfContained>false</SelfContained>
</PropertyGroup>
</Project>
@@ -1,3 +1,7 @@
using LiveRecorder.Infrastructure.Persistence;
using Microsoft.EntityFrameworkCore;
using Npgsql;
namespace LiveRecorder.WebApi.Middleware;
public sealed class ExceptionHandlingMiddleware
@@ -17,22 +21,80 @@ public sealed class ExceptionHandlingMiddleware
{
await _next(context);
}
catch (DatabaseCircuitOpenException ex)
{
// Circuit is open — fast-fail with 503
_logger.LogWarning(ex, "Database circuit breaker is open, returning 503");
context.Response.StatusCode = StatusCodes.Status503ServiceUnavailable;
await context.Response.WriteAsJsonAsync(new
{
message = "Database temporarily unavailable",
detail = "The database is currently unreachable. Requests will be accepted again once connectivity is restored.",
retryAfterSeconds = 30
});
}
catch (Exception ex)
{
_logger.LogError(ex, "Unhandled exception");
context.Response.StatusCode = ex switch
// Detect database-related exceptions and return 503 instead of 500
var statusCode = ex switch
{
DatabaseCircuitOpenException => StatusCodes.Status503ServiceUnavailable,
KeyNotFoundException => StatusCodes.Status404NotFound,
InvalidOperationException => StatusCodes.Status400BadRequest,
NotSupportedException => StatusCodes.Status400BadRequest,
_ when IsDatabaseException(ex) => StatusCodes.Status503ServiceUnavailable,
_ => StatusCodes.Status500InternalServerError
};
context.Response.StatusCode = statusCode;
await context.Response.WriteAsJsonAsync(new
{
message = ex.Message,
detail = context.Response.StatusCode == StatusCodes.Status500InternalServerError ? "Internal Server Error" : null
message = statusCode == StatusCodes.Status503ServiceUnavailable
? "Database temporarily unavailable"
: ex.Message,
detail = statusCode switch
{
StatusCodes.Status503ServiceUnavailable => "The database is currently unreachable. Please retry later.",
StatusCodes.Status500InternalServerError => "Internal Server Error",
_ => null
}
});
}
}
/// <summary>
/// Check if an exception (or any of its inner exceptions) is a database-related error
/// that should be surfaced as 503 Service Unavailable.
/// </summary>
private static bool IsDatabaseException(Exception ex)
{
var current = ex;
while (current is not null)
{
if (current is NpgsqlException or DbUpdateException)
{
return true;
}
if (current is InvalidOperationException ioEx &&
(ioEx.Message.Contains("database", StringComparison.OrdinalIgnoreCase) ||
ioEx.Message.Contains("connection", StringComparison.OrdinalIgnoreCase) ||
ioEx.Message.Contains("Npgsql", StringComparison.OrdinalIgnoreCase)))
{
return true;
}
if (current is TimeoutException && current.Message.Contains("database", StringComparison.OrdinalIgnoreCase))
{
return true;
}
current = current.InnerException;
}
return false;
}
}
+62
View File
@@ -10,6 +10,7 @@ using LiveRecorder.Application.Abstractions.Recording;
using LiveRecorder.Application.Abstractions.Scripting;
using LiveRecorder.Application.Abstractions.Settings;
using LiveRecorder.Application.Abstractions.Storage;
using LiveRecorder.Application.Models.Reports;
using LiveRecorder.Application.Services;
using LiveRecorder.Infrastructure.Persistence;
using LiveRecorder.Infrastructure.Persistence.Repositories;
@@ -31,6 +32,7 @@ using LiveRecorder.Infrastructure.Platforms.YouTube;
using LiveRecorder.Infrastructure.Services;
using LiveRecorder.WebApi.Middleware;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Storage;
using Microsoft.OpenApi.Models;
using Npgsql;
@@ -149,6 +151,11 @@ builder.Services.AddDbContext<LiveRecorderDbContext>(options =>
maxRetryCount: 3,
maxRetryDelay: TimeSpan.FromSeconds(15),
errorCodesToAdd: null);
npgsql.ExecutionStrategy(dependencies =>
new CircuitAwareExecutionStrategy(
dependencies,
maxRetryCount: 3,
maxRetryDelay: TimeSpan.FromSeconds(15)));
}));
builder.Services.AddScoped<IUnitOfWork>(provider => provider.GetRequiredService<LiveRecorderDbContext>());
@@ -231,6 +238,61 @@ app.UseMiddleware<ApiTokenAuthenticationMiddleware>();
app.MapGet("/", () => Results.Redirect("/swagger"));
app.MapControllers();
// ── Health check endpoints ────────────────────────────────────────────
// /health — liveness: is the process alive and responding?
app.MapGet("/health", () => Results.Ok(new { status = "healthy", timestamp = DateTimeOffset.UtcNow }));
// /health/ready — readiness: can we reach the database?
app.MapGet("/health/ready", async (CancellationToken cancellationToken) =>
{
var readyResult = new HealthReadyResponse
{
Status = "ready",
Timestamp = DateTimeOffset.UtcNow,
Database = new DatabaseHealthStatus()
};
if (DatabaseCircuitBreaker.IsOpen)
{
readyResult.Status = "degraded";
readyResult.Database = new DatabaseHealthStatus
{
Reachable = false,
Reason = "Circuit breaker is open",
ConsecutiveFailures = DatabaseCircuitBreaker.ConsecutiveFailures
};
return Results.Json(readyResult, statusCode: StatusCodes.Status503ServiceUnavailable);
}
try
{
// Lightweight probe: just test connectivity with a 2-second timeout
await using var connection = new NpgsqlConnection(connectionStringBuilder.ConnectionString);
var probeCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
probeCts.CancelAfter(TimeSpan.FromSeconds(2));
await connection.OpenAsync(probeCts.Token);
await using var cmd = connection.CreateCommand();
cmd.CommandText = "SELECT 1";
await cmd.ExecuteScalarAsync(probeCts.Token);
readyResult.Database = new DatabaseHealthStatus { Reachable = true };
return Results.Ok(readyResult);
}
catch (Exception ex)
{
DatabaseCircuitBreaker.RecordFailure();
readyResult.Status = "unhealthy";
readyResult.Database = new DatabaseHealthStatus
{
Reachable = false,
Reason = ex.Message,
ConsecutiveFailures = DatabaseCircuitBreaker.ConsecutiveFailures
};
return Results.Json(readyResult, statusCode: StatusCodes.Status503ServiceUnavailable);
}
});
using (var scope = app.Services.CreateScope())
{
var initializer = scope.ServiceProvider.GetRequiredService<DatabaseInitializer>();
@@ -0,0 +1,122 @@
using System.Reflection;
using LiveRecorder.Infrastructure.Persistence;
using Npgsql;
namespace LiveRecorder.Tests;
public sealed class DatabaseCircuitBreakerTests
{
// Mirrors the private FailureThreshold in DatabaseCircuitBreaker so the assertions
// read intentionally. Keep in sync if the production threshold changes.
private const int FailureThreshold = 5;
public DatabaseCircuitBreakerTests()
{
// The breaker is a process-wide static, so reset it to a known closed state
// before each test to keep cases independent.
DatabaseCircuitBreaker.RecordSuccess();
SetOpenedAt(DateTimeOffset.MinValue);
}
[Fact]
public void RecordFailure_below_threshold_keeps_circuit_closed()
{
for (var i = 0; i < FailureThreshold - 1; i++)
{
DatabaseCircuitBreaker.RecordFailure();
}
Assert.False(DatabaseCircuitBreaker.IsOpen);
Assert.Equal(FailureThreshold - 1, DatabaseCircuitBreaker.ConsecutiveFailures);
}
[Fact]
public void RecordFailure_at_threshold_opens_circuit()
{
for (var i = 0; i < FailureThreshold; i++)
{
DatabaseCircuitBreaker.RecordFailure();
}
Assert.True(DatabaseCircuitBreaker.IsOpen);
Assert.Equal(FailureThreshold, DatabaseCircuitBreaker.ConsecutiveFailures);
Assert.NotEqual(DateTimeOffset.MinValue, DatabaseCircuitBreaker.OpenedAt);
}
[Fact]
public void RecordSuccess_closes_circuit_and_resets_failures()
{
for (var i = 0; i < FailureThreshold; i++)
{
DatabaseCircuitBreaker.RecordFailure();
}
Assert.True(DatabaseCircuitBreaker.IsOpen);
DatabaseCircuitBreaker.RecordSuccess();
Assert.False(DatabaseCircuitBreaker.IsOpen);
Assert.Equal(0, DatabaseCircuitBreaker.ConsecutiveFailures);
}
[Fact]
public void IsOpen_after_break_duration_transitions_to_half_open()
{
for (var i = 0; i < FailureThreshold; i++)
{
DatabaseCircuitBreaker.RecordFailure();
}
Assert.True(DatabaseCircuitBreaker.IsOpen);
// The breaker reads DateTimeOffset.UtcNow directly (no injectable clock), so move
// the recorded open time past the 30s break window to simulate it elapsing.
SetOpenedAt(DateTimeOffset.UtcNow - TimeSpan.FromSeconds(31));
// The first read past the window half-opens: it permits one probe and drops the
// failure count to one below the threshold.
Assert.False(DatabaseCircuitBreaker.IsOpen);
Assert.Equal(FailureThreshold - 1, DatabaseCircuitBreaker.ConsecutiveFailures);
}
[Theory]
[InlineData("53100")] // disk_full
[InlineData("53300")] // too_many_connections
[InlineData("28P01")] // invalid_password
public void IsNonTransient_returns_true_for_known_fatal_sql_states(string sqlState)
{
var exception = new PostgresException("fatal", "FATAL", "FATAL", sqlState);
Assert.True(DatabaseCircuitBreaker.IsNonTransient(exception));
}
[Fact]
public void IsNonTransient_returns_false_for_transient_sql_state()
{
// 40001 = serialization_failure, which is retryable and not in the fatal set.
var exception = new PostgresException("retry me", "ERROR", "ERROR", "40001");
Assert.False(DatabaseCircuitBreaker.IsNonTransient(exception));
}
[Fact]
public void IsNonTransient_unwraps_inner_exceptions()
{
var inner = new PostgresException("disk full", "FATAL", "FATAL", "53100");
var wrapper = new InvalidOperationException("save failed", inner);
Assert.True(DatabaseCircuitBreaker.IsNonTransient(wrapper));
}
[Fact]
public void IsNonTransient_returns_false_for_null_and_non_postgres_exceptions()
{
Assert.False(DatabaseCircuitBreaker.IsNonTransient(null));
Assert.False(DatabaseCircuitBreaker.IsNonTransient(new InvalidOperationException("boom")));
}
private static void SetOpenedAt(DateTimeOffset value) =>
typeof(DatabaseCircuitBreaker)
.GetField("_openedAt", BindingFlags.NonPublic | BindingFlags.Static)!
.SetValue(null, value);
}
@@ -20,6 +20,7 @@
<ItemGroup>
<ProjectReference Include="..\..\src\LiveRecorder.Application\LiveRecorder.Application.csproj" />
<ProjectReference Include="..\..\src\LiveRecorder.Domain\LiveRecorder.Domain.csproj" />
<ProjectReference Include="..\..\src\LiveRecorder.Infrastructure\LiveRecorder.Infrastructure.csproj" />
</ItemGroup>
</Project>