Compare commits
4
Commits
e640d5b5cc
...
a062bf84bf
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a062bf84bf | ||
|
|
df70b64956 | ||
|
|
5196ffa0f0 | ||
|
|
6da0690fd3 |
@@ -32,7 +32,7 @@ const router = useRouter();
|
||||
const { isMobile } = useViewport();
|
||||
|
||||
// Danmaku replay dialog
|
||||
const { danmakuEvents, loading: danmakuLoading, error: danmakuError, loadTaskDanmaku, clear: clearDanmaku } = useDanmakuPlayer();
|
||||
const { danmakuEvents: replayDanmakuEvents, loading: danmakuLoading, error: danmakuError, loadTaskDanmaku, clear: clearDanmaku } = useDanmakuPlayer();
|
||||
const danmakuDialogVisible = ref(false);
|
||||
const danmakuDialogTitle = ref("");
|
||||
const danmakuPreviewUrl = ref("");
|
||||
@@ -611,7 +611,7 @@ onMounted(loadDetail);
|
||||
<DanmakuPlayer
|
||||
v-else-if="danmakuPreviewUrl"
|
||||
:video-src="danmakuPreviewUrl"
|
||||
:danmaku-events="danmakuEvents"
|
||||
:danmaku-events="replayDanmakuEvents"
|
||||
/>
|
||||
<div v-else class="preview-empty">无法加载该分片的预览。</div>
|
||||
</el-dialog>
|
||||
|
||||
@@ -42,7 +42,7 @@ const uploadingTaskId = ref<string | null>(null);
|
||||
const triggeringSegmentCompletedTaskId = ref<string | null>(null);
|
||||
const cleanupOperation = ref<CleanupOperation | null>(null);
|
||||
const deleteDialogVisible = ref(false);
|
||||
const deleteDialogMode = ref<"tasks" | "sessions" | "conditional-sessions" | "mixed" | "empty-sessions">("tasks");
|
||||
const deleteDialogMode = ref<"tasks" | "sessions" | "conditional-sessions" | "mixed" | "empty-sessions" | "missing-file-tasks">("tasks");
|
||||
const deleteDialogTaskIds = ref<string[]>([]);
|
||||
const deleteDialogSessionIds = ref<string[]>([]);
|
||||
|
||||
@@ -174,6 +174,10 @@ const deleteDialogEyebrow = computed(() => {
|
||||
return "空闲会话清理";
|
||||
}
|
||||
|
||||
if (deleteDialogMode.value === "missing-file-tasks") {
|
||||
return "无文件分片清理";
|
||||
}
|
||||
|
||||
return "删除确认";
|
||||
});
|
||||
const deleteDialogTitle = computed(() => {
|
||||
@@ -193,6 +197,10 @@ const deleteDialogTitle = computed(() => {
|
||||
return "清理无分片会话";
|
||||
}
|
||||
|
||||
if (deleteDialogMode.value === "missing-file-tasks") {
|
||||
return "清理无文件分片";
|
||||
}
|
||||
|
||||
return "删除分片任务";
|
||||
});
|
||||
const deleteDialogLead = computed(() => {
|
||||
@@ -212,6 +220,10 @@ const deleteDialogLead = computed(() => {
|
||||
return `将自动找出所有没有任何分片任务的录制会话并批量删除。你也可以选择同时删除可能残留的本地文件。`;
|
||||
}
|
||||
|
||||
if (deleteDialogMode.value === "missing-file-tasks") {
|
||||
return `将自动找出所有视频文件已丢失的分片任务并批量删除(不限会话,只删命中的分片本身)。删除后若某个会话下不再有任何分片,会话也会一并清理;你也可以选择同时清理残留的弹幕 XML 文件。`;
|
||||
}
|
||||
|
||||
return `将删除 ${deleteDialogTaskCount.value} 个已选择分片任务。你可以只移除数据库记录,也可以同时清理本地视频和弹幕 XML 文件。`;
|
||||
});
|
||||
const deleteDialogNote = computed(() => {
|
||||
@@ -227,6 +239,10 @@ const deleteDialogNote = computed(() => {
|
||||
return "仅清理没有任何关联分片的空会话,不影响有录制产物的会话。";
|
||||
}
|
||||
|
||||
if (deleteDialogMode.value === "missing-file-tasks") {
|
||||
return "以分片为单位判定:仅当分片的视频文件在磁盘上不存在时才会删除。正在录制或处理中的分片会自动跳过,有视频文件的分片不受影响。";
|
||||
}
|
||||
|
||||
return "记录加文件会尝试删除视频文件和对应弹幕 XML。文件不存在时不会阻断删除,但会返回警告信息。";
|
||||
});
|
||||
function isActiveStatus(status: number) {
|
||||
@@ -572,6 +588,13 @@ function openDeleteEmptySessionsDialog() {
|
||||
deleteDialogVisible.value = true;
|
||||
}
|
||||
|
||||
function openDeleteMissingFileTasksDialog() {
|
||||
deleteDialogMode.value = "missing-file-tasks";
|
||||
deleteDialogSessionIds.value = [];
|
||||
deleteDialogTaskIds.value = [];
|
||||
deleteDialogVisible.value = true;
|
||||
}
|
||||
|
||||
async function confirmConditionalDelete() {
|
||||
conditionalDialogVisible.value = false;
|
||||
deleteDialogMode.value = "conditional-sessions";
|
||||
@@ -642,6 +665,7 @@ async function confirmDelete(deleteFiles: boolean) {
|
||||
const deletingSessions = currentMode === "sessions";
|
||||
const deletingConditionalSessions = currentMode === "conditional-sessions";
|
||||
const deletingEmptySessions = currentMode === "empty-sessions";
|
||||
const deletingMissingFileTasks = currentMode === "missing-file-tasks";
|
||||
const deletingMixed = currentMode === "mixed";
|
||||
const hasSelection = deletingMixed
|
||||
? deleteDialogSessionIds.value.length > 0 || deleteDialogTaskIds.value.length > 0
|
||||
@@ -649,7 +673,7 @@ async function confirmDelete(deleteFiles: boolean) {
|
||||
? deleteDialogSessionIds.value.length > 0
|
||||
: deleteDialogTaskIds.value.length > 0;
|
||||
|
||||
if (!deletingConditionalSessions && !deletingEmptySessions && !hasSelection) {
|
||||
if (!deletingConditionalSessions && !deletingEmptySessions && !deletingMissingFileTasks && !hasSelection) {
|
||||
closeDeleteDialog();
|
||||
return;
|
||||
}
|
||||
@@ -685,6 +709,11 @@ async function confirmDelete(deleteFiles: boolean) {
|
||||
deleteFiles
|
||||
});
|
||||
createdCleanupOperation = data;
|
||||
} else if (deletingMissingFileTasks) {
|
||||
const { data } = await apiClient.post<DeleteCompletedRecordTasksResult>("/record-tasks/delete-missing-files", {
|
||||
deleteFiles
|
||||
});
|
||||
deletedTaskResult = data;
|
||||
} else if (deletingMixed) {
|
||||
if (deleteDialogTaskIds.value.length > 0) {
|
||||
const { data } = await apiClient.post<DeleteCompletedRecordTasksResult>("/record-tasks/delete", {
|
||||
@@ -723,7 +752,17 @@ async function confirmDelete(deleteFiles: boolean) {
|
||||
deleteDialogVisible.value = false;
|
||||
resetDeleteDialogState();
|
||||
|
||||
if (deleteDialogMode.value === "tasks") {
|
||||
if (currentMode === "missing-file-tasks") {
|
||||
ElMessage.success(
|
||||
deletedTaskResult
|
||||
? `已清理 ${deletedTaskResult.deletedTaskIds.length} 个无文件分片。`
|
||||
: "未发现可清理的无文件分片。"
|
||||
);
|
||||
await loadSessions({ resetPanels: false, resetSelection: false });
|
||||
return;
|
||||
}
|
||||
|
||||
if (currentMode === "tasks") {
|
||||
ElMessage.success(
|
||||
deletedTaskResult ? `已删除 ${deletedTaskResult.deletedTaskIds.length} 个分片任务。` : "已删除分片任务。"
|
||||
);
|
||||
@@ -737,11 +776,7 @@ async function confirmDelete(deleteFiles: boolean) {
|
||||
return;
|
||||
}
|
||||
|
||||
ElMessage.success(
|
||||
currentMode === "tasks"
|
||||
? "已删除分片任务。"
|
||||
: "后台清理任务已创建,页面会自动轮询进度。"
|
||||
);
|
||||
ElMessage.success("后台清理任务已创建,页面会自动轮询进度。");
|
||||
} finally {
|
||||
deleting.value = false;
|
||||
}
|
||||
@@ -931,6 +966,9 @@ onBeforeUnmount(() => {
|
||||
<el-button plain :loading="deleting" @click="openDeleteEmptySessionsDialog">
|
||||
清理无分片会话
|
||||
</el-button>
|
||||
<el-button plain :loading="deleting" @click="openDeleteMissingFileTasksDialog">
|
||||
清理无文件分片
|
||||
</el-button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -108,6 +108,11 @@ public sealed class DeleteCompletedRecordTasksRequest
|
||||
public bool DeleteFiles { get; set; }
|
||||
}
|
||||
|
||||
public sealed class DeleteMissingFileRecordTasksRequest
|
||||
{
|
||||
public bool DeleteFiles { get; set; }
|
||||
}
|
||||
|
||||
public sealed class DeleteCompletedRecordTasksResultDto
|
||||
{
|
||||
public required IReadOnlyList<Guid> DeletedTaskIds { get; init; }
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
namespace LiveRecorder.Application.Models.Reports;
|
||||
|
||||
public sealed class HealthReadyResponse
|
||||
{
|
||||
public string Status { get; set; } = "ready";
|
||||
public DateTimeOffset Timestamp { get; set; }
|
||||
public DatabaseHealthStatus Database { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class DatabaseHealthStatus
|
||||
{
|
||||
public bool Reachable { get; set; }
|
||||
public string? Reason { get; set; }
|
||||
public int ConsecutiveFailures { get; set; }
|
||||
}
|
||||
@@ -504,6 +504,32 @@ public sealed class RecordService
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<DeleteCompletedRecordTasksResultDto> DeleteMissingFileTasksAsync(
|
||||
DeleteMissingFileRecordTasksRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
var tasks = await _recordTaskRepository.ListAsync(null, cancellationToken);
|
||||
var missingFileTaskIds = tasks
|
||||
.Where(static task => !IsActiveTaskStatus(task.Status) && !HasExistingVideoFile(task))
|
||||
.Select(static task => task.Id)
|
||||
.ToArray();
|
||||
|
||||
if (missingFileTaskIds.Length == 0)
|
||||
{
|
||||
return CreateEmptyDeleteResult();
|
||||
}
|
||||
|
||||
return await DeleteTasksAsync(
|
||||
new DeleteCompletedRecordTasksRequest
|
||||
{
|
||||
TaskIds = missingFileTaskIds,
|
||||
DeleteFiles = request.DeleteFiles
|
||||
},
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<RecordPreviewTicketDto> CreatePreviewTicketAsync(Guid id, string mediaBaseUrl, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var previewTicket = await _recordMediaService.CreatePreviewTicketAsync(id, cancellationToken);
|
||||
@@ -823,6 +849,24 @@ public sealed class RecordService
|
||||
or RecordTaskStatus.Stopping
|
||||
or RecordTaskStatus.Processing;
|
||||
|
||||
private static bool HasExistingVideoFile(RecordTask task)
|
||||
{
|
||||
var candidatePath = !string.IsNullOrWhiteSpace(task.Result?.FilePath)
|
||||
? task.Result!.FilePath
|
||||
: task.OutputFilePath;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(candidatePath))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var resolvedPath = Path.IsPathRooted(candidatePath)
|
||||
? candidatePath
|
||||
: Path.GetFullPath(candidatePath, AppContext.BaseDirectory);
|
||||
|
||||
return File.Exists(resolvedPath);
|
||||
}
|
||||
|
||||
private static bool IsActiveSessionStatus(RecordSessionStatus status) =>
|
||||
status is RecordSessionStatus.Starting or RecordSessionStatus.Running or RecordSessionStatus.Stopping;
|
||||
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Storage;
|
||||
using Npgsql.EntityFrameworkCore.PostgreSQL;
|
||||
|
||||
namespace LiveRecorder.Infrastructure.Persistence;
|
||||
|
||||
/// <summary>
|
||||
/// Custom EF Core execution strategy that integrates with <see cref="DatabaseCircuitBreaker"/>.
|
||||
///
|
||||
/// When the circuit is open, operations fail immediately without retrying.
|
||||
/// When a non-transient error occurs (e.g. disk_full), the operation does not retry.
|
||||
/// After each failure, the circuit is recorded, and after each success, the circuit is reset.
|
||||
/// </summary>
|
||||
public sealed class CircuitAwareExecutionStrategy : NpgsqlRetryingExecutionStrategy
|
||||
{
|
||||
private static readonly TimeSpan DefaultMaxRetryDelay = TimeSpan.FromSeconds(15);
|
||||
|
||||
public CircuitAwareExecutionStrategy(
|
||||
ExecutionStrategyDependencies dependencies,
|
||||
int maxRetryCount,
|
||||
TimeSpan maxRetryDelay)
|
||||
: base(dependencies, maxRetryCount, maxRetryDelay, errorCodesToAdd: null)
|
||||
{
|
||||
}
|
||||
|
||||
public CircuitAwareExecutionStrategy(
|
||||
ExecutionStrategyDependencies dependencies,
|
||||
int maxRetryCount,
|
||||
TimeSpan maxRetryDelay,
|
||||
ICollection<string>? errorCodesToAdd)
|
||||
: base(dependencies, maxRetryCount, maxRetryDelay, errorCodesToAdd)
|
||||
{
|
||||
}
|
||||
|
||||
protected override bool ShouldRetryOn(Exception? exception)
|
||||
{
|
||||
// Fast-fail: circuit is open
|
||||
if (DatabaseCircuitBreaker.IsOpen)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Fast-fail: non-transient errors (disk_full, out_of_memory, etc.)
|
||||
if (DatabaseCircuitBreaker.IsNonTransient(exception))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Delegate to default Npgsql retry logic for transient errors
|
||||
return base.ShouldRetryOn(exception);
|
||||
}
|
||||
|
||||
protected override void OnFirstExecution()
|
||||
{
|
||||
// If circuit is open, throw immediately before even attempting
|
||||
if (DatabaseCircuitBreaker.IsOpen)
|
||||
{
|
||||
throw new DatabaseCircuitOpenException(
|
||||
$"Database circuit breaker is open. Consecutive failures: {DatabaseCircuitBreaker.ConsecutiveFailures}. " +
|
||||
$"Circuit opened at: {DatabaseCircuitBreaker.OpenedAt:O}.");
|
||||
}
|
||||
|
||||
base.OnFirstExecution();
|
||||
}
|
||||
|
||||
public override TResult Execute<TState, TResult>(
|
||||
TState state,
|
||||
Func<DbContext, TState, TResult> operation,
|
||||
Func<DbContext, TState, ExecutionResult<TResult>>? verifySucceeded)
|
||||
{
|
||||
try
|
||||
{
|
||||
var result = base.Execute(state, operation, verifySucceeded);
|
||||
DatabaseCircuitBreaker.RecordSuccess();
|
||||
return result;
|
||||
}
|
||||
catch
|
||||
{
|
||||
DatabaseCircuitBreaker.RecordFailure();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public override async Task<TResult> ExecuteAsync<TState, TResult>(
|
||||
TState state,
|
||||
Func<DbContext, TState, CancellationToken, Task<TResult>> operation,
|
||||
Func<DbContext, TState, CancellationToken, Task<ExecutionResult<TResult>>>? verifySucceeded,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var result = await base.ExecuteAsync(state, operation, verifySucceeded, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
DatabaseCircuitBreaker.RecordSuccess();
|
||||
return result;
|
||||
}
|
||||
catch
|
||||
{
|
||||
DatabaseCircuitBreaker.RecordFailure();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace LiveRecorder.Infrastructure.Persistence;
|
||||
|
||||
/// <summary>
|
||||
/// Thread-safe static circuit breaker for database operations.
|
||||
/// When the database becomes unavailable (e.g. disk full), this prevents
|
||||
/// every API request from wasting 45 seconds on doomed retries.
|
||||
/// </summary>
|
||||
public static class DatabaseCircuitBreaker
|
||||
{
|
||||
private static readonly object Lock = new();
|
||||
|
||||
/// <summary>Consecutive failures before the circuit opens.</summary>
|
||||
private const int FailureThreshold = 5;
|
||||
|
||||
/// <summary>How long the circuit stays open before allowing a probe.</summary>
|
||||
private static readonly TimeSpan BreakDuration = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>Npgsql error codes that are NOT transient — retrying is futile.</summary>
|
||||
private static readonly HashSet<string> NonTransientCodes = new(StringComparer.Ordinal)
|
||||
{
|
||||
"53100", // disk_full
|
||||
"53200", // out_of_memory
|
||||
"53300", // too_many_connections
|
||||
"08006", // connection_failure (persistent)
|
||||
"57P03", // cannot_connect_now
|
||||
"42601", // syntax_error (bug, not transient)
|
||||
"42501", // insufficient_privilege
|
||||
"3D000", // invalid_catalog_name
|
||||
"28P01", // invalid_password
|
||||
};
|
||||
|
||||
private static int _consecutiveFailures;
|
||||
private static DateTimeOffset _openedAt = DateTimeOffset.MinValue;
|
||||
|
||||
/// <summary>Whether the circuit is currently open (failing fast).</summary>
|
||||
public static bool IsOpen
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_consecutiveFailures < FailureThreshold)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (DateTimeOffset.UtcNow - _openedAt > BreakDuration)
|
||||
{
|
||||
// Transition to half-open: allow one probe
|
||||
lock (Lock)
|
||||
{
|
||||
if (_consecutiveFailures >= FailureThreshold &&
|
||||
DateTimeOffset.UtcNow - _openedAt > BreakDuration)
|
||||
{
|
||||
// Reset to just below threshold so the next call probes
|
||||
_consecutiveFailures = FailureThreshold - 1;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public static int ConsecutiveFailures => Volatile.Read(ref _consecutiveFailures);
|
||||
|
||||
public static DateTimeOffset OpenedAt => _openedAt;
|
||||
|
||||
/// <summary>Record a successful database operation.</summary>
|
||||
public static void RecordSuccess()
|
||||
{
|
||||
lock (Lock)
|
||||
{
|
||||
_consecutiveFailures = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Record a failed database operation.</summary>
|
||||
public static void RecordFailure()
|
||||
{
|
||||
lock (Lock)
|
||||
{
|
||||
_consecutiveFailures++;
|
||||
if (_consecutiveFailures >= FailureThreshold)
|
||||
{
|
||||
_openedAt = DateTimeOffset.UtcNow;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check whether a given exception is a non-transient database error
|
||||
/// that should NOT be retried. Returns true if retrying would be futile.
|
||||
/// </summary>
|
||||
public static bool IsNonTransient(Exception? ex)
|
||||
{
|
||||
while (ex is not null)
|
||||
{
|
||||
if (ex is Npgsql.NpgsqlException npgEx && npgEx.SqlState is { Length: 5 } state)
|
||||
{
|
||||
return NonTransientCodes.Contains(state);
|
||||
}
|
||||
|
||||
ex = ex.InnerException;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>Log the current circuit state.</summary>
|
||||
public static void LogState(ILogger logger)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"DatabaseCircuitBreaker state: Open={IsOpen}, ConsecutiveFailures={Failures}, OpenedAt={OpenedAt}",
|
||||
IsOpen,
|
||||
ConsecutiveFailures,
|
||||
OpenedAt == DateTimeOffset.MinValue ? "never" : OpenedAt.ToString("O"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
namespace LiveRecorder.Infrastructure.Persistence;
|
||||
|
||||
/// <summary>
|
||||
/// Thrown when a database operation is rejected because the circuit breaker is open.
|
||||
/// This is a fast-fail — the request will not be retried.
|
||||
/// </summary>
|
||||
public sealed class DatabaseCircuitOpenException : InvalidOperationException
|
||||
{
|
||||
public DatabaseCircuitOpenException(string message) : base(message)
|
||||
{
|
||||
}
|
||||
|
||||
public DatabaseCircuitOpenException(string message, Exception innerException) : base(message, innerException)
|
||||
{
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
using LiveRecorder.Infrastructure.Persistence;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -7,6 +8,8 @@ namespace LiveRecorder.Infrastructure.Services;
|
||||
public sealed class CleanupOperationBackgroundService : BackgroundService
|
||||
{
|
||||
private static readonly TimeSpan IdleDelay = TimeSpan.FromSeconds(2);
|
||||
private static readonly TimeSpan ErrorBaseDelay = TimeSpan.FromSeconds(5);
|
||||
private static readonly TimeSpan ErrorMaxDelay = TimeSpan.FromMinutes(5);
|
||||
private readonly IServiceScopeFactory _serviceScopeFactory;
|
||||
private readonly ILogger<CleanupOperationBackgroundService> _logger;
|
||||
|
||||
@@ -20,6 +23,7 @@ public sealed class CleanupOperationBackgroundService : BackgroundService
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
// Startup: requeue interrupted operations
|
||||
try
|
||||
{
|
||||
using var startupScope = _serviceScopeFactory.CreateScope();
|
||||
@@ -31,17 +35,36 @@ public sealed class CleanupOperationBackgroundService : BackgroundService
|
||||
_logger.LogWarning(ex, "Failed to requeue interrupted cleanup operations at startup");
|
||||
}
|
||||
|
||||
var consecutiveErrors = 0;
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Skip processing if the circuit is open — don't waste resources
|
||||
if (DatabaseCircuitBreaker.IsOpen)
|
||||
{
|
||||
consecutiveErrors = await DelayWithBackoff(ErrorBaseDelay, ErrorMaxDelay, consecutiveErrors, stoppingToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
using var scope = _serviceScopeFactory.CreateScope();
|
||||
var coordinator = scope.ServiceProvider.GetRequiredService<CleanupOperationCoordinator>();
|
||||
var processed = await coordinator.ProcessNextQueuedOperationAsync(stoppingToken);
|
||||
|
||||
if (processed)
|
||||
{
|
||||
consecutiveErrors = 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
// No queued operations — idle delay
|
||||
consecutiveErrors = 0;
|
||||
await Task.Delay(IdleDelay, stoppingToken);
|
||||
}
|
||||
catch (DatabaseCircuitOpenException)
|
||||
{
|
||||
consecutiveErrors = await DelayWithBackoff(ErrorBaseDelay, ErrorMaxDelay, consecutiveErrors, stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
@@ -49,17 +72,47 @@ public sealed class CleanupOperationBackgroundService : BackgroundService
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Cleanup operation background worker failed");
|
||||
}
|
||||
var isDatabaseError = DatabaseCircuitBreaker.IsNonTransient(ex) ||
|
||||
ex is Npgsql.NpgsqlException ||
|
||||
ex is Microsoft.EntityFrameworkCore.DbUpdateException;
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(IdleDelay, stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
if (isDatabaseError)
|
||||
{
|
||||
DatabaseCircuitBreaker.RecordFailure();
|
||||
_logger.LogWarning(ex,
|
||||
"Cleanup background worker: database error (#{ErrorCount}). Circuit state: Open={IsOpen}, Failures={Failures}",
|
||||
consecutiveErrors + 1,
|
||||
DatabaseCircuitBreaker.IsOpen,
|
||||
DatabaseCircuitBreaker.ConsecutiveFailures);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogError(ex, "Cleanup operation background worker failed");
|
||||
}
|
||||
|
||||
consecutiveErrors = await DelayWithBackoff(ErrorBaseDelay, ErrorMaxDelay, consecutiveErrors, stoppingToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<int> DelayWithBackoff(
|
||||
TimeSpan baseDelay, TimeSpan maxDelay, int errorCount, CancellationToken cancellationToken)
|
||||
{
|
||||
errorCount++;
|
||||
// Exponential backoff: 5s, 10s, 20s, 40s, 80s, 160s, capping at 5min
|
||||
var factor = Math.Pow(2, Math.Min(errorCount - 1, 6));
|
||||
var delay = TimeSpan.FromMilliseconds(
|
||||
Math.Min(baseDelay.TotalMilliseconds * factor, maxDelay.TotalMilliseconds));
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(delay, cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Swallow — loop will exit on next iteration
|
||||
}
|
||||
|
||||
return errorCount;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -146,8 +146,20 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (DatabaseCircuitOpenException)
|
||||
{
|
||||
// Circuit is open — skip this iteration and wait
|
||||
_logger.LogDebug("Polling loop skipped: database circuit breaker is open");
|
||||
delay = TimeSpan.FromSeconds(30);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Record database failures to the circuit breaker
|
||||
if (IsTransientDatabaseException(ex) || DatabaseCircuitBreaker.IsNonTransient(ex))
|
||||
{
|
||||
DatabaseCircuitBreaker.RecordFailure();
|
||||
}
|
||||
|
||||
_logger.LogError(ex, "Background live room polling failed");
|
||||
|
||||
try
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using LiveRecorder.Infrastructure.Persistence;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -7,7 +8,8 @@ namespace LiveRecorder.Infrastructure.Services;
|
||||
public sealed class RetentionCleanupBackgroundService : BackgroundService
|
||||
{
|
||||
private static readonly TimeSpan CleanupInterval = TimeSpan.FromHours(24);
|
||||
|
||||
private static readonly TimeSpan ErrorBaseDelay = TimeSpan.FromSeconds(10);
|
||||
private static readonly TimeSpan ErrorMaxDelay = TimeSpan.FromMinutes(5);
|
||||
private readonly IServiceScopeFactory _serviceScopeFactory;
|
||||
private readonly ILogger<RetentionCleanupBackgroundService> _logger;
|
||||
|
||||
@@ -21,13 +23,31 @@ public sealed class RetentionCleanupBackgroundService : BackgroundService
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
var consecutiveErrors = 0;
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Skip if circuit is already open
|
||||
if (DatabaseCircuitBreaker.IsOpen)
|
||||
{
|
||||
_logger.LogDebug("Retention cleanup skipped: database circuit breaker is open");
|
||||
await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
using var scope = _serviceScopeFactory.CreateScope();
|
||||
var cleanupService = scope.ServiceProvider.GetRequiredService<RetentionCleanupService>();
|
||||
await cleanupService.TryEnqueueAsync(ignoreEnabledSetting: false, cancellationToken: stoppingToken);
|
||||
|
||||
consecutiveErrors = 0;
|
||||
}
|
||||
catch (DatabaseCircuitOpenException)
|
||||
{
|
||||
// Silent — circuit is already logged
|
||||
await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken);
|
||||
continue;
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
@@ -35,7 +55,38 @@ public sealed class RetentionCleanupBackgroundService : BackgroundService
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Retention cleanup background task failed");
|
||||
var isDatabaseError = DatabaseCircuitBreaker.IsNonTransient(ex) ||
|
||||
ex is Npgsql.NpgsqlException ||
|
||||
ex is Microsoft.EntityFrameworkCore.DbUpdateException;
|
||||
|
||||
consecutiveErrors++;
|
||||
var delay = TimeSpan.FromMilliseconds(
|
||||
Math.Min(ErrorBaseDelay.TotalMilliseconds * Math.Pow(2, Math.Min(consecutiveErrors - 1, 6)),
|
||||
ErrorMaxDelay.TotalMilliseconds));
|
||||
|
||||
if (isDatabaseError)
|
||||
{
|
||||
DatabaseCircuitBreaker.RecordFailure();
|
||||
_logger.LogWarning(ex,
|
||||
"Retention cleanup: database error (#{ErrorCount}). Circuit: Open={IsOpen}, Failures={Failures}",
|
||||
consecutiveErrors,
|
||||
DatabaseCircuitBreaker.IsOpen,
|
||||
DatabaseCircuitBreaker.ConsecutiveFailures);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogError(ex, "Retention cleanup background task failed");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(delay, stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
|
||||
@@ -61,6 +61,12 @@ public sealed class RecordTasksController : ControllerBase
|
||||
CancellationToken cancellationToken) =>
|
||||
Ok(await _recordService.DeleteTasksAsync(request, cancellationToken));
|
||||
|
||||
[HttpPost("delete-missing-files")]
|
||||
public async Task<ActionResult<DeleteCompletedRecordTasksResultDto>> DeleteMissingFiles(
|
||||
[FromBody] DeleteMissingFileRecordTasksRequest request,
|
||||
CancellationToken cancellationToken) =>
|
||||
Ok(await _recordService.DeleteMissingFileTasksAsync(request, cancellationToken));
|
||||
|
||||
[HttpPost("{id:guid}/preview-ticket")]
|
||||
public async Task<ActionResult<RecordPreviewTicketDto>> CreatePreviewTicket(Guid id, CancellationToken cancellationToken)
|
||||
{
|
||||
|
||||
@@ -4,16 +4,15 @@ ARG DOTNET_RUNTIME_IMAGE=mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim
|
||||
FROM ${DOTNET_SDK_IMAGE} AS build
|
||||
WORKDIR /src
|
||||
|
||||
COPY ["LiveRecorder.sln", "./"]
|
||||
COPY ["src/LiveRecorder.Domain/LiveRecorder.Domain.csproj", "src/LiveRecorder.Domain/"]
|
||||
COPY ["src/LiveRecorder.Application/LiveRecorder.Application.csproj", "src/LiveRecorder.Application/"]
|
||||
COPY ["src/LiveRecorder.Infrastructure/LiveRecorder.Infrastructure.csproj", "src/LiveRecorder.Infrastructure/"]
|
||||
COPY ["src/LiveRecorder.WebApi/LiveRecorder.WebApi.csproj", "src/LiveRecorder.WebApi/"]
|
||||
|
||||
RUN dotnet restore "src/LiveRecorder.WebApi/LiveRecorder.WebApi.csproj"
|
||||
|
||||
COPY . .
|
||||
RUN dotnet publish "src/LiveRecorder.WebApi/LiveRecorder.WebApi.csproj" -c Release -o /app/publish /p:UseAppHost=false
|
||||
|
||||
# Workaround for QEMU ARM64 emulation
|
||||
ENV DOTNET_EnableWriteXorExecute=0
|
||||
ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1
|
||||
ENV DOTNET_GCConserveMemory=9
|
||||
|
||||
RUN dotnet restore "src/LiveRecorder.WebApi/LiveRecorder.WebApi.csproj" -p:RestoreUseStaticGraphEvaluation=true
|
||||
RUN dotnet publish "src/LiveRecorder.WebApi/LiveRecorder.WebApi.csproj" -c Release -o /app/publish /p:UseAppHost=false /p:DebugType=None /p:DebugSymbols=false /maxcpucount:1
|
||||
|
||||
FROM ${DOTNET_RUNTIME_IMAGE} AS runtime
|
||||
|
||||
|
||||
@@ -17,6 +17,8 @@
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<RuntimeIdentifier>linux-arm64</RuntimeIdentifier>
|
||||
<SelfContained>false</SelfContained>
|
||||
</PropertyGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
using LiveRecorder.Infrastructure.Persistence;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Npgsql;
|
||||
|
||||
namespace LiveRecorder.WebApi.Middleware;
|
||||
|
||||
public sealed class ExceptionHandlingMiddleware
|
||||
@@ -17,22 +21,80 @@ public sealed class ExceptionHandlingMiddleware
|
||||
{
|
||||
await _next(context);
|
||||
}
|
||||
catch (DatabaseCircuitOpenException ex)
|
||||
{
|
||||
// Circuit is open — fast-fail with 503
|
||||
_logger.LogWarning(ex, "Database circuit breaker is open, returning 503");
|
||||
context.Response.StatusCode = StatusCodes.Status503ServiceUnavailable;
|
||||
await context.Response.WriteAsJsonAsync(new
|
||||
{
|
||||
message = "Database temporarily unavailable",
|
||||
detail = "The database is currently unreachable. Requests will be accepted again once connectivity is restored.",
|
||||
retryAfterSeconds = 30
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Unhandled exception");
|
||||
context.Response.StatusCode = ex switch
|
||||
|
||||
// Detect database-related exceptions and return 503 instead of 500
|
||||
var statusCode = ex switch
|
||||
{
|
||||
DatabaseCircuitOpenException => StatusCodes.Status503ServiceUnavailable,
|
||||
KeyNotFoundException => StatusCodes.Status404NotFound,
|
||||
InvalidOperationException => StatusCodes.Status400BadRequest,
|
||||
NotSupportedException => StatusCodes.Status400BadRequest,
|
||||
_ when IsDatabaseException(ex) => StatusCodes.Status503ServiceUnavailable,
|
||||
_ => StatusCodes.Status500InternalServerError
|
||||
};
|
||||
|
||||
context.Response.StatusCode = statusCode;
|
||||
|
||||
await context.Response.WriteAsJsonAsync(new
|
||||
{
|
||||
message = ex.Message,
|
||||
detail = context.Response.StatusCode == StatusCodes.Status500InternalServerError ? "Internal Server Error" : null
|
||||
message = statusCode == StatusCodes.Status503ServiceUnavailable
|
||||
? "Database temporarily unavailable"
|
||||
: ex.Message,
|
||||
detail = statusCode switch
|
||||
{
|
||||
StatusCodes.Status503ServiceUnavailable => "The database is currently unreachable. Please retry later.",
|
||||
StatusCodes.Status500InternalServerError => "Internal Server Error",
|
||||
_ => null
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if an exception (or any of its inner exceptions) is a database-related error
|
||||
/// that should be surfaced as 503 Service Unavailable.
|
||||
/// </summary>
|
||||
private static bool IsDatabaseException(Exception ex)
|
||||
{
|
||||
var current = ex;
|
||||
while (current is not null)
|
||||
{
|
||||
if (current is NpgsqlException or DbUpdateException)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (current is InvalidOperationException ioEx &&
|
||||
(ioEx.Message.Contains("database", StringComparison.OrdinalIgnoreCase) ||
|
||||
ioEx.Message.Contains("connection", StringComparison.OrdinalIgnoreCase) ||
|
||||
ioEx.Message.Contains("Npgsql", StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (current is TimeoutException && current.Message.Contains("database", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
current = current.InnerException;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ using LiveRecorder.Application.Abstractions.Recording;
|
||||
using LiveRecorder.Application.Abstractions.Scripting;
|
||||
using LiveRecorder.Application.Abstractions.Settings;
|
||||
using LiveRecorder.Application.Abstractions.Storage;
|
||||
using LiveRecorder.Application.Models.Reports;
|
||||
using LiveRecorder.Application.Services;
|
||||
using LiveRecorder.Infrastructure.Persistence;
|
||||
using LiveRecorder.Infrastructure.Persistence.Repositories;
|
||||
@@ -31,6 +32,7 @@ using LiveRecorder.Infrastructure.Platforms.YouTube;
|
||||
using LiveRecorder.Infrastructure.Services;
|
||||
using LiveRecorder.WebApi.Middleware;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Storage;
|
||||
using Microsoft.OpenApi.Models;
|
||||
using Npgsql;
|
||||
|
||||
@@ -149,6 +151,11 @@ builder.Services.AddDbContext<LiveRecorderDbContext>(options =>
|
||||
maxRetryCount: 3,
|
||||
maxRetryDelay: TimeSpan.FromSeconds(15),
|
||||
errorCodesToAdd: null);
|
||||
npgsql.ExecutionStrategy(dependencies =>
|
||||
new CircuitAwareExecutionStrategy(
|
||||
dependencies,
|
||||
maxRetryCount: 3,
|
||||
maxRetryDelay: TimeSpan.FromSeconds(15)));
|
||||
}));
|
||||
|
||||
builder.Services.AddScoped<IUnitOfWork>(provider => provider.GetRequiredService<LiveRecorderDbContext>());
|
||||
@@ -231,6 +238,61 @@ app.UseMiddleware<ApiTokenAuthenticationMiddleware>();
|
||||
app.MapGet("/", () => Results.Redirect("/swagger"));
|
||||
app.MapControllers();
|
||||
|
||||
// ── Health check endpoints ────────────────────────────────────────────
|
||||
// /health — liveness: is the process alive and responding?
|
||||
app.MapGet("/health", () => Results.Ok(new { status = "healthy", timestamp = DateTimeOffset.UtcNow }));
|
||||
|
||||
// /health/ready — readiness: can we reach the database?
|
||||
app.MapGet("/health/ready", async (CancellationToken cancellationToken) =>
|
||||
{
|
||||
var readyResult = new HealthReadyResponse
|
||||
{
|
||||
Status = "ready",
|
||||
Timestamp = DateTimeOffset.UtcNow,
|
||||
Database = new DatabaseHealthStatus()
|
||||
};
|
||||
|
||||
if (DatabaseCircuitBreaker.IsOpen)
|
||||
{
|
||||
readyResult.Status = "degraded";
|
||||
readyResult.Database = new DatabaseHealthStatus
|
||||
{
|
||||
Reachable = false,
|
||||
Reason = "Circuit breaker is open",
|
||||
ConsecutiveFailures = DatabaseCircuitBreaker.ConsecutiveFailures
|
||||
};
|
||||
return Results.Json(readyResult, statusCode: StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// Lightweight probe: just test connectivity with a 2-second timeout
|
||||
await using var connection = new NpgsqlConnection(connectionStringBuilder.ConnectionString);
|
||||
var probeCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
probeCts.CancelAfter(TimeSpan.FromSeconds(2));
|
||||
|
||||
await connection.OpenAsync(probeCts.Token);
|
||||
await using var cmd = connection.CreateCommand();
|
||||
cmd.CommandText = "SELECT 1";
|
||||
await cmd.ExecuteScalarAsync(probeCts.Token);
|
||||
|
||||
readyResult.Database = new DatabaseHealthStatus { Reachable = true };
|
||||
return Results.Ok(readyResult);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
DatabaseCircuitBreaker.RecordFailure();
|
||||
readyResult.Status = "unhealthy";
|
||||
readyResult.Database = new DatabaseHealthStatus
|
||||
{
|
||||
Reachable = false,
|
||||
Reason = ex.Message,
|
||||
ConsecutiveFailures = DatabaseCircuitBreaker.ConsecutiveFailures
|
||||
};
|
||||
return Results.Json(readyResult, statusCode: StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
});
|
||||
|
||||
using (var scope = app.Services.CreateScope())
|
||||
{
|
||||
var initializer = scope.ServiceProvider.GetRequiredService<DatabaseInitializer>();
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
using System.Reflection;
|
||||
using LiveRecorder.Infrastructure.Persistence;
|
||||
using Npgsql;
|
||||
|
||||
namespace LiveRecorder.Tests;
|
||||
|
||||
public sealed class DatabaseCircuitBreakerTests
|
||||
{
|
||||
// Mirrors the private FailureThreshold in DatabaseCircuitBreaker so the assertions
|
||||
// read intentionally. Keep in sync if the production threshold changes.
|
||||
private const int FailureThreshold = 5;
|
||||
|
||||
public DatabaseCircuitBreakerTests()
|
||||
{
|
||||
// The breaker is a process-wide static, so reset it to a known closed state
|
||||
// before each test to keep cases independent.
|
||||
DatabaseCircuitBreaker.RecordSuccess();
|
||||
SetOpenedAt(DateTimeOffset.MinValue);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordFailure_below_threshold_keeps_circuit_closed()
|
||||
{
|
||||
for (var i = 0; i < FailureThreshold - 1; i++)
|
||||
{
|
||||
DatabaseCircuitBreaker.RecordFailure();
|
||||
}
|
||||
|
||||
Assert.False(DatabaseCircuitBreaker.IsOpen);
|
||||
Assert.Equal(FailureThreshold - 1, DatabaseCircuitBreaker.ConsecutiveFailures);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordFailure_at_threshold_opens_circuit()
|
||||
{
|
||||
for (var i = 0; i < FailureThreshold; i++)
|
||||
{
|
||||
DatabaseCircuitBreaker.RecordFailure();
|
||||
}
|
||||
|
||||
Assert.True(DatabaseCircuitBreaker.IsOpen);
|
||||
Assert.Equal(FailureThreshold, DatabaseCircuitBreaker.ConsecutiveFailures);
|
||||
Assert.NotEqual(DateTimeOffset.MinValue, DatabaseCircuitBreaker.OpenedAt);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordSuccess_closes_circuit_and_resets_failures()
|
||||
{
|
||||
for (var i = 0; i < FailureThreshold; i++)
|
||||
{
|
||||
DatabaseCircuitBreaker.RecordFailure();
|
||||
}
|
||||
|
||||
Assert.True(DatabaseCircuitBreaker.IsOpen);
|
||||
|
||||
DatabaseCircuitBreaker.RecordSuccess();
|
||||
|
||||
Assert.False(DatabaseCircuitBreaker.IsOpen);
|
||||
Assert.Equal(0, DatabaseCircuitBreaker.ConsecutiveFailures);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IsOpen_after_break_duration_transitions_to_half_open()
|
||||
{
|
||||
for (var i = 0; i < FailureThreshold; i++)
|
||||
{
|
||||
DatabaseCircuitBreaker.RecordFailure();
|
||||
}
|
||||
|
||||
Assert.True(DatabaseCircuitBreaker.IsOpen);
|
||||
|
||||
// The breaker reads DateTimeOffset.UtcNow directly (no injectable clock), so move
|
||||
// the recorded open time past the 30s break window to simulate it elapsing.
|
||||
SetOpenedAt(DateTimeOffset.UtcNow - TimeSpan.FromSeconds(31));
|
||||
|
||||
// The first read past the window half-opens: it permits one probe and drops the
|
||||
// failure count to one below the threshold.
|
||||
Assert.False(DatabaseCircuitBreaker.IsOpen);
|
||||
Assert.Equal(FailureThreshold - 1, DatabaseCircuitBreaker.ConsecutiveFailures);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("53100")] // disk_full
|
||||
[InlineData("53300")] // too_many_connections
|
||||
[InlineData("28P01")] // invalid_password
|
||||
public void IsNonTransient_returns_true_for_known_fatal_sql_states(string sqlState)
|
||||
{
|
||||
var exception = new PostgresException("fatal", "FATAL", "FATAL", sqlState);
|
||||
|
||||
Assert.True(DatabaseCircuitBreaker.IsNonTransient(exception));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IsNonTransient_returns_false_for_transient_sql_state()
|
||||
{
|
||||
// 40001 = serialization_failure, which is retryable and not in the fatal set.
|
||||
var exception = new PostgresException("retry me", "ERROR", "ERROR", "40001");
|
||||
|
||||
Assert.False(DatabaseCircuitBreaker.IsNonTransient(exception));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IsNonTransient_unwraps_inner_exceptions()
|
||||
{
|
||||
var inner = new PostgresException("disk full", "FATAL", "FATAL", "53100");
|
||||
var wrapper = new InvalidOperationException("save failed", inner);
|
||||
|
||||
Assert.True(DatabaseCircuitBreaker.IsNonTransient(wrapper));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IsNonTransient_returns_false_for_null_and_non_postgres_exceptions()
|
||||
{
|
||||
Assert.False(DatabaseCircuitBreaker.IsNonTransient(null));
|
||||
Assert.False(DatabaseCircuitBreaker.IsNonTransient(new InvalidOperationException("boom")));
|
||||
}
|
||||
|
||||
private static void SetOpenedAt(DateTimeOffset value) =>
|
||||
typeof(DatabaseCircuitBreaker)
|
||||
.GetField("_openedAt", BindingFlags.NonPublic | BindingFlags.Static)!
|
||||
.SetValue(null, value);
|
||||
}
|
||||
@@ -20,6 +20,7 @@
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\LiveRecorder.Application\LiveRecorder.Application.csproj" />
|
||||
<ProjectReference Include="..\..\src\LiveRecorder.Domain\LiveRecorder.Domain.csproj" />
|
||||
<ProjectReference Include="..\..\src\LiveRecorder.Infrastructure\LiveRecorder.Infrastructure.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
Reference in New Issue
Block a user