Compare commits
4
Commits
e640d5b5cc
...
a062bf84bf
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a062bf84bf | ||
|
|
df70b64956 | ||
|
|
5196ffa0f0 | ||
|
|
6da0690fd3 |
@@ -32,7 +32,7 @@ const router = useRouter();
|
|||||||
const { isMobile } = useViewport();
|
const { isMobile } = useViewport();
|
||||||
|
|
||||||
// Danmaku replay dialog
|
// Danmaku replay dialog
|
||||||
const { danmakuEvents, loading: danmakuLoading, error: danmakuError, loadTaskDanmaku, clear: clearDanmaku } = useDanmakuPlayer();
|
const { danmakuEvents: replayDanmakuEvents, loading: danmakuLoading, error: danmakuError, loadTaskDanmaku, clear: clearDanmaku } = useDanmakuPlayer();
|
||||||
const danmakuDialogVisible = ref(false);
|
const danmakuDialogVisible = ref(false);
|
||||||
const danmakuDialogTitle = ref("");
|
const danmakuDialogTitle = ref("");
|
||||||
const danmakuPreviewUrl = ref("");
|
const danmakuPreviewUrl = ref("");
|
||||||
@@ -611,7 +611,7 @@ onMounted(loadDetail);
|
|||||||
<DanmakuPlayer
|
<DanmakuPlayer
|
||||||
v-else-if="danmakuPreviewUrl"
|
v-else-if="danmakuPreviewUrl"
|
||||||
:video-src="danmakuPreviewUrl"
|
:video-src="danmakuPreviewUrl"
|
||||||
:danmaku-events="danmakuEvents"
|
:danmaku-events="replayDanmakuEvents"
|
||||||
/>
|
/>
|
||||||
<div v-else class="preview-empty">无法加载该分片的预览。</div>
|
<div v-else class="preview-empty">无法加载该分片的预览。</div>
|
||||||
</el-dialog>
|
</el-dialog>
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ const uploadingTaskId = ref<string | null>(null);
|
|||||||
const triggeringSegmentCompletedTaskId = ref<string | null>(null);
|
const triggeringSegmentCompletedTaskId = ref<string | null>(null);
|
||||||
const cleanupOperation = ref<CleanupOperation | null>(null);
|
const cleanupOperation = ref<CleanupOperation | null>(null);
|
||||||
const deleteDialogVisible = ref(false);
|
const deleteDialogVisible = ref(false);
|
||||||
const deleteDialogMode = ref<"tasks" | "sessions" | "conditional-sessions" | "mixed" | "empty-sessions">("tasks");
|
const deleteDialogMode = ref<"tasks" | "sessions" | "conditional-sessions" | "mixed" | "empty-sessions" | "missing-file-tasks">("tasks");
|
||||||
const deleteDialogTaskIds = ref<string[]>([]);
|
const deleteDialogTaskIds = ref<string[]>([]);
|
||||||
const deleteDialogSessionIds = ref<string[]>([]);
|
const deleteDialogSessionIds = ref<string[]>([]);
|
||||||
|
|
||||||
@@ -174,6 +174,10 @@ const deleteDialogEyebrow = computed(() => {
|
|||||||
return "空闲会话清理";
|
return "空闲会话清理";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (deleteDialogMode.value === "missing-file-tasks") {
|
||||||
|
return "无文件分片清理";
|
||||||
|
}
|
||||||
|
|
||||||
return "删除确认";
|
return "删除确认";
|
||||||
});
|
});
|
||||||
const deleteDialogTitle = computed(() => {
|
const deleteDialogTitle = computed(() => {
|
||||||
@@ -193,6 +197,10 @@ const deleteDialogTitle = computed(() => {
|
|||||||
return "清理无分片会话";
|
return "清理无分片会话";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (deleteDialogMode.value === "missing-file-tasks") {
|
||||||
|
return "清理无文件分片";
|
||||||
|
}
|
||||||
|
|
||||||
return "删除分片任务";
|
return "删除分片任务";
|
||||||
});
|
});
|
||||||
const deleteDialogLead = computed(() => {
|
const deleteDialogLead = computed(() => {
|
||||||
@@ -212,6 +220,10 @@ const deleteDialogLead = computed(() => {
|
|||||||
return `将自动找出所有没有任何分片任务的录制会话并批量删除。你也可以选择同时删除可能残留的本地文件。`;
|
return `将自动找出所有没有任何分片任务的录制会话并批量删除。你也可以选择同时删除可能残留的本地文件。`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (deleteDialogMode.value === "missing-file-tasks") {
|
||||||
|
return `将自动找出所有视频文件已丢失的分片任务并批量删除(不限会话,只删命中的分片本身)。删除后若某个会话下不再有任何分片,会话也会一并清理;你也可以选择同时清理残留的弹幕 XML 文件。`;
|
||||||
|
}
|
||||||
|
|
||||||
return `将删除 ${deleteDialogTaskCount.value} 个已选择分片任务。你可以只移除数据库记录,也可以同时清理本地视频和弹幕 XML 文件。`;
|
return `将删除 ${deleteDialogTaskCount.value} 个已选择分片任务。你可以只移除数据库记录,也可以同时清理本地视频和弹幕 XML 文件。`;
|
||||||
});
|
});
|
||||||
const deleteDialogNote = computed(() => {
|
const deleteDialogNote = computed(() => {
|
||||||
@@ -227,6 +239,10 @@ const deleteDialogNote = computed(() => {
|
|||||||
return "仅清理没有任何关联分片的空会话,不影响有录制产物的会话。";
|
return "仅清理没有任何关联分片的空会话,不影响有录制产物的会话。";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (deleteDialogMode.value === "missing-file-tasks") {
|
||||||
|
return "以分片为单位判定:仅当分片的视频文件在磁盘上不存在时才会删除。正在录制或处理中的分片会自动跳过,有视频文件的分片不受影响。";
|
||||||
|
}
|
||||||
|
|
||||||
return "记录加文件会尝试删除视频文件和对应弹幕 XML。文件不存在时不会阻断删除,但会返回警告信息。";
|
return "记录加文件会尝试删除视频文件和对应弹幕 XML。文件不存在时不会阻断删除,但会返回警告信息。";
|
||||||
});
|
});
|
||||||
function isActiveStatus(status: number) {
|
function isActiveStatus(status: number) {
|
||||||
@@ -572,6 +588,13 @@ function openDeleteEmptySessionsDialog() {
|
|||||||
deleteDialogVisible.value = true;
|
deleteDialogVisible.value = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function openDeleteMissingFileTasksDialog() {
|
||||||
|
deleteDialogMode.value = "missing-file-tasks";
|
||||||
|
deleteDialogSessionIds.value = [];
|
||||||
|
deleteDialogTaskIds.value = [];
|
||||||
|
deleteDialogVisible.value = true;
|
||||||
|
}
|
||||||
|
|
||||||
async function confirmConditionalDelete() {
|
async function confirmConditionalDelete() {
|
||||||
conditionalDialogVisible.value = false;
|
conditionalDialogVisible.value = false;
|
||||||
deleteDialogMode.value = "conditional-sessions";
|
deleteDialogMode.value = "conditional-sessions";
|
||||||
@@ -642,6 +665,7 @@ async function confirmDelete(deleteFiles: boolean) {
|
|||||||
const deletingSessions = currentMode === "sessions";
|
const deletingSessions = currentMode === "sessions";
|
||||||
const deletingConditionalSessions = currentMode === "conditional-sessions";
|
const deletingConditionalSessions = currentMode === "conditional-sessions";
|
||||||
const deletingEmptySessions = currentMode === "empty-sessions";
|
const deletingEmptySessions = currentMode === "empty-sessions";
|
||||||
|
const deletingMissingFileTasks = currentMode === "missing-file-tasks";
|
||||||
const deletingMixed = currentMode === "mixed";
|
const deletingMixed = currentMode === "mixed";
|
||||||
const hasSelection = deletingMixed
|
const hasSelection = deletingMixed
|
||||||
? deleteDialogSessionIds.value.length > 0 || deleteDialogTaskIds.value.length > 0
|
? deleteDialogSessionIds.value.length > 0 || deleteDialogTaskIds.value.length > 0
|
||||||
@@ -649,7 +673,7 @@ async function confirmDelete(deleteFiles: boolean) {
|
|||||||
? deleteDialogSessionIds.value.length > 0
|
? deleteDialogSessionIds.value.length > 0
|
||||||
: deleteDialogTaskIds.value.length > 0;
|
: deleteDialogTaskIds.value.length > 0;
|
||||||
|
|
||||||
if (!deletingConditionalSessions && !deletingEmptySessions && !hasSelection) {
|
if (!deletingConditionalSessions && !deletingEmptySessions && !deletingMissingFileTasks && !hasSelection) {
|
||||||
closeDeleteDialog();
|
closeDeleteDialog();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -685,6 +709,11 @@ async function confirmDelete(deleteFiles: boolean) {
|
|||||||
deleteFiles
|
deleteFiles
|
||||||
});
|
});
|
||||||
createdCleanupOperation = data;
|
createdCleanupOperation = data;
|
||||||
|
} else if (deletingMissingFileTasks) {
|
||||||
|
const { data } = await apiClient.post<DeleteCompletedRecordTasksResult>("/record-tasks/delete-missing-files", {
|
||||||
|
deleteFiles
|
||||||
|
});
|
||||||
|
deletedTaskResult = data;
|
||||||
} else if (deletingMixed) {
|
} else if (deletingMixed) {
|
||||||
if (deleteDialogTaskIds.value.length > 0) {
|
if (deleteDialogTaskIds.value.length > 0) {
|
||||||
const { data } = await apiClient.post<DeleteCompletedRecordTasksResult>("/record-tasks/delete", {
|
const { data } = await apiClient.post<DeleteCompletedRecordTasksResult>("/record-tasks/delete", {
|
||||||
@@ -723,7 +752,17 @@ async function confirmDelete(deleteFiles: boolean) {
|
|||||||
deleteDialogVisible.value = false;
|
deleteDialogVisible.value = false;
|
||||||
resetDeleteDialogState();
|
resetDeleteDialogState();
|
||||||
|
|
||||||
if (deleteDialogMode.value === "tasks") {
|
if (currentMode === "missing-file-tasks") {
|
||||||
|
ElMessage.success(
|
||||||
|
deletedTaskResult
|
||||||
|
? `已清理 ${deletedTaskResult.deletedTaskIds.length} 个无文件分片。`
|
||||||
|
: "未发现可清理的无文件分片。"
|
||||||
|
);
|
||||||
|
await loadSessions({ resetPanels: false, resetSelection: false });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (currentMode === "tasks") {
|
||||||
ElMessage.success(
|
ElMessage.success(
|
||||||
deletedTaskResult ? `已删除 ${deletedTaskResult.deletedTaskIds.length} 个分片任务。` : "已删除分片任务。"
|
deletedTaskResult ? `已删除 ${deletedTaskResult.deletedTaskIds.length} 个分片任务。` : "已删除分片任务。"
|
||||||
);
|
);
|
||||||
@@ -737,11 +776,7 @@ async function confirmDelete(deleteFiles: boolean) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
ElMessage.success(
|
ElMessage.success("后台清理任务已创建,页面会自动轮询进度。");
|
||||||
currentMode === "tasks"
|
|
||||||
? "已删除分片任务。"
|
|
||||||
: "后台清理任务已创建,页面会自动轮询进度。"
|
|
||||||
);
|
|
||||||
} finally {
|
} finally {
|
||||||
deleting.value = false;
|
deleting.value = false;
|
||||||
}
|
}
|
||||||
@@ -931,6 +966,9 @@ onBeforeUnmount(() => {
|
|||||||
<el-button plain :loading="deleting" @click="openDeleteEmptySessionsDialog">
|
<el-button plain :loading="deleting" @click="openDeleteEmptySessionsDialog">
|
||||||
清理无分片会话
|
清理无分片会话
|
||||||
</el-button>
|
</el-button>
|
||||||
|
<el-button plain :loading="deleting" @click="openDeleteMissingFileTasksDialog">
|
||||||
|
清理无文件分片
|
||||||
|
</el-button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|||||||
@@ -108,6 +108,11 @@ public sealed class DeleteCompletedRecordTasksRequest
|
|||||||
public bool DeleteFiles { get; set; }
|
public bool DeleteFiles { get; set; }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public sealed class DeleteMissingFileRecordTasksRequest
|
||||||
|
{
|
||||||
|
public bool DeleteFiles { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
public sealed class DeleteCompletedRecordTasksResultDto
|
public sealed class DeleteCompletedRecordTasksResultDto
|
||||||
{
|
{
|
||||||
public required IReadOnlyList<Guid> DeletedTaskIds { get; init; }
|
public required IReadOnlyList<Guid> DeletedTaskIds { get; init; }
|
||||||
|
|||||||
@@ -0,0 +1,15 @@
|
|||||||
|
namespace LiveRecorder.Application.Models.Reports;
|
||||||
|
|
||||||
|
public sealed class HealthReadyResponse
|
||||||
|
{
|
||||||
|
public string Status { get; set; } = "ready";
|
||||||
|
public DateTimeOffset Timestamp { get; set; }
|
||||||
|
public DatabaseHealthStatus Database { get; set; } = new();
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class DatabaseHealthStatus
|
||||||
|
{
|
||||||
|
public bool Reachable { get; set; }
|
||||||
|
public string? Reason { get; set; }
|
||||||
|
public int ConsecutiveFailures { get; set; }
|
||||||
|
}
|
||||||
@@ -504,6 +504,32 @@ public sealed class RecordService
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async Task<DeleteCompletedRecordTasksResultDto> DeleteMissingFileTasksAsync(
|
||||||
|
DeleteMissingFileRecordTasksRequest request,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
ArgumentNullException.ThrowIfNull(request);
|
||||||
|
|
||||||
|
var tasks = await _recordTaskRepository.ListAsync(null, cancellationToken);
|
||||||
|
var missingFileTaskIds = tasks
|
||||||
|
.Where(static task => !IsActiveTaskStatus(task.Status) && !HasExistingVideoFile(task))
|
||||||
|
.Select(static task => task.Id)
|
||||||
|
.ToArray();
|
||||||
|
|
||||||
|
if (missingFileTaskIds.Length == 0)
|
||||||
|
{
|
||||||
|
return CreateEmptyDeleteResult();
|
||||||
|
}
|
||||||
|
|
||||||
|
return await DeleteTasksAsync(
|
||||||
|
new DeleteCompletedRecordTasksRequest
|
||||||
|
{
|
||||||
|
TaskIds = missingFileTaskIds,
|
||||||
|
DeleteFiles = request.DeleteFiles
|
||||||
|
},
|
||||||
|
cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
public async Task<RecordPreviewTicketDto> CreatePreviewTicketAsync(Guid id, string mediaBaseUrl, CancellationToken cancellationToken = default)
|
public async Task<RecordPreviewTicketDto> CreatePreviewTicketAsync(Guid id, string mediaBaseUrl, CancellationToken cancellationToken = default)
|
||||||
{
|
{
|
||||||
var previewTicket = await _recordMediaService.CreatePreviewTicketAsync(id, cancellationToken);
|
var previewTicket = await _recordMediaService.CreatePreviewTicketAsync(id, cancellationToken);
|
||||||
@@ -823,6 +849,24 @@ public sealed class RecordService
|
|||||||
or RecordTaskStatus.Stopping
|
or RecordTaskStatus.Stopping
|
||||||
or RecordTaskStatus.Processing;
|
or RecordTaskStatus.Processing;
|
||||||
|
|
||||||
|
private static bool HasExistingVideoFile(RecordTask task)
|
||||||
|
{
|
||||||
|
var candidatePath = !string.IsNullOrWhiteSpace(task.Result?.FilePath)
|
||||||
|
? task.Result!.FilePath
|
||||||
|
: task.OutputFilePath;
|
||||||
|
|
||||||
|
if (string.IsNullOrWhiteSpace(candidatePath))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
var resolvedPath = Path.IsPathRooted(candidatePath)
|
||||||
|
? candidatePath
|
||||||
|
: Path.GetFullPath(candidatePath, AppContext.BaseDirectory);
|
||||||
|
|
||||||
|
return File.Exists(resolvedPath);
|
||||||
|
}
|
||||||
|
|
||||||
private static bool IsActiveSessionStatus(RecordSessionStatus status) =>
|
private static bool IsActiveSessionStatus(RecordSessionStatus status) =>
|
||||||
status is RecordSessionStatus.Starting or RecordSessionStatus.Running or RecordSessionStatus.Stopping;
|
status is RecordSessionStatus.Starting or RecordSessionStatus.Running or RecordSessionStatus.Stopping;
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,103 @@
|
|||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.EntityFrameworkCore.Storage;
|
||||||
|
using Npgsql.EntityFrameworkCore.PostgreSQL;
|
||||||
|
|
||||||
|
namespace LiveRecorder.Infrastructure.Persistence;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Custom EF Core execution strategy that integrates with <see cref="DatabaseCircuitBreaker"/>.
|
||||||
|
///
|
||||||
|
/// When the circuit is open, operations fail immediately without retrying.
|
||||||
|
/// When a non-transient error occurs (e.g. disk_full), the operation does not retry.
|
||||||
|
/// After each failure, the circuit is recorded, and after each success, the circuit is reset.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class CircuitAwareExecutionStrategy : NpgsqlRetryingExecutionStrategy
|
||||||
|
{
|
||||||
|
private static readonly TimeSpan DefaultMaxRetryDelay = TimeSpan.FromSeconds(15);
|
||||||
|
|
||||||
|
public CircuitAwareExecutionStrategy(
|
||||||
|
ExecutionStrategyDependencies dependencies,
|
||||||
|
int maxRetryCount,
|
||||||
|
TimeSpan maxRetryDelay)
|
||||||
|
: base(dependencies, maxRetryCount, maxRetryDelay, errorCodesToAdd: null)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
public CircuitAwareExecutionStrategy(
|
||||||
|
ExecutionStrategyDependencies dependencies,
|
||||||
|
int maxRetryCount,
|
||||||
|
TimeSpan maxRetryDelay,
|
||||||
|
ICollection<string>? errorCodesToAdd)
|
||||||
|
: base(dependencies, maxRetryCount, maxRetryDelay, errorCodesToAdd)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override bool ShouldRetryOn(Exception? exception)
|
||||||
|
{
|
||||||
|
// Fast-fail: circuit is open
|
||||||
|
if (DatabaseCircuitBreaker.IsOpen)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fast-fail: non-transient errors (disk_full, out_of_memory, etc.)
|
||||||
|
if (DatabaseCircuitBreaker.IsNonTransient(exception))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delegate to default Npgsql retry logic for transient errors
|
||||||
|
return base.ShouldRetryOn(exception);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void OnFirstExecution()
|
||||||
|
{
|
||||||
|
// If circuit is open, throw immediately before even attempting
|
||||||
|
if (DatabaseCircuitBreaker.IsOpen)
|
||||||
|
{
|
||||||
|
throw new DatabaseCircuitOpenException(
|
||||||
|
$"Database circuit breaker is open. Consecutive failures: {DatabaseCircuitBreaker.ConsecutiveFailures}. " +
|
||||||
|
$"Circuit opened at: {DatabaseCircuitBreaker.OpenedAt:O}.");
|
||||||
|
}
|
||||||
|
|
||||||
|
base.OnFirstExecution();
|
||||||
|
}
|
||||||
|
|
||||||
|
public override TResult Execute<TState, TResult>(
|
||||||
|
TState state,
|
||||||
|
Func<DbContext, TState, TResult> operation,
|
||||||
|
Func<DbContext, TState, ExecutionResult<TResult>>? verifySucceeded)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var result = base.Execute(state, operation, verifySucceeded);
|
||||||
|
DatabaseCircuitBreaker.RecordSuccess();
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
DatabaseCircuitBreaker.RecordFailure();
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public override async Task<TResult> ExecuteAsync<TState, TResult>(
|
||||||
|
TState state,
|
||||||
|
Func<DbContext, TState, CancellationToken, Task<TResult>> operation,
|
||||||
|
Func<DbContext, TState, CancellationToken, Task<ExecutionResult<TResult>>>? verifySucceeded,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var result = await base.ExecuteAsync(state, operation, verifySucceeded, cancellationToken)
|
||||||
|
.ConfigureAwait(false);
|
||||||
|
DatabaseCircuitBreaker.RecordSuccess();
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
DatabaseCircuitBreaker.RecordFailure();
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
|
||||||
|
namespace LiveRecorder.Infrastructure.Persistence;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Thread-safe static circuit breaker for database operations.
|
||||||
|
/// When the database becomes unavailable (e.g. disk full), this prevents
|
||||||
|
/// every API request from wasting 45 seconds on doomed retries.
|
||||||
|
/// </summary>
|
||||||
|
public static class DatabaseCircuitBreaker
|
||||||
|
{
|
||||||
|
private static readonly object Lock = new();
|
||||||
|
|
||||||
|
/// <summary>Consecutive failures before the circuit opens.</summary>
|
||||||
|
private const int FailureThreshold = 5;
|
||||||
|
|
||||||
|
/// <summary>How long the circuit stays open before allowing a probe.</summary>
|
||||||
|
private static readonly TimeSpan BreakDuration = TimeSpan.FromSeconds(30);
|
||||||
|
|
||||||
|
/// <summary>Npgsql error codes that are NOT transient — retrying is futile.</summary>
|
||||||
|
private static readonly HashSet<string> NonTransientCodes = new(StringComparer.Ordinal)
|
||||||
|
{
|
||||||
|
"53100", // disk_full
|
||||||
|
"53200", // out_of_memory
|
||||||
|
"53300", // too_many_connections
|
||||||
|
"08006", // connection_failure (persistent)
|
||||||
|
"57P03", // cannot_connect_now
|
||||||
|
"42601", // syntax_error (bug, not transient)
|
||||||
|
"42501", // insufficient_privilege
|
||||||
|
"3D000", // invalid_catalog_name
|
||||||
|
"28P01", // invalid_password
|
||||||
|
};
|
||||||
|
|
||||||
|
private static int _consecutiveFailures;
|
||||||
|
private static DateTimeOffset _openedAt = DateTimeOffset.MinValue;
|
||||||
|
|
||||||
|
/// <summary>Whether the circuit is currently open (failing fast).</summary>
|
||||||
|
public static bool IsOpen
|
||||||
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
|
if (_consecutiveFailures < FailureThreshold)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (DateTimeOffset.UtcNow - _openedAt > BreakDuration)
|
||||||
|
{
|
||||||
|
// Transition to half-open: allow one probe
|
||||||
|
lock (Lock)
|
||||||
|
{
|
||||||
|
if (_consecutiveFailures >= FailureThreshold &&
|
||||||
|
DateTimeOffset.UtcNow - _openedAt > BreakDuration)
|
||||||
|
{
|
||||||
|
// Reset to just below threshold so the next call probes
|
||||||
|
_consecutiveFailures = FailureThreshold - 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static int ConsecutiveFailures => Volatile.Read(ref _consecutiveFailures);
|
||||||
|
|
||||||
|
public static DateTimeOffset OpenedAt => _openedAt;
|
||||||
|
|
||||||
|
/// <summary>Record a successful database operation.</summary>
|
||||||
|
public static void RecordSuccess()
|
||||||
|
{
|
||||||
|
lock (Lock)
|
||||||
|
{
|
||||||
|
_consecutiveFailures = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Record a failed database operation.</summary>
|
||||||
|
public static void RecordFailure()
|
||||||
|
{
|
||||||
|
lock (Lock)
|
||||||
|
{
|
||||||
|
_consecutiveFailures++;
|
||||||
|
if (_consecutiveFailures >= FailureThreshold)
|
||||||
|
{
|
||||||
|
_openedAt = DateTimeOffset.UtcNow;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Check whether a given exception is a non-transient database error
|
||||||
|
/// that should NOT be retried. Returns true if retrying would be futile.
|
||||||
|
/// </summary>
|
||||||
|
public static bool IsNonTransient(Exception? ex)
|
||||||
|
{
|
||||||
|
while (ex is not null)
|
||||||
|
{
|
||||||
|
if (ex is Npgsql.NpgsqlException npgEx && npgEx.SqlState is { Length: 5 } state)
|
||||||
|
{
|
||||||
|
return NonTransientCodes.Contains(state);
|
||||||
|
}
|
||||||
|
|
||||||
|
ex = ex.InnerException;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Log the current circuit state.</summary>
|
||||||
|
public static void LogState(ILogger logger)
|
||||||
|
{
|
||||||
|
logger.LogInformation(
|
||||||
|
"DatabaseCircuitBreaker state: Open={IsOpen}, ConsecutiveFailures={Failures}, OpenedAt={OpenedAt}",
|
||||||
|
IsOpen,
|
||||||
|
ConsecutiveFailures,
|
||||||
|
OpenedAt == DateTimeOffset.MinValue ? "never" : OpenedAt.ToString("O"));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
namespace LiveRecorder.Infrastructure.Persistence;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Thrown when a database operation is rejected because the circuit breaker is open.
|
||||||
|
/// This is a fast-fail — the request will not be retried.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class DatabaseCircuitOpenException : InvalidOperationException
|
||||||
|
{
|
||||||
|
public DatabaseCircuitOpenException(string message) : base(message)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
public DatabaseCircuitOpenException(string message, Exception innerException) : base(message, innerException)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
|
using LiveRecorder.Infrastructure.Persistence;
|
||||||
using Microsoft.Extensions.DependencyInjection;
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
using Microsoft.Extensions.Hosting;
|
using Microsoft.Extensions.Hosting;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
@@ -7,6 +8,8 @@ namespace LiveRecorder.Infrastructure.Services;
|
|||||||
public sealed class CleanupOperationBackgroundService : BackgroundService
|
public sealed class CleanupOperationBackgroundService : BackgroundService
|
||||||
{
|
{
|
||||||
private static readonly TimeSpan IdleDelay = TimeSpan.FromSeconds(2);
|
private static readonly TimeSpan IdleDelay = TimeSpan.FromSeconds(2);
|
||||||
|
private static readonly TimeSpan ErrorBaseDelay = TimeSpan.FromSeconds(5);
|
||||||
|
private static readonly TimeSpan ErrorMaxDelay = TimeSpan.FromMinutes(5);
|
||||||
private readonly IServiceScopeFactory _serviceScopeFactory;
|
private readonly IServiceScopeFactory _serviceScopeFactory;
|
||||||
private readonly ILogger<CleanupOperationBackgroundService> _logger;
|
private readonly ILogger<CleanupOperationBackgroundService> _logger;
|
||||||
|
|
||||||
@@ -20,6 +23,7 @@ public sealed class CleanupOperationBackgroundService : BackgroundService
|
|||||||
|
|
||||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||||
{
|
{
|
||||||
|
// Startup: requeue interrupted operations
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
using var startupScope = _serviceScopeFactory.CreateScope();
|
using var startupScope = _serviceScopeFactory.CreateScope();
|
||||||
@@ -31,17 +35,36 @@ public sealed class CleanupOperationBackgroundService : BackgroundService
|
|||||||
_logger.LogWarning(ex, "Failed to requeue interrupted cleanup operations at startup");
|
_logger.LogWarning(ex, "Failed to requeue interrupted cleanup operations at startup");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var consecutiveErrors = 0;
|
||||||
|
|
||||||
while (!stoppingToken.IsCancellationRequested)
|
while (!stoppingToken.IsCancellationRequested)
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
|
// Skip processing if the circuit is open — don't waste resources
|
||||||
|
if (DatabaseCircuitBreaker.IsOpen)
|
||||||
|
{
|
||||||
|
consecutiveErrors = await DelayWithBackoff(ErrorBaseDelay, ErrorMaxDelay, consecutiveErrors, stoppingToken);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
using var scope = _serviceScopeFactory.CreateScope();
|
using var scope = _serviceScopeFactory.CreateScope();
|
||||||
var coordinator = scope.ServiceProvider.GetRequiredService<CleanupOperationCoordinator>();
|
var coordinator = scope.ServiceProvider.GetRequiredService<CleanupOperationCoordinator>();
|
||||||
var processed = await coordinator.ProcessNextQueuedOperationAsync(stoppingToken);
|
var processed = await coordinator.ProcessNextQueuedOperationAsync(stoppingToken);
|
||||||
|
|
||||||
if (processed)
|
if (processed)
|
||||||
{
|
{
|
||||||
|
consecutiveErrors = 0;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// No queued operations — idle delay
|
||||||
|
consecutiveErrors = 0;
|
||||||
|
await Task.Delay(IdleDelay, stoppingToken);
|
||||||
|
}
|
||||||
|
catch (DatabaseCircuitOpenException)
|
||||||
|
{
|
||||||
|
consecutiveErrors = await DelayWithBackoff(ErrorBaseDelay, ErrorMaxDelay, consecutiveErrors, stoppingToken);
|
||||||
}
|
}
|
||||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||||
{
|
{
|
||||||
@@ -49,17 +72,47 @@ public sealed class CleanupOperationBackgroundService : BackgroundService
|
|||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
_logger.LogError(ex, "Cleanup operation background worker failed");
|
var isDatabaseError = DatabaseCircuitBreaker.IsNonTransient(ex) ||
|
||||||
}
|
ex is Npgsql.NpgsqlException ||
|
||||||
|
ex is Microsoft.EntityFrameworkCore.DbUpdateException;
|
||||||
|
|
||||||
try
|
if (isDatabaseError)
|
||||||
{
|
{
|
||||||
await Task.Delay(IdleDelay, stoppingToken);
|
DatabaseCircuitBreaker.RecordFailure();
|
||||||
}
|
_logger.LogWarning(ex,
|
||||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
"Cleanup background worker: database error (#{ErrorCount}). Circuit state: Open={IsOpen}, Failures={Failures}",
|
||||||
{
|
consecutiveErrors + 1,
|
||||||
break;
|
DatabaseCircuitBreaker.IsOpen,
|
||||||
|
DatabaseCircuitBreaker.ConsecutiveFailures);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "Cleanup operation background worker failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
consecutiveErrors = await DelayWithBackoff(ErrorBaseDelay, ErrorMaxDelay, consecutiveErrors, stoppingToken);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static async Task<int> DelayWithBackoff(
|
||||||
|
TimeSpan baseDelay, TimeSpan maxDelay, int errorCount, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
errorCount++;
|
||||||
|
// Exponential backoff: 5s, 10s, 20s, 40s, 80s, 160s, capping at 5min
|
||||||
|
var factor = Math.Pow(2, Math.Min(errorCount - 1, 6));
|
||||||
|
var delay = TimeSpan.FromMilliseconds(
|
||||||
|
Math.Min(baseDelay.TotalMilliseconds * factor, maxDelay.TotalMilliseconds));
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await Task.Delay(delay, cancellationToken);
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException)
|
||||||
|
{
|
||||||
|
// Swallow — loop will exit on next iteration
|
||||||
|
}
|
||||||
|
|
||||||
|
return errorCount;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -146,8 +146,20 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
|
|||||||
{
|
{
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
catch (DatabaseCircuitOpenException)
|
||||||
|
{
|
||||||
|
// Circuit is open — skip this iteration and wait
|
||||||
|
_logger.LogDebug("Polling loop skipped: database circuit breaker is open");
|
||||||
|
delay = TimeSpan.FromSeconds(30);
|
||||||
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
|
// Record database failures to the circuit breaker
|
||||||
|
if (IsTransientDatabaseException(ex) || DatabaseCircuitBreaker.IsNonTransient(ex))
|
||||||
|
{
|
||||||
|
DatabaseCircuitBreaker.RecordFailure();
|
||||||
|
}
|
||||||
|
|
||||||
_logger.LogError(ex, "Background live room polling failed");
|
_logger.LogError(ex, "Background live room polling failed");
|
||||||
|
|
||||||
try
|
try
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
using LiveRecorder.Infrastructure.Persistence;
|
||||||
using Microsoft.Extensions.DependencyInjection;
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
using Microsoft.Extensions.Hosting;
|
using Microsoft.Extensions.Hosting;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
@@ -7,7 +8,8 @@ namespace LiveRecorder.Infrastructure.Services;
|
|||||||
public sealed class RetentionCleanupBackgroundService : BackgroundService
|
public sealed class RetentionCleanupBackgroundService : BackgroundService
|
||||||
{
|
{
|
||||||
private static readonly TimeSpan CleanupInterval = TimeSpan.FromHours(24);
|
private static readonly TimeSpan CleanupInterval = TimeSpan.FromHours(24);
|
||||||
|
private static readonly TimeSpan ErrorBaseDelay = TimeSpan.FromSeconds(10);
|
||||||
|
private static readonly TimeSpan ErrorMaxDelay = TimeSpan.FromMinutes(5);
|
||||||
private readonly IServiceScopeFactory _serviceScopeFactory;
|
private readonly IServiceScopeFactory _serviceScopeFactory;
|
||||||
private readonly ILogger<RetentionCleanupBackgroundService> _logger;
|
private readonly ILogger<RetentionCleanupBackgroundService> _logger;
|
||||||
|
|
||||||
@@ -21,13 +23,31 @@ public sealed class RetentionCleanupBackgroundService : BackgroundService
|
|||||||
|
|
||||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||||
{
|
{
|
||||||
|
var consecutiveErrors = 0;
|
||||||
|
|
||||||
while (!stoppingToken.IsCancellationRequested)
|
while (!stoppingToken.IsCancellationRequested)
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
|
// Skip if circuit is already open
|
||||||
|
if (DatabaseCircuitBreaker.IsOpen)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("Retention cleanup skipped: database circuit breaker is open");
|
||||||
|
await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
using var scope = _serviceScopeFactory.CreateScope();
|
using var scope = _serviceScopeFactory.CreateScope();
|
||||||
var cleanupService = scope.ServiceProvider.GetRequiredService<RetentionCleanupService>();
|
var cleanupService = scope.ServiceProvider.GetRequiredService<RetentionCleanupService>();
|
||||||
await cleanupService.TryEnqueueAsync(ignoreEnabledSetting: false, cancellationToken: stoppingToken);
|
await cleanupService.TryEnqueueAsync(ignoreEnabledSetting: false, cancellationToken: stoppingToken);
|
||||||
|
|
||||||
|
consecutiveErrors = 0;
|
||||||
|
}
|
||||||
|
catch (DatabaseCircuitOpenException)
|
||||||
|
{
|
||||||
|
// Silent — circuit is already logged
|
||||||
|
await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken);
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||||
{
|
{
|
||||||
@@ -35,7 +55,38 @@ public sealed class RetentionCleanupBackgroundService : BackgroundService
|
|||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
_logger.LogError(ex, "Retention cleanup background task failed");
|
var isDatabaseError = DatabaseCircuitBreaker.IsNonTransient(ex) ||
|
||||||
|
ex is Npgsql.NpgsqlException ||
|
||||||
|
ex is Microsoft.EntityFrameworkCore.DbUpdateException;
|
||||||
|
|
||||||
|
consecutiveErrors++;
|
||||||
|
var delay = TimeSpan.FromMilliseconds(
|
||||||
|
Math.Min(ErrorBaseDelay.TotalMilliseconds * Math.Pow(2, Math.Min(consecutiveErrors - 1, 6)),
|
||||||
|
ErrorMaxDelay.TotalMilliseconds));
|
||||||
|
|
||||||
|
if (isDatabaseError)
|
||||||
|
{
|
||||||
|
DatabaseCircuitBreaker.RecordFailure();
|
||||||
|
_logger.LogWarning(ex,
|
||||||
|
"Retention cleanup: database error (#{ErrorCount}). Circuit: Open={IsOpen}, Failures={Failures}",
|
||||||
|
consecutiveErrors,
|
||||||
|
DatabaseCircuitBreaker.IsOpen,
|
||||||
|
DatabaseCircuitBreaker.ConsecutiveFailures);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "Retention cleanup background task failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await Task.Delay(delay, stoppingToken);
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException)
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
try
|
try
|
||||||
|
|||||||
@@ -61,6 +61,12 @@ public sealed class RecordTasksController : ControllerBase
|
|||||||
CancellationToken cancellationToken) =>
|
CancellationToken cancellationToken) =>
|
||||||
Ok(await _recordService.DeleteTasksAsync(request, cancellationToken));
|
Ok(await _recordService.DeleteTasksAsync(request, cancellationToken));
|
||||||
|
|
||||||
|
[HttpPost("delete-missing-files")]
|
||||||
|
public async Task<ActionResult<DeleteCompletedRecordTasksResultDto>> DeleteMissingFiles(
|
||||||
|
[FromBody] DeleteMissingFileRecordTasksRequest request,
|
||||||
|
CancellationToken cancellationToken) =>
|
||||||
|
Ok(await _recordService.DeleteMissingFileTasksAsync(request, cancellationToken));
|
||||||
|
|
||||||
[HttpPost("{id:guid}/preview-ticket")]
|
[HttpPost("{id:guid}/preview-ticket")]
|
||||||
public async Task<ActionResult<RecordPreviewTicketDto>> CreatePreviewTicket(Guid id, CancellationToken cancellationToken)
|
public async Task<ActionResult<RecordPreviewTicketDto>> CreatePreviewTicket(Guid id, CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -4,16 +4,15 @@ ARG DOTNET_RUNTIME_IMAGE=mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim
|
|||||||
FROM ${DOTNET_SDK_IMAGE} AS build
|
FROM ${DOTNET_SDK_IMAGE} AS build
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
|
|
||||||
COPY ["LiveRecorder.sln", "./"]
|
|
||||||
COPY ["src/LiveRecorder.Domain/LiveRecorder.Domain.csproj", "src/LiveRecorder.Domain/"]
|
|
||||||
COPY ["src/LiveRecorder.Application/LiveRecorder.Application.csproj", "src/LiveRecorder.Application/"]
|
|
||||||
COPY ["src/LiveRecorder.Infrastructure/LiveRecorder.Infrastructure.csproj", "src/LiveRecorder.Infrastructure/"]
|
|
||||||
COPY ["src/LiveRecorder.WebApi/LiveRecorder.WebApi.csproj", "src/LiveRecorder.WebApi/"]
|
|
||||||
|
|
||||||
RUN dotnet restore "src/LiveRecorder.WebApi/LiveRecorder.WebApi.csproj"
|
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN dotnet publish "src/LiveRecorder.WebApi/LiveRecorder.WebApi.csproj" -c Release -o /app/publish /p:UseAppHost=false
|
|
||||||
|
# Workaround for QEMU ARM64 emulation
|
||||||
|
ENV DOTNET_EnableWriteXorExecute=0
|
||||||
|
ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1
|
||||||
|
ENV DOTNET_GCConserveMemory=9
|
||||||
|
|
||||||
|
RUN dotnet restore "src/LiveRecorder.WebApi/LiveRecorder.WebApi.csproj" -p:RestoreUseStaticGraphEvaluation=true
|
||||||
|
RUN dotnet publish "src/LiveRecorder.WebApi/LiveRecorder.WebApi.csproj" -c Release -o /app/publish /p:UseAppHost=false /p:DebugType=None /p:DebugSymbols=false /maxcpucount:1
|
||||||
|
|
||||||
FROM ${DOTNET_RUNTIME_IMAGE} AS runtime
|
FROM ${DOTNET_RUNTIME_IMAGE} AS runtime
|
||||||
|
|
||||||
|
|||||||
@@ -17,6 +17,8 @@
|
|||||||
<TargetFramework>net8.0</TargetFramework>
|
<TargetFramework>net8.0</TargetFramework>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
<RuntimeIdentifier>linux-arm64</RuntimeIdentifier>
|
||||||
|
<SelfContained>false</SelfContained>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
|||||||
@@ -1,3 +1,7 @@
|
|||||||
|
using LiveRecorder.Infrastructure.Persistence;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Npgsql;
|
||||||
|
|
||||||
namespace LiveRecorder.WebApi.Middleware;
|
namespace LiveRecorder.WebApi.Middleware;
|
||||||
|
|
||||||
public sealed class ExceptionHandlingMiddleware
|
public sealed class ExceptionHandlingMiddleware
|
||||||
@@ -17,22 +21,80 @@ public sealed class ExceptionHandlingMiddleware
|
|||||||
{
|
{
|
||||||
await _next(context);
|
await _next(context);
|
||||||
}
|
}
|
||||||
|
catch (DatabaseCircuitOpenException ex)
|
||||||
|
{
|
||||||
|
// Circuit is open — fast-fail with 503
|
||||||
|
_logger.LogWarning(ex, "Database circuit breaker is open, returning 503");
|
||||||
|
context.Response.StatusCode = StatusCodes.Status503ServiceUnavailable;
|
||||||
|
await context.Response.WriteAsJsonAsync(new
|
||||||
|
{
|
||||||
|
message = "Database temporarily unavailable",
|
||||||
|
detail = "The database is currently unreachable. Requests will be accepted again once connectivity is restored.",
|
||||||
|
retryAfterSeconds = 30
|
||||||
|
});
|
||||||
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
_logger.LogError(ex, "Unhandled exception");
|
_logger.LogError(ex, "Unhandled exception");
|
||||||
context.Response.StatusCode = ex switch
|
|
||||||
|
// Detect database-related exceptions and return 503 instead of 500
|
||||||
|
var statusCode = ex switch
|
||||||
{
|
{
|
||||||
|
DatabaseCircuitOpenException => StatusCodes.Status503ServiceUnavailable,
|
||||||
KeyNotFoundException => StatusCodes.Status404NotFound,
|
KeyNotFoundException => StatusCodes.Status404NotFound,
|
||||||
InvalidOperationException => StatusCodes.Status400BadRequest,
|
InvalidOperationException => StatusCodes.Status400BadRequest,
|
||||||
NotSupportedException => StatusCodes.Status400BadRequest,
|
NotSupportedException => StatusCodes.Status400BadRequest,
|
||||||
|
_ when IsDatabaseException(ex) => StatusCodes.Status503ServiceUnavailable,
|
||||||
_ => StatusCodes.Status500InternalServerError
|
_ => StatusCodes.Status500InternalServerError
|
||||||
};
|
};
|
||||||
|
|
||||||
|
context.Response.StatusCode = statusCode;
|
||||||
|
|
||||||
await context.Response.WriteAsJsonAsync(new
|
await context.Response.WriteAsJsonAsync(new
|
||||||
{
|
{
|
||||||
message = ex.Message,
|
message = statusCode == StatusCodes.Status503ServiceUnavailable
|
||||||
detail = context.Response.StatusCode == StatusCodes.Status500InternalServerError ? "Internal Server Error" : null
|
? "Database temporarily unavailable"
|
||||||
|
: ex.Message,
|
||||||
|
detail = statusCode switch
|
||||||
|
{
|
||||||
|
StatusCodes.Status503ServiceUnavailable => "The database is currently unreachable. Please retry later.",
|
||||||
|
StatusCodes.Status500InternalServerError => "Internal Server Error",
|
||||||
|
_ => null
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Check if an exception (or any of its inner exceptions) is a database-related error
|
||||||
|
/// that should be surfaced as 503 Service Unavailable.
|
||||||
|
/// </summary>
|
||||||
|
private static bool IsDatabaseException(Exception ex)
|
||||||
|
{
|
||||||
|
var current = ex;
|
||||||
|
while (current is not null)
|
||||||
|
{
|
||||||
|
if (current is NpgsqlException or DbUpdateException)
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (current is InvalidOperationException ioEx &&
|
||||||
|
(ioEx.Message.Contains("database", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
ioEx.Message.Contains("connection", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
ioEx.Message.Contains("Npgsql", StringComparison.OrdinalIgnoreCase)))
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (current is TimeoutException && current.Message.Contains("database", StringComparison.OrdinalIgnoreCase))
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
current = current.InnerException;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ using LiveRecorder.Application.Abstractions.Recording;
|
|||||||
using LiveRecorder.Application.Abstractions.Scripting;
|
using LiveRecorder.Application.Abstractions.Scripting;
|
||||||
using LiveRecorder.Application.Abstractions.Settings;
|
using LiveRecorder.Application.Abstractions.Settings;
|
||||||
using LiveRecorder.Application.Abstractions.Storage;
|
using LiveRecorder.Application.Abstractions.Storage;
|
||||||
|
using LiveRecorder.Application.Models.Reports;
|
||||||
using LiveRecorder.Application.Services;
|
using LiveRecorder.Application.Services;
|
||||||
using LiveRecorder.Infrastructure.Persistence;
|
using LiveRecorder.Infrastructure.Persistence;
|
||||||
using LiveRecorder.Infrastructure.Persistence.Repositories;
|
using LiveRecorder.Infrastructure.Persistence.Repositories;
|
||||||
@@ -31,6 +32,7 @@ using LiveRecorder.Infrastructure.Platforms.YouTube;
|
|||||||
using LiveRecorder.Infrastructure.Services;
|
using LiveRecorder.Infrastructure.Services;
|
||||||
using LiveRecorder.WebApi.Middleware;
|
using LiveRecorder.WebApi.Middleware;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.EntityFrameworkCore.Storage;
|
||||||
using Microsoft.OpenApi.Models;
|
using Microsoft.OpenApi.Models;
|
||||||
using Npgsql;
|
using Npgsql;
|
||||||
|
|
||||||
@@ -149,6 +151,11 @@ builder.Services.AddDbContext<LiveRecorderDbContext>(options =>
|
|||||||
maxRetryCount: 3,
|
maxRetryCount: 3,
|
||||||
maxRetryDelay: TimeSpan.FromSeconds(15),
|
maxRetryDelay: TimeSpan.FromSeconds(15),
|
||||||
errorCodesToAdd: null);
|
errorCodesToAdd: null);
|
||||||
|
npgsql.ExecutionStrategy(dependencies =>
|
||||||
|
new CircuitAwareExecutionStrategy(
|
||||||
|
dependencies,
|
||||||
|
maxRetryCount: 3,
|
||||||
|
maxRetryDelay: TimeSpan.FromSeconds(15)));
|
||||||
}));
|
}));
|
||||||
|
|
||||||
builder.Services.AddScoped<IUnitOfWork>(provider => provider.GetRequiredService<LiveRecorderDbContext>());
|
builder.Services.AddScoped<IUnitOfWork>(provider => provider.GetRequiredService<LiveRecorderDbContext>());
|
||||||
@@ -231,6 +238,61 @@ app.UseMiddleware<ApiTokenAuthenticationMiddleware>();
|
|||||||
app.MapGet("/", () => Results.Redirect("/swagger"));
|
app.MapGet("/", () => Results.Redirect("/swagger"));
|
||||||
app.MapControllers();
|
app.MapControllers();
|
||||||
|
|
||||||
|
// ── Health check endpoints ────────────────────────────────────────────
|
||||||
|
// /health — liveness: is the process alive and responding?
|
||||||
|
app.MapGet("/health", () => Results.Ok(new { status = "healthy", timestamp = DateTimeOffset.UtcNow }));
|
||||||
|
|
||||||
|
// /health/ready — readiness: can we reach the database?
|
||||||
|
app.MapGet("/health/ready", async (CancellationToken cancellationToken) =>
|
||||||
|
{
|
||||||
|
var readyResult = new HealthReadyResponse
|
||||||
|
{
|
||||||
|
Status = "ready",
|
||||||
|
Timestamp = DateTimeOffset.UtcNow,
|
||||||
|
Database = new DatabaseHealthStatus()
|
||||||
|
};
|
||||||
|
|
||||||
|
if (DatabaseCircuitBreaker.IsOpen)
|
||||||
|
{
|
||||||
|
readyResult.Status = "degraded";
|
||||||
|
readyResult.Database = new DatabaseHealthStatus
|
||||||
|
{
|
||||||
|
Reachable = false,
|
||||||
|
Reason = "Circuit breaker is open",
|
||||||
|
ConsecutiveFailures = DatabaseCircuitBreaker.ConsecutiveFailures
|
||||||
|
};
|
||||||
|
return Results.Json(readyResult, statusCode: StatusCodes.Status503ServiceUnavailable);
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
// Lightweight probe: just test connectivity with a 2-second timeout
|
||||||
|
await using var connection = new NpgsqlConnection(connectionStringBuilder.ConnectionString);
|
||||||
|
var probeCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||||
|
probeCts.CancelAfter(TimeSpan.FromSeconds(2));
|
||||||
|
|
||||||
|
await connection.OpenAsync(probeCts.Token);
|
||||||
|
await using var cmd = connection.CreateCommand();
|
||||||
|
cmd.CommandText = "SELECT 1";
|
||||||
|
await cmd.ExecuteScalarAsync(probeCts.Token);
|
||||||
|
|
||||||
|
readyResult.Database = new DatabaseHealthStatus { Reachable = true };
|
||||||
|
return Results.Ok(readyResult);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
DatabaseCircuitBreaker.RecordFailure();
|
||||||
|
readyResult.Status = "unhealthy";
|
||||||
|
readyResult.Database = new DatabaseHealthStatus
|
||||||
|
{
|
||||||
|
Reachable = false,
|
||||||
|
Reason = ex.Message,
|
||||||
|
ConsecutiveFailures = DatabaseCircuitBreaker.ConsecutiveFailures
|
||||||
|
};
|
||||||
|
return Results.Json(readyResult, statusCode: StatusCodes.Status503ServiceUnavailable);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
using (var scope = app.Services.CreateScope())
|
using (var scope = app.Services.CreateScope())
|
||||||
{
|
{
|
||||||
var initializer = scope.ServiceProvider.GetRequiredService<DatabaseInitializer>();
|
var initializer = scope.ServiceProvider.GetRequiredService<DatabaseInitializer>();
|
||||||
|
|||||||
@@ -0,0 +1,122 @@
|
|||||||
|
using System.Reflection;
|
||||||
|
using LiveRecorder.Infrastructure.Persistence;
|
||||||
|
using Npgsql;
|
||||||
|
|
||||||
|
namespace LiveRecorder.Tests;
|
||||||
|
|
||||||
|
public sealed class DatabaseCircuitBreakerTests
|
||||||
|
{
|
||||||
|
// Mirrors the private FailureThreshold in DatabaseCircuitBreaker so the assertions
|
||||||
|
// read intentionally. Keep in sync if the production threshold changes.
|
||||||
|
private const int FailureThreshold = 5;
|
||||||
|
|
||||||
|
public DatabaseCircuitBreakerTests()
|
||||||
|
{
|
||||||
|
// The breaker is a process-wide static, so reset it to a known closed state
|
||||||
|
// before each test to keep cases independent.
|
||||||
|
DatabaseCircuitBreaker.RecordSuccess();
|
||||||
|
SetOpenedAt(DateTimeOffset.MinValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void RecordFailure_below_threshold_keeps_circuit_closed()
|
||||||
|
{
|
||||||
|
for (var i = 0; i < FailureThreshold - 1; i++)
|
||||||
|
{
|
||||||
|
DatabaseCircuitBreaker.RecordFailure();
|
||||||
|
}
|
||||||
|
|
||||||
|
Assert.False(DatabaseCircuitBreaker.IsOpen);
|
||||||
|
Assert.Equal(FailureThreshold - 1, DatabaseCircuitBreaker.ConsecutiveFailures);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void RecordFailure_at_threshold_opens_circuit()
|
||||||
|
{
|
||||||
|
for (var i = 0; i < FailureThreshold; i++)
|
||||||
|
{
|
||||||
|
DatabaseCircuitBreaker.RecordFailure();
|
||||||
|
}
|
||||||
|
|
||||||
|
Assert.True(DatabaseCircuitBreaker.IsOpen);
|
||||||
|
Assert.Equal(FailureThreshold, DatabaseCircuitBreaker.ConsecutiveFailures);
|
||||||
|
Assert.NotEqual(DateTimeOffset.MinValue, DatabaseCircuitBreaker.OpenedAt);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void RecordSuccess_closes_circuit_and_resets_failures()
|
||||||
|
{
|
||||||
|
for (var i = 0; i < FailureThreshold; i++)
|
||||||
|
{
|
||||||
|
DatabaseCircuitBreaker.RecordFailure();
|
||||||
|
}
|
||||||
|
|
||||||
|
Assert.True(DatabaseCircuitBreaker.IsOpen);
|
||||||
|
|
||||||
|
DatabaseCircuitBreaker.RecordSuccess();
|
||||||
|
|
||||||
|
Assert.False(DatabaseCircuitBreaker.IsOpen);
|
||||||
|
Assert.Equal(0, DatabaseCircuitBreaker.ConsecutiveFailures);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void IsOpen_after_break_duration_transitions_to_half_open()
|
||||||
|
{
|
||||||
|
for (var i = 0; i < FailureThreshold; i++)
|
||||||
|
{
|
||||||
|
DatabaseCircuitBreaker.RecordFailure();
|
||||||
|
}
|
||||||
|
|
||||||
|
Assert.True(DatabaseCircuitBreaker.IsOpen);
|
||||||
|
|
||||||
|
// The breaker reads DateTimeOffset.UtcNow directly (no injectable clock), so move
|
||||||
|
// the recorded open time past the 30s break window to simulate it elapsing.
|
||||||
|
SetOpenedAt(DateTimeOffset.UtcNow - TimeSpan.FromSeconds(31));
|
||||||
|
|
||||||
|
// The first read past the window half-opens: it permits one probe and drops the
|
||||||
|
// failure count to one below the threshold.
|
||||||
|
Assert.False(DatabaseCircuitBreaker.IsOpen);
|
||||||
|
Assert.Equal(FailureThreshold - 1, DatabaseCircuitBreaker.ConsecutiveFailures);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Theory]
|
||||||
|
[InlineData("53100")] // disk_full
|
||||||
|
[InlineData("53300")] // too_many_connections
|
||||||
|
[InlineData("28P01")] // invalid_password
|
||||||
|
public void IsNonTransient_returns_true_for_known_fatal_sql_states(string sqlState)
|
||||||
|
{
|
||||||
|
var exception = new PostgresException("fatal", "FATAL", "FATAL", sqlState);
|
||||||
|
|
||||||
|
Assert.True(DatabaseCircuitBreaker.IsNonTransient(exception));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void IsNonTransient_returns_false_for_transient_sql_state()
|
||||||
|
{
|
||||||
|
// 40001 = serialization_failure, which is retryable and not in the fatal set.
|
||||||
|
var exception = new PostgresException("retry me", "ERROR", "ERROR", "40001");
|
||||||
|
|
||||||
|
Assert.False(DatabaseCircuitBreaker.IsNonTransient(exception));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void IsNonTransient_unwraps_inner_exceptions()
|
||||||
|
{
|
||||||
|
var inner = new PostgresException("disk full", "FATAL", "FATAL", "53100");
|
||||||
|
var wrapper = new InvalidOperationException("save failed", inner);
|
||||||
|
|
||||||
|
Assert.True(DatabaseCircuitBreaker.IsNonTransient(wrapper));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void IsNonTransient_returns_false_for_null_and_non_postgres_exceptions()
|
||||||
|
{
|
||||||
|
Assert.False(DatabaseCircuitBreaker.IsNonTransient(null));
|
||||||
|
Assert.False(DatabaseCircuitBreaker.IsNonTransient(new InvalidOperationException("boom")));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void SetOpenedAt(DateTimeOffset value) =>
|
||||||
|
typeof(DatabaseCircuitBreaker)
|
||||||
|
.GetField("_openedAt", BindingFlags.NonPublic | BindingFlags.Static)!
|
||||||
|
.SetValue(null, value);
|
||||||
|
}
|
||||||
@@ -20,6 +20,7 @@
|
|||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<ProjectReference Include="..\..\src\LiveRecorder.Application\LiveRecorder.Application.csproj" />
|
<ProjectReference Include="..\..\src\LiveRecorder.Application\LiveRecorder.Application.csproj" />
|
||||||
<ProjectReference Include="..\..\src\LiveRecorder.Domain\LiveRecorder.Domain.csproj" />
|
<ProjectReference Include="..\..\src\LiveRecorder.Domain\LiveRecorder.Domain.csproj" />
|
||||||
|
<ProjectReference Include="..\..\src\LiveRecorder.Infrastructure\LiveRecorder.Infrastructure.csproj" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
|||||||
Reference in New Issue
Block a user