diff --git a/src/LiveRecorder.Infrastructure/Persistence/CircuitAwareExecutionStrategy.cs b/src/LiveRecorder.Infrastructure/Persistence/CircuitAwareExecutionStrategy.cs index a5386b4..ec4c087 100644 --- a/src/LiveRecorder.Infrastructure/Persistence/CircuitAwareExecutionStrategy.cs +++ b/src/LiveRecorder.Infrastructure/Persistence/CircuitAwareExecutionStrategy.cs @@ -74,9 +74,9 @@ public sealed class CircuitAwareExecutionStrategy : NpgsqlRetryingExecutionStrat DatabaseCircuitBreaker.RecordSuccess(); return result; } - catch + catch (Exception ex) { - DatabaseCircuitBreaker.RecordFailure(); + DatabaseCircuitBreaker.RecordFailure(ex); throw; } } @@ -94,9 +94,9 @@ public sealed class CircuitAwareExecutionStrategy : NpgsqlRetryingExecutionStrat DatabaseCircuitBreaker.RecordSuccess(); return result; } - catch + catch (Exception ex) { - DatabaseCircuitBreaker.RecordFailure(); + DatabaseCircuitBreaker.RecordFailure(ex); throw; } } diff --git a/src/LiveRecorder.Infrastructure/Persistence/DatabaseCircuitBreaker.cs b/src/LiveRecorder.Infrastructure/Persistence/DatabaseCircuitBreaker.cs index e46f26f..0d280d6 100644 --- a/src/LiveRecorder.Infrastructure/Persistence/DatabaseCircuitBreaker.cs +++ b/src/LiveRecorder.Infrastructure/Persistence/DatabaseCircuitBreaker.cs @@ -78,14 +78,27 @@ public static class DatabaseCircuitBreaker } /// Record a failed database operation. - public static void RecordFailure() + public static void RecordFailure(Exception? exception = null) { lock (Lock) { + // Rejected requests are not new database failures. Counting them + // causes the failure counter and logs to grow without bound. + if (exception is DatabaseCircuitOpenException) + { + return; + } + + var now = DateTimeOffset.UtcNow; + if (_consecutiveFailures >= FailureThreshold && now - _openedAt <= BreakDuration) + { + return; + } + _consecutiveFailures++; if (_consecutiveFailures >= FailureThreshold) { - _openedAt = DateTimeOffset.UtcNow; + _openedAt = now; } } } diff --git a/src/LiveRecorder.Infrastructure/Services/DriveInfoStorageCapacityProvider.cs b/src/LiveRecorder.Infrastructure/Services/DriveInfoStorageCapacityProvider.cs index abe159a..deb678c 100644 --- a/src/LiveRecorder.Infrastructure/Services/DriveInfoStorageCapacityProvider.cs +++ b/src/LiveRecorder.Infrastructure/Services/DriveInfoStorageCapacityProvider.cs @@ -19,8 +19,10 @@ public sealed class DriveInfoStorageCapacityProvider : IStorageCapacityProvider drive.AvailableFreeSpace)); } } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or OverflowException or ArgumentException or NotSupportedException) { + // Ignore malformed FUSE/cloud mount capacities and continue + // looking for the volume that contains the output path. } } diff --git a/src/LiveRecorder.WebApi/Middleware/ExceptionHandlingMiddleware.cs b/src/LiveRecorder.WebApi/Middleware/ExceptionHandlingMiddleware.cs index 20d3a2f..8f72d9f 100644 --- a/src/LiveRecorder.WebApi/Middleware/ExceptionHandlingMiddleware.cs +++ b/src/LiveRecorder.WebApi/Middleware/ExceptionHandlingMiddleware.cs @@ -21,10 +21,12 @@ public sealed class ExceptionHandlingMiddleware { await _next(context); } - catch (DatabaseCircuitOpenException ex) + catch (DatabaseCircuitOpenException) { // Circuit is open — fast-fail with 503 - _logger.LogWarning(ex, "Database circuit breaker is open, returning 503"); + _logger.LogDebug( + "Database circuit breaker is open, returning 503. Failures={Failures}", + DatabaseCircuitBreaker.ConsecutiveFailures); context.Response.StatusCode = StatusCodes.Status503ServiceUnavailable; await context.Response.WriteAsJsonAsync(new { diff --git a/tests/LiveRecorder.Tests/DatabaseCircuitBreakerTests.cs b/tests/LiveRecorder.Tests/DatabaseCircuitBreakerTests.cs index a79bb31..b57c1fa 100644 --- a/tests/LiveRecorder.Tests/DatabaseCircuitBreakerTests.cs +++ b/tests/LiveRecorder.Tests/DatabaseCircuitBreakerTests.cs @@ -59,6 +59,23 @@ public sealed class DatabaseCircuitBreakerTests Assert.Equal(0, DatabaseCircuitBreaker.ConsecutiveFailures); } + [Fact] + public void RecordFailure_while_circuit_is_open_does_not_grow_failure_counter() + { + for (var i = 0; i < FailureThreshold; i++) + { + DatabaseCircuitBreaker.RecordFailure(); + } + + Assert.True(DatabaseCircuitBreaker.IsOpen); + var failures = DatabaseCircuitBreaker.ConsecutiveFailures; + + DatabaseCircuitBreaker.RecordFailure(); + DatabaseCircuitBreaker.RecordFailure(new DatabaseCircuitOpenException("already open")); + + Assert.Equal(failures, DatabaseCircuitBreaker.ConsecutiveFailures); + } + [Fact] public void IsOpen_after_break_duration_transitions_to_half_open() {