fix: prevent database circuit failure storm
This commit is contained in:
@@ -74,9 +74,9 @@ public sealed class CircuitAwareExecutionStrategy : NpgsqlRetryingExecutionStrat
|
||||
DatabaseCircuitBreaker.RecordSuccess();
|
||||
return result;
|
||||
}
|
||||
catch
|
||||
catch (Exception ex)
|
||||
{
|
||||
DatabaseCircuitBreaker.RecordFailure();
|
||||
DatabaseCircuitBreaker.RecordFailure(ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -94,9 +94,9 @@ public sealed class CircuitAwareExecutionStrategy : NpgsqlRetryingExecutionStrat
|
||||
DatabaseCircuitBreaker.RecordSuccess();
|
||||
return result;
|
||||
}
|
||||
catch
|
||||
catch (Exception ex)
|
||||
{
|
||||
DatabaseCircuitBreaker.RecordFailure();
|
||||
DatabaseCircuitBreaker.RecordFailure(ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,14 +78,27 @@ public static class DatabaseCircuitBreaker
|
||||
}
|
||||
|
||||
/// <summary>Record a failed database operation.</summary>
|
||||
public static void RecordFailure()
|
||||
public static void RecordFailure(Exception? exception = null)
|
||||
{
|
||||
lock (Lock)
|
||||
{
|
||||
// Rejected requests are not new database failures. Counting them
|
||||
// causes the failure counter and logs to grow without bound.
|
||||
if (exception is DatabaseCircuitOpenException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
if (_consecutiveFailures >= FailureThreshold && now - _openedAt <= BreakDuration)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_consecutiveFailures++;
|
||||
if (_consecutiveFailures >= FailureThreshold)
|
||||
{
|
||||
_openedAt = DateTimeOffset.UtcNow;
|
||||
_openedAt = now;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,8 +19,10 @@ public sealed class DriveInfoStorageCapacityProvider : IStorageCapacityProvider
|
||||
drive.AvailableFreeSpace));
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or OverflowException or ArgumentException or NotSupportedException)
|
||||
{
|
||||
// Ignore malformed FUSE/cloud mount capacities and continue
|
||||
// looking for the volume that contains the output path.
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -21,10 +21,12 @@ public sealed class ExceptionHandlingMiddleware
|
||||
{
|
||||
await _next(context);
|
||||
}
|
||||
catch (DatabaseCircuitOpenException ex)
|
||||
catch (DatabaseCircuitOpenException)
|
||||
{
|
||||
// Circuit is open — fast-fail with 503
|
||||
_logger.LogWarning(ex, "Database circuit breaker is open, returning 503");
|
||||
_logger.LogDebug(
|
||||
"Database circuit breaker is open, returning 503. Failures={Failures}",
|
||||
DatabaseCircuitBreaker.ConsecutiveFailures);
|
||||
context.Response.StatusCode = StatusCodes.Status503ServiceUnavailable;
|
||||
await context.Response.WriteAsJsonAsync(new
|
||||
{
|
||||
|
||||
@@ -59,6 +59,23 @@ public sealed class DatabaseCircuitBreakerTests
|
||||
Assert.Equal(0, DatabaseCircuitBreaker.ConsecutiveFailures);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordFailure_while_circuit_is_open_does_not_grow_failure_counter()
|
||||
{
|
||||
for (var i = 0; i < FailureThreshold; i++)
|
||||
{
|
||||
DatabaseCircuitBreaker.RecordFailure();
|
||||
}
|
||||
|
||||
Assert.True(DatabaseCircuitBreaker.IsOpen);
|
||||
var failures = DatabaseCircuitBreaker.ConsecutiveFailures;
|
||||
|
||||
DatabaseCircuitBreaker.RecordFailure();
|
||||
DatabaseCircuitBreaker.RecordFailure(new DatabaseCircuitOpenException("already open"));
|
||||
|
||||
Assert.Equal(failures, DatabaseCircuitBreaker.ConsecutiveFailures);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IsOpen_after_break_duration_transitions_to_half_open()
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user