fix: prevent database circuit failure storm
This commit is contained in:
@@ -74,9 +74,9 @@ public sealed class CircuitAwareExecutionStrategy : NpgsqlRetryingExecutionStrat
|
|||||||
DatabaseCircuitBreaker.RecordSuccess();
|
DatabaseCircuitBreaker.RecordSuccess();
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
catch
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
DatabaseCircuitBreaker.RecordFailure();
|
DatabaseCircuitBreaker.RecordFailure(ex);
|
||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -94,9 +94,9 @@ public sealed class CircuitAwareExecutionStrategy : NpgsqlRetryingExecutionStrat
|
|||||||
DatabaseCircuitBreaker.RecordSuccess();
|
DatabaseCircuitBreaker.RecordSuccess();
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
catch
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
DatabaseCircuitBreaker.RecordFailure();
|
DatabaseCircuitBreaker.RecordFailure(ex);
|
||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -78,14 +78,27 @@ public static class DatabaseCircuitBreaker
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>Record a failed database operation.</summary>
|
/// <summary>Record a failed database operation.</summary>
|
||||||
public static void RecordFailure()
|
public static void RecordFailure(Exception? exception = null)
|
||||||
{
|
{
|
||||||
lock (Lock)
|
lock (Lock)
|
||||||
{
|
{
|
||||||
|
// Rejected requests are not new database failures. Counting them
|
||||||
|
// causes the failure counter and logs to grow without bound.
|
||||||
|
if (exception is DatabaseCircuitOpenException)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var now = DateTimeOffset.UtcNow;
|
||||||
|
if (_consecutiveFailures >= FailureThreshold && now - _openedAt <= BreakDuration)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
_consecutiveFailures++;
|
_consecutiveFailures++;
|
||||||
if (_consecutiveFailures >= FailureThreshold)
|
if (_consecutiveFailures >= FailureThreshold)
|
||||||
{
|
{
|
||||||
_openedAt = DateTimeOffset.UtcNow;
|
_openedAt = now;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,8 +19,10 @@ public sealed class DriveInfoStorageCapacityProvider : IStorageCapacityProvider
|
|||||||
drive.AvailableFreeSpace));
|
drive.AvailableFreeSpace));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or OverflowException or ArgumentException or NotSupportedException)
|
||||||
{
|
{
|
||||||
|
// Ignore malformed FUSE/cloud mount capacities and continue
|
||||||
|
// looking for the volume that contains the output path.
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -21,10 +21,12 @@ public sealed class ExceptionHandlingMiddleware
|
|||||||
{
|
{
|
||||||
await _next(context);
|
await _next(context);
|
||||||
}
|
}
|
||||||
catch (DatabaseCircuitOpenException ex)
|
catch (DatabaseCircuitOpenException)
|
||||||
{
|
{
|
||||||
// Circuit is open — fast-fail with 503
|
// Circuit is open — fast-fail with 503
|
||||||
_logger.LogWarning(ex, "Database circuit breaker is open, returning 503");
|
_logger.LogDebug(
|
||||||
|
"Database circuit breaker is open, returning 503. Failures={Failures}",
|
||||||
|
DatabaseCircuitBreaker.ConsecutiveFailures);
|
||||||
context.Response.StatusCode = StatusCodes.Status503ServiceUnavailable;
|
context.Response.StatusCode = StatusCodes.Status503ServiceUnavailable;
|
||||||
await context.Response.WriteAsJsonAsync(new
|
await context.Response.WriteAsJsonAsync(new
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -59,6 +59,23 @@ public sealed class DatabaseCircuitBreakerTests
|
|||||||
Assert.Equal(0, DatabaseCircuitBreaker.ConsecutiveFailures);
|
Assert.Equal(0, DatabaseCircuitBreaker.ConsecutiveFailures);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void RecordFailure_while_circuit_is_open_does_not_grow_failure_counter()
|
||||||
|
{
|
||||||
|
for (var i = 0; i < FailureThreshold; i++)
|
||||||
|
{
|
||||||
|
DatabaseCircuitBreaker.RecordFailure();
|
||||||
|
}
|
||||||
|
|
||||||
|
Assert.True(DatabaseCircuitBreaker.IsOpen);
|
||||||
|
var failures = DatabaseCircuitBreaker.ConsecutiveFailures;
|
||||||
|
|
||||||
|
DatabaseCircuitBreaker.RecordFailure();
|
||||||
|
DatabaseCircuitBreaker.RecordFailure(new DatabaseCircuitOpenException("already open"));
|
||||||
|
|
||||||
|
Assert.Equal(failures, DatabaseCircuitBreaker.ConsecutiveFailures);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public void IsOpen_after_break_duration_transitions_to_half_open()
|
public void IsOpen_after_break_duration_transitions_to_half_open()
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user