fix: prevent database circuit failure storm

This commit is contained in:
2026-09-02 20:41:51 +08:00
parent 60d1930447
commit c056a3e741
5 changed files with 43 additions and 9 deletions
@@ -74,9 +74,9 @@ public sealed class CircuitAwareExecutionStrategy : NpgsqlRetryingExecutionStrat
DatabaseCircuitBreaker.RecordSuccess(); DatabaseCircuitBreaker.RecordSuccess();
return result; return result;
} }
catch catch (Exception ex)
{ {
DatabaseCircuitBreaker.RecordFailure(); DatabaseCircuitBreaker.RecordFailure(ex);
throw; throw;
} }
} }
@@ -94,9 +94,9 @@ public sealed class CircuitAwareExecutionStrategy : NpgsqlRetryingExecutionStrat
DatabaseCircuitBreaker.RecordSuccess(); DatabaseCircuitBreaker.RecordSuccess();
return result; return result;
} }
catch catch (Exception ex)
{ {
DatabaseCircuitBreaker.RecordFailure(); DatabaseCircuitBreaker.RecordFailure(ex);
throw; throw;
} }
} }
@@ -78,14 +78,27 @@ public static class DatabaseCircuitBreaker
} }
/// <summary>Record a failed database operation.</summary> /// <summary>Record a failed database operation.</summary>
public static void RecordFailure() public static void RecordFailure(Exception? exception = null)
{ {
lock (Lock) lock (Lock)
{ {
// Rejected requests are not new database failures. Counting them
// causes the failure counter and logs to grow without bound.
if (exception is DatabaseCircuitOpenException)
{
return;
}
var now = DateTimeOffset.UtcNow;
if (_consecutiveFailures >= FailureThreshold && now - _openedAt <= BreakDuration)
{
return;
}
_consecutiveFailures++; _consecutiveFailures++;
if (_consecutiveFailures >= FailureThreshold) if (_consecutiveFailures >= FailureThreshold)
{ {
_openedAt = DateTimeOffset.UtcNow; _openedAt = now;
} }
} }
} }
@@ -19,8 +19,10 @@ public sealed class DriveInfoStorageCapacityProvider : IStorageCapacityProvider
drive.AvailableFreeSpace)); drive.AvailableFreeSpace));
} }
} }
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or OverflowException or ArgumentException or NotSupportedException)
{ {
// Ignore malformed FUSE/cloud mount capacities and continue
// looking for the volume that contains the output path.
} }
} }
@@ -21,10 +21,12 @@ public sealed class ExceptionHandlingMiddleware
{ {
await _next(context); await _next(context);
} }
catch (DatabaseCircuitOpenException ex) catch (DatabaseCircuitOpenException)
{ {
// Circuit is open — fast-fail with 503 // Circuit is open — fast-fail with 503
_logger.LogWarning(ex, "Database circuit breaker is open, returning 503"); _logger.LogDebug(
"Database circuit breaker is open, returning 503. Failures={Failures}",
DatabaseCircuitBreaker.ConsecutiveFailures);
context.Response.StatusCode = StatusCodes.Status503ServiceUnavailable; context.Response.StatusCode = StatusCodes.Status503ServiceUnavailable;
await context.Response.WriteAsJsonAsync(new await context.Response.WriteAsJsonAsync(new
{ {
@@ -59,6 +59,23 @@ public sealed class DatabaseCircuitBreakerTests
Assert.Equal(0, DatabaseCircuitBreaker.ConsecutiveFailures); Assert.Equal(0, DatabaseCircuitBreaker.ConsecutiveFailures);
} }
[Fact]
public void RecordFailure_while_circuit_is_open_does_not_grow_failure_counter()
{
for (var i = 0; i < FailureThreshold; i++)
{
DatabaseCircuitBreaker.RecordFailure();
}
Assert.True(DatabaseCircuitBreaker.IsOpen);
var failures = DatabaseCircuitBreaker.ConsecutiveFailures;
DatabaseCircuitBreaker.RecordFailure();
DatabaseCircuitBreaker.RecordFailure(new DatabaseCircuitOpenException("already open"));
Assert.Equal(failures, DatabaseCircuitBreaker.ConsecutiveFailures);
}
[Fact] [Fact]
public void IsOpen_after_break_duration_transitions_to_half_open() public void IsOpen_after_break_duration_transitions_to_half_open()
{ {