fix: prevent database circuit failure storm

This commit is contained in:
2026-09-02 20:41:51 +08:00
parent 60d1930447
commit c056a3e741
5 changed files with 43 additions and 9 deletions
@@ -74,9 +74,9 @@ public sealed class CircuitAwareExecutionStrategy : NpgsqlRetryingExecutionStrat
DatabaseCircuitBreaker.RecordSuccess();
return result;
}
catch
catch (Exception ex)
{
DatabaseCircuitBreaker.RecordFailure();
DatabaseCircuitBreaker.RecordFailure(ex);
throw;
}
}
@@ -94,9 +94,9 @@ public sealed class CircuitAwareExecutionStrategy : NpgsqlRetryingExecutionStrat
DatabaseCircuitBreaker.RecordSuccess();
return result;
}
catch
catch (Exception ex)
{
DatabaseCircuitBreaker.RecordFailure();
DatabaseCircuitBreaker.RecordFailure(ex);
throw;
}
}
@@ -78,14 +78,27 @@ public static class DatabaseCircuitBreaker
}
/// <summary>Record a failed database operation.</summary>
public static void RecordFailure()
public static void RecordFailure(Exception? exception = null)
{
lock (Lock)
{
// Rejected requests are not new database failures. Counting them
// causes the failure counter and logs to grow without bound.
if (exception is DatabaseCircuitOpenException)
{
return;
}
var now = DateTimeOffset.UtcNow;
if (_consecutiveFailures >= FailureThreshold && now - _openedAt <= BreakDuration)
{
return;
}
_consecutiveFailures++;
if (_consecutiveFailures >= FailureThreshold)
{
_openedAt = DateTimeOffset.UtcNow;
_openedAt = now;
}
}
}
@@ -19,8 +19,10 @@ public sealed class DriveInfoStorageCapacityProvider : IStorageCapacityProvider
drive.AvailableFreeSpace));
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or OverflowException or ArgumentException or NotSupportedException)
{
// Ignore malformed FUSE/cloud mount capacities and continue
// looking for the volume that contains the output path.
}
}
@@ -21,10 +21,12 @@ public sealed class ExceptionHandlingMiddleware
{
await _next(context);
}
catch (DatabaseCircuitOpenException ex)
catch (DatabaseCircuitOpenException)
{
// Circuit is open — fast-fail with 503
_logger.LogWarning(ex, "Database circuit breaker is open, returning 503");
_logger.LogDebug(
"Database circuit breaker is open, returning 503. Failures={Failures}",
DatabaseCircuitBreaker.ConsecutiveFailures);
context.Response.StatusCode = StatusCodes.Status503ServiceUnavailable;
await context.Response.WriteAsJsonAsync(new
{
@@ -59,6 +59,23 @@ public sealed class DatabaseCircuitBreakerTests
Assert.Equal(0, DatabaseCircuitBreaker.ConsecutiveFailures);
}
[Fact]
public void RecordFailure_while_circuit_is_open_does_not_grow_failure_counter()
{
for (var i = 0; i < FailureThreshold; i++)
{
DatabaseCircuitBreaker.RecordFailure();
}
Assert.True(DatabaseCircuitBreaker.IsOpen);
var failures = DatabaseCircuitBreaker.ConsecutiveFailures;
DatabaseCircuitBreaker.RecordFailure();
DatabaseCircuitBreaker.RecordFailure(new DatabaseCircuitOpenException("already open"));
Assert.Equal(failures, DatabaseCircuitBreaker.ConsecutiveFailures);
}
[Fact]
public void IsOpen_after_break_duration_transitions_to_half_open()
{