Introduce a process-wide DatabaseCircuitBreaker that fails fast when Postgres is unavailable (e.g. disk full) instead of letting every request burn doomed EF Core retries. CircuitAwareExecutionStrategy derives from NpgsqlRetryingExecutionStrategy and records success/failure around the public Execute/ExecuteAsync seams; background workers skip work and back off while the circuit is open; the exception middleware maps an open circuit (and other DB outages) to 503. Adds /health (liveness) and /health/ready (readiness, reporting circuit state), plus unit tests for the open/half-open transitions and non-transient SQLSTATE detection. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
123 lines
4.1 KiB
C#
123 lines
4.1 KiB
C#
using System.Reflection;
|
|
using LiveRecorder.Infrastructure.Persistence;
|
|
using Npgsql;
|
|
|
|
namespace LiveRecorder.Tests;
|
|
|
|
public sealed class DatabaseCircuitBreakerTests
|
|
{
|
|
// Mirrors the private FailureThreshold in DatabaseCircuitBreaker so the assertions
|
|
// read intentionally. Keep in sync if the production threshold changes.
|
|
private const int FailureThreshold = 5;
|
|
|
|
public DatabaseCircuitBreakerTests()
|
|
{
|
|
// The breaker is a process-wide static, so reset it to a known closed state
|
|
// before each test to keep cases independent.
|
|
DatabaseCircuitBreaker.RecordSuccess();
|
|
SetOpenedAt(DateTimeOffset.MinValue);
|
|
}
|
|
|
|
[Fact]
|
|
public void RecordFailure_below_threshold_keeps_circuit_closed()
|
|
{
|
|
for (var i = 0; i < FailureThreshold - 1; i++)
|
|
{
|
|
DatabaseCircuitBreaker.RecordFailure();
|
|
}
|
|
|
|
Assert.False(DatabaseCircuitBreaker.IsOpen);
|
|
Assert.Equal(FailureThreshold - 1, DatabaseCircuitBreaker.ConsecutiveFailures);
|
|
}
|
|
|
|
[Fact]
|
|
public void RecordFailure_at_threshold_opens_circuit()
|
|
{
|
|
for (var i = 0; i < FailureThreshold; i++)
|
|
{
|
|
DatabaseCircuitBreaker.RecordFailure();
|
|
}
|
|
|
|
Assert.True(DatabaseCircuitBreaker.IsOpen);
|
|
Assert.Equal(FailureThreshold, DatabaseCircuitBreaker.ConsecutiveFailures);
|
|
Assert.NotEqual(DateTimeOffset.MinValue, DatabaseCircuitBreaker.OpenedAt);
|
|
}
|
|
|
|
[Fact]
|
|
public void RecordSuccess_closes_circuit_and_resets_failures()
|
|
{
|
|
for (var i = 0; i < FailureThreshold; i++)
|
|
{
|
|
DatabaseCircuitBreaker.RecordFailure();
|
|
}
|
|
|
|
Assert.True(DatabaseCircuitBreaker.IsOpen);
|
|
|
|
DatabaseCircuitBreaker.RecordSuccess();
|
|
|
|
Assert.False(DatabaseCircuitBreaker.IsOpen);
|
|
Assert.Equal(0, DatabaseCircuitBreaker.ConsecutiveFailures);
|
|
}
|
|
|
|
[Fact]
|
|
public void IsOpen_after_break_duration_transitions_to_half_open()
|
|
{
|
|
for (var i = 0; i < FailureThreshold; i++)
|
|
{
|
|
DatabaseCircuitBreaker.RecordFailure();
|
|
}
|
|
|
|
Assert.True(DatabaseCircuitBreaker.IsOpen);
|
|
|
|
// The breaker reads DateTimeOffset.UtcNow directly (no injectable clock), so move
|
|
// the recorded open time past the 30s break window to simulate it elapsing.
|
|
SetOpenedAt(DateTimeOffset.UtcNow - TimeSpan.FromSeconds(31));
|
|
|
|
// The first read past the window half-opens: it permits one probe and drops the
|
|
// failure count to one below the threshold.
|
|
Assert.False(DatabaseCircuitBreaker.IsOpen);
|
|
Assert.Equal(FailureThreshold - 1, DatabaseCircuitBreaker.ConsecutiveFailures);
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData("53100")] // disk_full
|
|
[InlineData("53300")] // too_many_connections
|
|
[InlineData("28P01")] // invalid_password
|
|
public void IsNonTransient_returns_true_for_known_fatal_sql_states(string sqlState)
|
|
{
|
|
var exception = new PostgresException("fatal", "FATAL", "FATAL", sqlState);
|
|
|
|
Assert.True(DatabaseCircuitBreaker.IsNonTransient(exception));
|
|
}
|
|
|
|
[Fact]
|
|
public void IsNonTransient_returns_false_for_transient_sql_state()
|
|
{
|
|
// 40001 = serialization_failure, which is retryable and not in the fatal set.
|
|
var exception = new PostgresException("retry me", "ERROR", "ERROR", "40001");
|
|
|
|
Assert.False(DatabaseCircuitBreaker.IsNonTransient(exception));
|
|
}
|
|
|
|
[Fact]
|
|
public void IsNonTransient_unwraps_inner_exceptions()
|
|
{
|
|
var inner = new PostgresException("disk full", "FATAL", "FATAL", "53100");
|
|
var wrapper = new InvalidOperationException("save failed", inner);
|
|
|
|
Assert.True(DatabaseCircuitBreaker.IsNonTransient(wrapper));
|
|
}
|
|
|
|
[Fact]
|
|
public void IsNonTransient_returns_false_for_null_and_non_postgres_exceptions()
|
|
{
|
|
Assert.False(DatabaseCircuitBreaker.IsNonTransient(null));
|
|
Assert.False(DatabaseCircuitBreaker.IsNonTransient(new InvalidOperationException("boom")));
|
|
}
|
|
|
|
private static void SetOpenedAt(DateTimeOffset value) =>
|
|
typeof(DatabaseCircuitBreaker)
|
|
.GetField("_openedAt", BindingFlags.NonPublic | BindingFlags.Static)!
|
|
.SetValue(null, value);
|
|
}
|