Files
live_recorder/tests/LiveRecorder.Tests/DatabaseCircuitBreakerTests.cs
T
nanxunandClaude Opus 4.8 5196ffa0f0 feat: add database circuit breaker and health-readiness endpoint
Introduce a process-wide DatabaseCircuitBreaker that fails fast when Postgres is unavailable (e.g. disk full) instead of letting every request burn doomed EF Core retries. CircuitAwareExecutionStrategy derives from NpgsqlRetryingExecutionStrategy and records success/failure around the public Execute/ExecuteAsync seams; background workers skip work and back off while the circuit is open; the exception middleware maps an open circuit (and other DB outages) to 503. Adds /health (liveness) and /health/ready (readiness, reporting circuit state), plus unit tests for the open/half-open transitions and non-transient SQLSTATE detection.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-11 17:52:18 +08:00

123 lines
4.1 KiB
C#

using System.Reflection;
using LiveRecorder.Infrastructure.Persistence;
using Npgsql;
namespace LiveRecorder.Tests;
public sealed class DatabaseCircuitBreakerTests
{
// Mirrors the private FailureThreshold in DatabaseCircuitBreaker so the assertions
// read intentionally. Keep in sync if the production threshold changes.
private const int FailureThreshold = 5;
public DatabaseCircuitBreakerTests()
{
// The breaker is a process-wide static, so reset it to a known closed state
// before each test to keep cases independent.
DatabaseCircuitBreaker.RecordSuccess();
SetOpenedAt(DateTimeOffset.MinValue);
}
[Fact]
public void RecordFailure_below_threshold_keeps_circuit_closed()
{
for (var i = 0; i < FailureThreshold - 1; i++)
{
DatabaseCircuitBreaker.RecordFailure();
}
Assert.False(DatabaseCircuitBreaker.IsOpen);
Assert.Equal(FailureThreshold - 1, DatabaseCircuitBreaker.ConsecutiveFailures);
}
[Fact]
public void RecordFailure_at_threshold_opens_circuit()
{
for (var i = 0; i < FailureThreshold; i++)
{
DatabaseCircuitBreaker.RecordFailure();
}
Assert.True(DatabaseCircuitBreaker.IsOpen);
Assert.Equal(FailureThreshold, DatabaseCircuitBreaker.ConsecutiveFailures);
Assert.NotEqual(DateTimeOffset.MinValue, DatabaseCircuitBreaker.OpenedAt);
}
[Fact]
public void RecordSuccess_closes_circuit_and_resets_failures()
{
for (var i = 0; i < FailureThreshold; i++)
{
DatabaseCircuitBreaker.RecordFailure();
}
Assert.True(DatabaseCircuitBreaker.IsOpen);
DatabaseCircuitBreaker.RecordSuccess();
Assert.False(DatabaseCircuitBreaker.IsOpen);
Assert.Equal(0, DatabaseCircuitBreaker.ConsecutiveFailures);
}
[Fact]
public void IsOpen_after_break_duration_transitions_to_half_open()
{
for (var i = 0; i < FailureThreshold; i++)
{
DatabaseCircuitBreaker.RecordFailure();
}
Assert.True(DatabaseCircuitBreaker.IsOpen);
// The breaker reads DateTimeOffset.UtcNow directly (no injectable clock), so move
// the recorded open time past the 30s break window to simulate it elapsing.
SetOpenedAt(DateTimeOffset.UtcNow - TimeSpan.FromSeconds(31));
// The first read past the window half-opens: it permits one probe and drops the
// failure count to one below the threshold.
Assert.False(DatabaseCircuitBreaker.IsOpen);
Assert.Equal(FailureThreshold - 1, DatabaseCircuitBreaker.ConsecutiveFailures);
}
[Theory]
[InlineData("53100")] // disk_full
[InlineData("53300")] // too_many_connections
[InlineData("28P01")] // invalid_password
public void IsNonTransient_returns_true_for_known_fatal_sql_states(string sqlState)
{
var exception = new PostgresException("fatal", "FATAL", "FATAL", sqlState);
Assert.True(DatabaseCircuitBreaker.IsNonTransient(exception));
}
[Fact]
public void IsNonTransient_returns_false_for_transient_sql_state()
{
// 40001 = serialization_failure, which is retryable and not in the fatal set.
var exception = new PostgresException("retry me", "ERROR", "ERROR", "40001");
Assert.False(DatabaseCircuitBreaker.IsNonTransient(exception));
}
[Fact]
public void IsNonTransient_unwraps_inner_exceptions()
{
var inner = new PostgresException("disk full", "FATAL", "FATAL", "53100");
var wrapper = new InvalidOperationException("save failed", inner);
Assert.True(DatabaseCircuitBreaker.IsNonTransient(wrapper));
}
[Fact]
public void IsNonTransient_returns_false_for_null_and_non_postgres_exceptions()
{
Assert.False(DatabaseCircuitBreaker.IsNonTransient(null));
Assert.False(DatabaseCircuitBreaker.IsNonTransient(new InvalidOperationException("boom")));
}
private static void SetOpenedAt(DateTimeOffset value) =>
typeof(DatabaseCircuitBreaker)
.GetField("_openedAt", BindingFlags.NonPublic | BindingFlags.Static)!
.SetValue(null, value);
}