feat: add database circuit breaker and health-readiness endpoint

Introduce a process-wide DatabaseCircuitBreaker that fails fast when Postgres is unavailable (e.g. disk full) instead of letting every request burn doomed EF Core retries. CircuitAwareExecutionStrategy derives from NpgsqlRetryingExecutionStrategy and records success/failure around the public Execute/ExecuteAsync seams; background workers skip work and back off while the circuit is open; the exception middleware maps an open circuit (and other DB outages) to 503. Adds /health (liveness) and /health/ready (readiness, reporting circuit state), plus unit tests for the open/half-open transitions and non-transient SQLSTATE detection.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

tests/LiveRecorder.Tests/DatabaseCircuitBreakerTests.cs

@@ -0,0 +1,122 @@
+using System.Reflection;
+using LiveRecorder.Infrastructure.Persistence;
+using Npgsql;
+
+namespace LiveRecorder.Tests;
+
+public sealed class DatabaseCircuitBreakerTests
+{
+    // Mirrors the private FailureThreshold in DatabaseCircuitBreaker so the assertions
+    // read intentionally. Keep in sync if the production threshold changes.
+    private const int FailureThreshold = 5;
+
+    public DatabaseCircuitBreakerTests()
+    {
+        // The breaker is a process-wide static, so reset it to a known closed state
+        // before each test to keep cases independent.
+        DatabaseCircuitBreaker.RecordSuccess();
+        SetOpenedAt(DateTimeOffset.MinValue);
+    }
+
+    [Fact]
+    public void RecordFailure_below_threshold_keeps_circuit_closed()
+    {
+        for (var i = 0; i < FailureThreshold - 1; i++)
+        {
+            DatabaseCircuitBreaker.RecordFailure();
+        }
+
+        Assert.False(DatabaseCircuitBreaker.IsOpen);
+        Assert.Equal(FailureThreshold - 1, DatabaseCircuitBreaker.ConsecutiveFailures);
+    }
+
+    [Fact]
+    public void RecordFailure_at_threshold_opens_circuit()
+    {
+        for (var i = 0; i < FailureThreshold; i++)
+        {
+            DatabaseCircuitBreaker.RecordFailure();
+        }
+
+        Assert.True(DatabaseCircuitBreaker.IsOpen);
+        Assert.Equal(FailureThreshold, DatabaseCircuitBreaker.ConsecutiveFailures);
+        Assert.NotEqual(DateTimeOffset.MinValue, DatabaseCircuitBreaker.OpenedAt);
+    }
+
+    [Fact]
+    public void RecordSuccess_closes_circuit_and_resets_failures()
+    {
+        for (var i = 0; i < FailureThreshold; i++)
+        {
+            DatabaseCircuitBreaker.RecordFailure();
+        }
+
+        Assert.True(DatabaseCircuitBreaker.IsOpen);
+
+        DatabaseCircuitBreaker.RecordSuccess();
+
+        Assert.False(DatabaseCircuitBreaker.IsOpen);
+        Assert.Equal(0, DatabaseCircuitBreaker.ConsecutiveFailures);
+    }
+
+    [Fact]
+    public void IsOpen_after_break_duration_transitions_to_half_open()
+    {
+        for (var i = 0; i < FailureThreshold; i++)
+        {
+            DatabaseCircuitBreaker.RecordFailure();
+        }
+
+        Assert.True(DatabaseCircuitBreaker.IsOpen);
+
+        // The breaker reads DateTimeOffset.UtcNow directly (no injectable clock), so move
+        // the recorded open time past the 30s break window to simulate it elapsing.
+        SetOpenedAt(DateTimeOffset.UtcNow - TimeSpan.FromSeconds(31));
+
+        // The first read past the window half-opens: it permits one probe and drops the
+        // failure count to one below the threshold.
+        Assert.False(DatabaseCircuitBreaker.IsOpen);
+        Assert.Equal(FailureThreshold - 1, DatabaseCircuitBreaker.ConsecutiveFailures);
+    }
+
+    [Theory]
+    [InlineData("53100")] // disk_full
+    [InlineData("53300")] // too_many_connections
+    [InlineData("28P01")] // invalid_password
+    public void IsNonTransient_returns_true_for_known_fatal_sql_states(string sqlState)
+    {
+        var exception = new PostgresException("fatal", "FATAL", "FATAL", sqlState);
+
+        Assert.True(DatabaseCircuitBreaker.IsNonTransient(exception));
+    }
+
+    [Fact]
+    public void IsNonTransient_returns_false_for_transient_sql_state()
+    {
+        // 40001 = serialization_failure, which is retryable and not in the fatal set.
+        var exception = new PostgresException("retry me", "ERROR", "ERROR", "40001");
+
+        Assert.False(DatabaseCircuitBreaker.IsNonTransient(exception));
+    }
+
+    [Fact]
+    public void IsNonTransient_unwraps_inner_exceptions()
+    {
+        var inner = new PostgresException("disk full", "FATAL", "FATAL", "53100");
+        var wrapper = new InvalidOperationException("save failed", inner);
+
+        Assert.True(DatabaseCircuitBreaker.IsNonTransient(wrapper));
+    }
+
+    [Fact]
+    public void IsNonTransient_returns_false_for_null_and_non_postgres_exceptions()
+    {
+        Assert.False(DatabaseCircuitBreaker.IsNonTransient(null));
+        Assert.False(DatabaseCircuitBreaker.IsNonTransient(new InvalidOperationException("boom")));
+    }
+
+    private static void SetOpenedAt(DateTimeOffset value) =>
+        typeof(DatabaseCircuitBreaker)
+            .GetField("_openedAt", BindingFlags.NonPublic | BindingFlags.Static)!
+            .SetValue(null, value);
+}

tests/LiveRecorder.Tests/LiveRecorder.Tests.csproj

@@ -20,6 +20,7 @@
   <ItemGroup>
     <ProjectReference Include="..\..\src\LiveRecorder.Application\LiveRecorder.Application.csproj" />
     <ProjectReference Include="..\..\src\LiveRecorder.Domain\LiveRecorder.Domain.csproj" />
+    <ProjectReference Include="..\..\src\LiveRecorder.Infrastructure\LiveRecorder.Infrastructure.csproj" />
   </ItemGroup>
 
 </Project>