feat: add database circuit breaker and health-readiness endpoint

Introduce a process-wide DatabaseCircuitBreaker that fails fast when Postgres is unavailable (e.g. disk full) instead of letting every request burn doomed EF Core retries. CircuitAwareExecutionStrategy derives from NpgsqlRetryingExecutionStrategy and records success/failure around the public Execute/ExecuteAsync seams; background workers skip work and back off while the circuit is open; the exception middleware maps an open circuit (and other DB outages) to 503. Adds /health (liveness) and /health/ready (readiness, reporting circuit state), plus unit tests for the open/half-open transitions and non-transient SQLSTATE detection.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

src/LiveRecorder.Infrastructure/Services/CleanupOperationBackgroundService.cs

@@ -1,3 +1,4 @@
+using LiveRecorder.Infrastructure.Persistence;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
@@ -7,6 +8,8 @@ namespace LiveRecorder.Infrastructure.Services;
 public sealed class CleanupOperationBackgroundService : BackgroundService
 {
     private static readonly TimeSpan IdleDelay = TimeSpan.FromSeconds(2);
+    private static readonly TimeSpan ErrorBaseDelay = TimeSpan.FromSeconds(5);
+    private static readonly TimeSpan ErrorMaxDelay = TimeSpan.FromMinutes(5);
     private readonly IServiceScopeFactory _serviceScopeFactory;
     private readonly ILogger<CleanupOperationBackgroundService> _logger;
 
@@ -20,6 +23,7 @@ public sealed class CleanupOperationBackgroundService : BackgroundService
 
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
+        // Startup: requeue interrupted operations
         try
         {
             using var startupScope = _serviceScopeFactory.CreateScope();
@@ -31,17 +35,36 @@ public sealed class CleanupOperationBackgroundService : BackgroundService
             _logger.LogWarning(ex, "Failed to requeue interrupted cleanup operations at startup");
         }
 
+        var consecutiveErrors = 0;
+
         while (!stoppingToken.IsCancellationRequested)
         {
             try
             {
+                // Skip processing if the circuit is open — don't waste resources
+                if (DatabaseCircuitBreaker.IsOpen)
+                {
+                    consecutiveErrors = await DelayWithBackoff(ErrorBaseDelay, ErrorMaxDelay, consecutiveErrors, stoppingToken);
+                    continue;
+                }
+
                 using var scope = _serviceScopeFactory.CreateScope();
                 var coordinator = scope.ServiceProvider.GetRequiredService<CleanupOperationCoordinator>();
                 var processed = await coordinator.ProcessNextQueuedOperationAsync(stoppingToken);
+
                 if (processed)
                 {
+                    consecutiveErrors = 0;
                     continue;
                 }
+
+                // No queued operations — idle delay
+                consecutiveErrors = 0;
+                await Task.Delay(IdleDelay, stoppingToken);
+            }
+            catch (DatabaseCircuitOpenException)
+            {
+                consecutiveErrors = await DelayWithBackoff(ErrorBaseDelay, ErrorMaxDelay, consecutiveErrors, stoppingToken);
             }
             catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
             {
@@ -49,17 +72,47 @@ public sealed class CleanupOperationBackgroundService : BackgroundService
             }
             catch (Exception ex)
             {
-                _logger.LogError(ex, "Cleanup operation background worker failed");
-            }
+                var isDatabaseError = DatabaseCircuitBreaker.IsNonTransient(ex) ||
+                                      ex is Npgsql.NpgsqlException ||
+                                      ex is Microsoft.EntityFrameworkCore.DbUpdateException;
 
-            try
-            {
-                await Task.Delay(IdleDelay, stoppingToken);
-            }
-            catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
-            {
-                break;
+                if (isDatabaseError)
+                {
+                    DatabaseCircuitBreaker.RecordFailure();
+                    _logger.LogWarning(ex,
+                        "Cleanup background worker: database error (#{ErrorCount}). Circuit state: Open={IsOpen}, Failures={Failures}",
+                        consecutiveErrors + 1,
+                        DatabaseCircuitBreaker.IsOpen,
+                        DatabaseCircuitBreaker.ConsecutiveFailures);
+                }
+                else
+                {
+                    _logger.LogError(ex, "Cleanup operation background worker failed");
+                }
+
+                consecutiveErrors = await DelayWithBackoff(ErrorBaseDelay, ErrorMaxDelay, consecutiveErrors, stoppingToken);
             }
         }
     }
+
+    private static async Task<int> DelayWithBackoff(
+        TimeSpan baseDelay, TimeSpan maxDelay, int errorCount, CancellationToken cancellationToken)
+    {
+        errorCount++;
+        // Exponential backoff: 5s, 10s, 20s, 40s, 80s, 160s, capping at 5min
+        var factor = Math.Pow(2, Math.Min(errorCount - 1, 6));
+        var delay = TimeSpan.FromMilliseconds(
+            Math.Min(baseDelay.TotalMilliseconds * factor, maxDelay.TotalMilliseconds));
+
+        try
+        {
+            await Task.Delay(delay, cancellationToken);
+        }
+        catch (OperationCanceledException)
+        {
+            // Swallow — loop will exit on next iteration
+        }
+
+        return errorCount;
+    }
 }

src/LiveRecorder.Infrastructure/Services/LiveRoomPollingBackgroundService.cs

@@ -146,8 +146,20 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService, ILiveR
             {
                 break;
             }
+            catch (DatabaseCircuitOpenException)
+            {
+                // Circuit is open — skip this iteration and wait
+                _logger.LogDebug("Polling loop skipped: database circuit breaker is open");
+                delay = TimeSpan.FromSeconds(30);
+            }
             catch (Exception ex)
             {
+                // Record database failures to the circuit breaker
+                if (IsTransientDatabaseException(ex) || DatabaseCircuitBreaker.IsNonTransient(ex))
+                {
+                    DatabaseCircuitBreaker.RecordFailure();
+                }
+
                 _logger.LogError(ex, "Background live room polling failed");
 
                 try

src/LiveRecorder.Infrastructure/Services/RetentionCleanupBackgroundService.cs

@@ -1,3 +1,4 @@
+using LiveRecorder.Infrastructure.Persistence;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
@@ -7,7 +8,8 @@ namespace LiveRecorder.Infrastructure.Services;
 public sealed class RetentionCleanupBackgroundService : BackgroundService
 {
     private static readonly TimeSpan CleanupInterval = TimeSpan.FromHours(24);
-
+    private static readonly TimeSpan ErrorBaseDelay = TimeSpan.FromSeconds(10);
+    private static readonly TimeSpan ErrorMaxDelay = TimeSpan.FromMinutes(5);
     private readonly IServiceScopeFactory _serviceScopeFactory;
     private readonly ILogger<RetentionCleanupBackgroundService> _logger;
 
@@ -21,13 +23,31 @@ public sealed class RetentionCleanupBackgroundService : BackgroundService
 
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
+        var consecutiveErrors = 0;
+
         while (!stoppingToken.IsCancellationRequested)
         {
             try
             {
+                // Skip if circuit is already open
+                if (DatabaseCircuitBreaker.IsOpen)
+                {
+                    _logger.LogDebug("Retention cleanup skipped: database circuit breaker is open");
+                    await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken);
+                    continue;
+                }
+
                 using var scope = _serviceScopeFactory.CreateScope();
                 var cleanupService = scope.ServiceProvider.GetRequiredService<RetentionCleanupService>();
                 await cleanupService.TryEnqueueAsync(ignoreEnabledSetting: false, cancellationToken: stoppingToken);
+
+                consecutiveErrors = 0;
+            }
+            catch (DatabaseCircuitOpenException)
+            {
+                // Silent — circuit is already logged
+                await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken);
+                continue;
             }
             catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
             {
@@ -35,7 +55,38 @@ public sealed class RetentionCleanupBackgroundService : BackgroundService
             }
             catch (Exception ex)
             {
-                _logger.LogError(ex, "Retention cleanup background task failed");
+                var isDatabaseError = DatabaseCircuitBreaker.IsNonTransient(ex) ||
+                                      ex is Npgsql.NpgsqlException ||
+                                      ex is Microsoft.EntityFrameworkCore.DbUpdateException;
+
+                consecutiveErrors++;
+                var delay = TimeSpan.FromMilliseconds(
+                    Math.Min(ErrorBaseDelay.TotalMilliseconds * Math.Pow(2, Math.Min(consecutiveErrors - 1, 6)),
+                             ErrorMaxDelay.TotalMilliseconds));
+
+                if (isDatabaseError)
+                {
+                    DatabaseCircuitBreaker.RecordFailure();
+                    _logger.LogWarning(ex,
+                        "Retention cleanup: database error (#{ErrorCount}). Circuit: Open={IsOpen}, Failures={Failures}",
+                        consecutiveErrors,
+                        DatabaseCircuitBreaker.IsOpen,
+                        DatabaseCircuitBreaker.ConsecutiveFailures);
+                }
+                else
+                {
+                    _logger.LogError(ex, "Retention cleanup background task failed");
+                }
+
+                try
+                {
+                    await Task.Delay(delay, stoppingToken);
+                }
+                catch (OperationCanceledException)
+                {
+                    break;
+                }
+                continue;
             }
 
             try