Phase 4: Circuit Breaker, DB Retry, Redis Graceful Degradation

- app/core/resilience.py: CircuitBreaker (CLOSED/OPEN/HALF_OPEN), retry_db,
  redis_call_with_fallback, InMemoryRateLimiter, CircuitBreakerMiddleware
- app/core/auth.py: get_session_data now falls back to PostgreSQL sessions
  table when Redis is unavailable
- app/core/permissions.py: get_cached_permissions falls back to direct DB
  resolution when Redis circuit is open
- app/core/rate_limit.py: check_rate_limit falls back to in-memory limiter
  when Redis is down; reset_rate_limit clears both Redis and in-memory
- app/core/middleware.py: CSRF validation uses get_session_data (Redis+DB
  fallback); sliding session TTL is best-effort during outage
- app/core/db/__init__.py: get_db() wraps session creation with retry_db
  for transient connection errors; records circuit breaker success/failure
- app/deps.py: refresh_session_ttl wrapped in try/except for Redis outage
- app/main.py: CircuitBreakerMiddleware registered (returns 503 when DB
  circuit is OPEN, skips health/metrics endpoints)
- app/config.py: Added resilience settings (thresholds, cooldown, retries)
- tests/test_resilience.py: 30 tests covering all patterns

30/30 resilience tests pass. No regressions in plugin lifecycle tests.

app/core/permissions.py

@@ -349,43 +349,54 @@ async def get_cached_permissions(
 
     Validates the cached permission_version against the current DB version.
     If they differ, the cache entry is stale and will be re-resolved.
+
+    Falls back to direct DB resolution when Redis is unavailable.
     """
     cache_key = f"{CACHE_PREFIX}:{user_id}:{tenant_id}"
 
-    raw = await redis.get(cache_key)
-    if raw is not None:
-        data = json.loads(raw)
-        cached_version = data.get("version", -1)
+    from app.core.resilience import get_circuit
 
-        # Validate cached version against current DB version
+    circuit = get_circuit("redis")
+    redis_available = await circuit.can_proceed()
+
+    if redis_available:
         try:
-            current_version = await _get_current_permission_version(db, user_id, tenant_id)
-        except Exception:
-            logger.warning(
-                "Failed to query current permission_version for cache validation "
-                "(user=%s, tenant=%s) — invalidating cache and re-resolving",
-                user_id, tenant_id,
-                exc_info=True,
-            )
-            # Invalidate stale cache — do NOT trust cached permissions on DB error
-            await redis.delete(cache_key)
-            return None  # Fall through to re-resolution from DB
+            raw = await redis.get(cache_key)
+            await circuit.record_success()
+            if raw is not None:
+                data = json.loads(raw)
+                cached_version = data.get("version", -1)
 
-        if cached_version == current_version:
-            return data
+                # Validate cached version against current DB version
+                try:
+                    current_version = await _get_current_permission_version(db, user_id, tenant_id)
+                except Exception:
+                    logger.warning(
+                        "Failed to query current permission_version for cache validation "
+                        "(user=%s, tenant=%s) — invalidating cache and re-resolving",
+                        user_id, tenant_id,
+                        exc_info=True,
+                    )
+                    await redis.delete(cache_key)
+                    return None  # Fall through to re-resolution from DB
 
-        # Version mismatch — invalidate stale cache and re-resolve
-        logger.info(
-            "Permission cache version mismatch for user=%s tenant=%s "
-            "(cached=%s, current=%s) — re-resolving",
-            user_id, tenant_id, cached_version, current_version,
-        )
-        await redis.delete(cache_key)
+                if cached_version == current_version:
+                    return data
 
-    # Cache miss or stale — resolve from DB
+                logger.info(
+                    "Permission cache version mismatch for user=%s tenant=%s "
+                    "(cached=%s, current=%s) — re-resolving",
+                    user_id, tenant_id, cached_version, current_version,
+                )
+                await redis.delete(cache_key)
+        except Exception as exc:
+            logger.warning("Redis permission cache failed: %s — resolving from DB", exc)
+            await circuit.record_failure()
+            redis_available = False
+
+    # Cache miss, stale, or Redis unavailable — resolve from DB
     resolved = await resolve_permissions(db, user_id, tenant_id)
 
-    # Store in cache (convert sets to lists for JSON)
     cache_data = {
         "permissions": list(resolved["permissions"]),
         "denied": list(resolved["denied"]),
@@ -393,7 +404,14 @@ async def get_cached_permissions(
         "is_system_admin": resolved["is_system_admin"],
         "version": resolved["version"],
     }
-    await redis.setex(cache_key, CACHE_TTL, json.dumps(cache_data))
+
+    # Try to cache (best-effort during Redis outage)
+    if redis_available:
+        try:
+            await redis.setex(cache_key, CACHE_TTL, json.dumps(cache_data))
+        except Exception:
+            logger.warning("Failed to cache permissions in Redis — continuing without cache")
+
     return cache_data