Phase 4: Circuit Breaker, DB Retry, Redis Graceful Degradation
- app/core/resilience.py: CircuitBreaker (CLOSED/OPEN/HALF_OPEN), retry_db, redis_call_with_fallback, InMemoryRateLimiter, CircuitBreakerMiddleware - app/core/auth.py: get_session_data now falls back to PostgreSQL sessions table when Redis is unavailable - app/core/permissions.py: get_cached_permissions falls back to direct DB resolution when Redis circuit is open - app/core/rate_limit.py: check_rate_limit falls back to in-memory limiter when Redis is down; reset_rate_limit clears both Redis and in-memory - app/core/middleware.py: CSRF validation uses get_session_data (Redis+DB fallback); sliding session TTL is best-effort during outage - app/core/db/__init__.py: get_db() wraps session creation with retry_db for transient connection errors; records circuit breaker success/failure - app/deps.py: refresh_session_ttl wrapped in try/except for Redis outage - app/main.py: CircuitBreakerMiddleware registered (returns 503 when DB circuit is OPEN, skips health/metrics endpoints) - app/config.py: Added resilience settings (thresholds, cooldown, retries) - tests/test_resilience.py: 30 tests covering all patterns 30/30 resilience tests pass. No regressions in plugin lifecycle tests.
This commit is contained in:
@@ -111,18 +111,17 @@ class CSRFMiddleware(BaseHTTPMiddleware):
|
||||
content={"detail": "No session for CSRF validation", "code": "csrf_no_session"},
|
||||
)
|
||||
|
||||
# Look up CSRF token from Redis session (use singleton)
|
||||
from app.core.auth import get_redis
|
||||
# Look up CSRF token from session (Redis with DB fallback)
|
||||
from app.core.auth import get_redis, get_session_data
|
||||
|
||||
redis = get_redis()
|
||||
raw = await redis.get(f"session:{session_id}")
|
||||
if raw is None:
|
||||
session_data = await get_session_data(redis, session_id)
|
||||
if session_data is None:
|
||||
return JSONResponse(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
content={"detail": "Session expired for CSRF validation", "code": "csrf_session_expired"},
|
||||
)
|
||||
|
||||
session_data = json.loads(raw)
|
||||
stored_token = session_data.get("csrf_token")
|
||||
|
||||
if not stored_token or stored_token != csrf_header:
|
||||
@@ -132,6 +131,10 @@ class CSRFMiddleware(BaseHTTPMiddleware):
|
||||
)
|
||||
|
||||
# Sliding session: also extend TTL on CSRF-validated unsafe requests
|
||||
await redis.expire(f"session:{session_id}", settings.session_ttl_seconds)
|
||||
# (best-effort — ignore Redis errors during outage)
|
||||
try:
|
||||
await redis.expire(f"session:{session_id}", settings.session_ttl_seconds)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return await call_next(request)
|
||||
|
||||
Reference in New Issue
Block a user