Phase 4: Circuit Breaker, DB Retry, Redis Graceful Degradation
- app/core/resilience.py: CircuitBreaker (CLOSED/OPEN/HALF_OPEN), retry_db, redis_call_with_fallback, InMemoryRateLimiter, CircuitBreakerMiddleware - app/core/auth.py: get_session_data now falls back to PostgreSQL sessions table when Redis is unavailable - app/core/permissions.py: get_cached_permissions falls back to direct DB resolution when Redis circuit is open - app/core/rate_limit.py: check_rate_limit falls back to in-memory limiter when Redis is down; reset_rate_limit clears both Redis and in-memory - app/core/middleware.py: CSRF validation uses get_session_data (Redis+DB fallback); sliding session TTL is best-effort during outage - app/core/db/__init__.py: get_db() wraps session creation with retry_db for transient connection errors; records circuit breaker success/failure - app/deps.py: refresh_session_ttl wrapped in try/except for Redis outage - app/main.py: CircuitBreakerMiddleware registered (returns 503 when DB circuit is OPEN, skips health/metrics endpoints) - app/config.py: Added resilience settings (thresholds, cooldown, retries) - tests/test_resilience.py: 30 tests covering all patterns 30/30 resilience tests pass. No regressions in plugin lifecycle tests.
This commit is contained in:
+42
-4
@@ -188,13 +188,51 @@ async def create_session(
|
||||
|
||||
|
||||
async def get_session_data(redis: aioredis.Redis, session_id: str) -> dict[str, Any] | None:
|
||||
"""Retrieve session data from Redis."""
|
||||
"""Retrieve session data from Redis with DB fallback.
|
||||
|
||||
Tries Redis first. If Redis is unavailable, falls back to PostgreSQL
|
||||
sessions table (audit trail) to keep users logged in during Redis outages.
|
||||
"""
|
||||
import json
|
||||
|
||||
raw = await redis.get(f"session:{session_id}")
|
||||
if raw is None:
|
||||
from app.core.resilience import get_circuit
|
||||
|
||||
circuit = get_circuit("redis")
|
||||
if await circuit.can_proceed():
|
||||
try:
|
||||
raw = await redis.get(f"session:{session_id}")
|
||||
await circuit.record_success()
|
||||
if raw is None:
|
||||
return None
|
||||
return json.loads(raw)
|
||||
except Exception as exc:
|
||||
logger.warning("Redis session lookup failed: %s — falling back to DB", exc)
|
||||
await circuit.record_failure()
|
||||
|
||||
# DB fallback: query sessions table
|
||||
try:
|
||||
from app.core.db import get_auth_session_factory
|
||||
from app.models.session import Session as SessionModel
|
||||
from sqlalchemy import select
|
||||
from datetime import UTC, datetime
|
||||
|
||||
factory = get_auth_session_factory()
|
||||
async with factory() as db:
|
||||
result = await db.execute(
|
||||
select(SessionModel).where(SessionModel.id == uuid.UUID(session_id))
|
||||
)
|
||||
session = result.scalar_one_or_none()
|
||||
if session is None or session.expires_at < datetime.now(UTC):
|
||||
return None
|
||||
return {
|
||||
"user_id": str(session.user_id),
|
||||
"tenant_id": str(session.tenant_id),
|
||||
"csrf_token": session.csrf_token,
|
||||
"is_active": True,
|
||||
}
|
||||
except Exception as db_exc:
|
||||
logger.error("DB fallback for session lookup also failed: %s", db_exc)
|
||||
return None
|
||||
return json.loads(raw)
|
||||
|
||||
|
||||
async def refresh_session_ttl(redis: aioredis.Redis, session_id: str) -> None:
|
||||
|
||||
Reference in New Issue
Block a user