Files
leocrm/app/main.py
T
Agent Zero a26405f15e Phase 4: Circuit Breaker, DB Retry, Redis Graceful Degradation
- app/core/resilience.py: CircuitBreaker (CLOSED/OPEN/HALF_OPEN), retry_db,
  redis_call_with_fallback, InMemoryRateLimiter, CircuitBreakerMiddleware
- app/core/auth.py: get_session_data now falls back to PostgreSQL sessions
  table when Redis is unavailable
- app/core/permissions.py: get_cached_permissions falls back to direct DB
  resolution when Redis circuit is open
- app/core/rate_limit.py: check_rate_limit falls back to in-memory limiter
  when Redis is down; reset_rate_limit clears both Redis and in-memory
- app/core/middleware.py: CSRF validation uses get_session_data (Redis+DB
  fallback); sliding session TTL is best-effort during outage
- app/core/db/__init__.py: get_db() wraps session creation with retry_db
  for transient connection errors; records circuit breaker success/failure
- app/deps.py: refresh_session_ttl wrapped in try/except for Redis outage
- app/main.py: CircuitBreakerMiddleware registered (returns 503 when DB
  circuit is OPEN, skips health/metrics endpoints)
- app/config.py: Added resilience settings (thresholds, cooldown, retries)
- tests/test_resilience.py: 30 tests covering all patterns

30/30 resilience tests pass. No regressions in plugin lifecycle tests.
2026-08-04 14:34:06 +02:00

541 lines
24 KiB
Python

"""FastAPI application - LeoCRM backend."""
from __future__ import annotations
import time
import traceback
from contextlib import asynccontextmanager
from fastapi import FastAPI, HTTPException, Request, Depends, APIRouter
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.routing import WebSocketRoute
import importlib
import logging
import os
logger = logging.getLogger(__name__)
from app.config import get_settings
from app.core.db import close_engine, get_engine
from app.core.error_codes import ApiError
from app.core.middleware import CSRFMiddleware, SecurityHeadersMiddleware
from app.core.rate_limit import GeneralRateLimitMiddleware
from app.core.resilience import CircuitBreakerMiddleware
from app.core.monitoring import record_error, record_request
from app.core.plugin_error_handler import wrap_plugin_route
from app.core.service_container import get_container
from app.plugins.registry import get_registry
from app.routes import (
addresses,
bank_accounts,
ai_copilot,
audit,
auth,
errors,
contact_folders,
contact_folder_permissions,
entity_permissions,
contacts,
dashboard,
entity_history,
groups,
health,
import_export,
metrics,
notifications,
plugins,
roles,
tenants,
users,
user_preferences,
workflows,
currencies,
taxes,
sequences,
system_settings,
attachments,
custom_field_definitions,
custom_fields,
saved_filters,
workspaces,
saved_views,
webhooks,
backups,
owner_transfer,
permission_templates,
delegations,
policies,
guest_auth,
guests,
outbox,
api_tokens,
)
class RequestLoggingMiddleware(BaseHTTPMiddleware):
"""Structured logging + Prometheus metrics for every HTTP request."""
async def dispatch(self, request: Request, call_next):
start_time = time.perf_counter()
method = request.method
path = request.url.path
# Extract tenant_id from session cookie if available (best-effort)
tenant_id = None
try:
response = await call_next(request)
except Exception as exc:
duration_ms = (time.perf_counter() - start_time) * 1000
tb_str = traceback.format_exc()
record_error(
event="unhandled_exception",
method=method,
path=path,
status_code=500,
error=str(exc),
traceback_str=tb_str,
tenant_id=tenant_id,
)
# Report to Forgejo error reporter
try:
from app.plugins.builtins.forgejo_error_reporter.service import report_error_to_forgejo
await report_error_to_forgejo({
"message": f"[Backend] {method} {path}: {exc}",
"stack": tb_str,
"url": str(request.url),
"context": {"method": method, "path": path, "source": "backend_middleware"},
})
except Exception:
pass # Never let error reporting break the request
raise
duration_ms = (time.perf_counter() - start_time) * 1000
status_code = response.status_code
# Report 4xx and 5xx errors to Forgejo (except 401/403 which are expected)
if status_code >= 400 and status_code not in (401, 403):
try:
from app.plugins.builtins.forgejo_error_reporter.service import report_error_to_forgejo
await report_error_to_forgejo({
"message": f"[Backend] {method} {path}{status_code}",
"url": str(request.url),
"context": {"method": method, "path": path, "status": status_code, "source": "backend_response"},
})
except Exception:
pass # Never let error reporting break the response
# Try to get tenant_id from response headers or request state
# (set by auth middleware/dependency — best-effort, never log credentials)
record_request(
method=method,
path=path,
status_code=status_code,
duration_ms=duration_ms,
tenant_id=tenant_id,
)
return response
@asynccontextmanager
async def lifespan(app: FastAPI):
"""Application lifespan: startup and shutdown."""
# Initialize global Redis client (singleton)
from app.core.auth import init_redis, close_redis
from app.core.jobs import init_job_pool, close_job_pool
await init_redis()
await init_job_pool()
# Initialize service container
container = get_container()
await container.initialize()
# Initialize plugin registry and discover built-in plugins
registry = get_registry()
from app.core.db import get_migration_engine
registry.initialize(get_migration_engine(), app)
registry.discover_builtins()
# Install discovered builtin plugins and activate only those marked active in DB
from sqlalchemy import select as sa_select
from sqlalchemy.ext.asyncio import async_sessionmaker
from app.models.plugin import Plugin as PluginModel
from app.core.event_bus import get_event_bus
event_bus = get_event_bus()
async_session = async_sessionmaker(get_engine(), expire_on_commit=False)
# Load all tenant IDs for per-tenant plugin activation (RLS fail-closed requires tenant context)
from app.models.tenant import Tenant as TenantModel
from app.core.db import set_tenant_context
async with async_session() as db:
tenant_result = await db.execute(sa_select(TenantModel.id))
all_tenant_ids = [row[0] for row in tenant_result]
logger.info(f"Loaded {len(all_tenant_ids)} tenants for plugin activation")
# Install plugin records and run migrations (global, no tenant context needed)
async with async_session() as db:
for name in registry.resolve_load_order():
plugin = registry.get_plugin(name)
if plugin is None:
continue
# Check if plugin already in DB
result = await db.execute(
sa_select(PluginModel).where(PluginModel.name == name)
)
plugin_record = result.scalar_one_or_none()
if plugin_record is None:
plugin_record = PluginModel(
name=name,
display_name=plugin.manifest.display_name,
version=plugin.manifest.version,
status="installed",
active=plugin.manifest.is_core,
is_core=plugin.manifest.is_core,
)
db.add(plugin_record)
await db.flush()
logger.info(f"Created plugin record: {name} (core={plugin.manifest.is_core})")
# Run migrations if not yet applied — use MIGRATION engine (crm_migration) for DDL
if plugin.manifest.migrations:
try:
from app.core.db import get_migration_session_factory
mig_session_factory = get_migration_session_factory()
async with mig_session_factory() as mig_db:
await registry.migration_runner.run_all_migrations(
mig_db, name, plugin.manifest.migrations
)
await mig_db.commit()
except Exception as exc:
logger.error(f"Migration FAILED for {name}: {exc}")
if plugin_record.active:
logger.error(f"Deactivating plugin {name} due to migration failure")
plugin_record.active = False
plugin_record.status = "migration_failed"
continue
# Only activate plugins that are marked active in DB
if not plugin_record.active:
logger.info(f"Plugin {name} is inactive — skipping activation")
continue
# Activate plugin with a FRESH session per plugin to avoid RLS state leakage
# RLS fail-closed requires app.current_tenant_id for tenant-table writes.
# Plugin activation may fail on duplicate cron job inserts — this is harmless
# since cron jobs already exist from previous startups.
plugin_activated = False
for tenant_id in all_tenant_ids:
try:
async with async_session() as plugin_db:
await set_tenant_context(plugin_db, tenant_id)
await plugin.on_activate(plugin_db, container, event_bus)
await plugin_db.flush()
await plugin_db.commit()
plugin_activated = True
except Exception as exc:
logger.warning(f"[STARTUP] Plugin {name} activation issue for tenant {tenant_id}: {exc}")
break
if plugin_activated:
plugin_record.status = "active"
logger.info(f"[STARTUP] Activated plugin: {name}")
try:
await db.commit()
except Exception as exc:
logger.warning(f"[STARTUP] Commit failed after plugin activation: {exc}")
await db.rollback()
# Initialize permission registry with active plugin names
from app.core.permission_registry import init_permission_registry, register_plugin_permissions
active_plugin_names: set[str] = set()
async with async_session() as db:
result = await db.execute(
sa_select(PluginModel).where(PluginModel.active == True) # noqa: E712
)
for record in result.scalars().all():
active_plugin_names.add(record.name)
plugin = registry.get_plugin(record.name)
if plugin and plugin.manifest.permissions:
register_plugin_permissions(record.name, plugin.manifest.permissions)
init_permission_registry(active_plugin_names)
logger.info("Permission registry initialized with %d active plugins", len(active_plugin_names))
# Register webhook dispatcher on the event bus
from app.core.webhook_dispatcher import register_webhook_event_handlers
register_webhook_event_handlers(event_bus)
logger.info("Webhook event handlers registered")
# Register field definitions from active plugins only
from app.core.permission_registry import get_permission_registry
for name in active_plugin_names:
plugin = registry.get_plugin(name)
if plugin:
field_defs = plugin.get_field_definitions()
if field_defs:
get_permission_registry().register_field_definitions(name, field_defs)
logger.info("Field definitions registered for %d active plugins", len(active_plugin_names))
# Seed default data (EUR currency, 19%/7% tax rates) for all tenants
from app.core.seeds import seed_default_data
async with async_session() as db:
try:
await seed_default_data(db)
await db.commit()
logger.info("Default data seeding completed")
except Exception as exc:
logger.warning(f"Default data seeding failed: {exc}")
await db.rollback()
yield
# Shutdown: close global Redis and ARQ pool
await close_job_pool()
await close_redis()
await close_engine()
def create_app() -> FastAPI:
"""Create and configure the FastAPI application."""
settings = get_settings()
app = FastAPI(
title="LeoCRM",
version="1.0.0",
lifespan=lifespan,
description=(
"LeoCRM — Self-hosted CRM system for small sales teams.\n\n"
"## Authentication\n"
"All endpoints (except `/api/v1/auth/login` and `/api/v1/health`) require "
"a valid session cookie. Obtain a session by calling `POST /api/v1/auth/login`.\n\n"
"## Multi-Tenancy\n"
"All data is tenant-scoped. The tenant context is set automatically from the "
"authenticated session.\n\n"
"## Plugins\n"
"LeoCRM uses a plugin architecture. Core routes are always available. "
"Plugin routes (DMS, Mail, Calendar, Automation, etc.) are registered when "
"the corresponding plugin is active."
),
openapi_tags=[
{"name": "health", "description": "Health check endpoints — no auth required."},
{"name": "metrics", "description": "Prometheus metrics endpoint."},
{"name": "auth", "description": "Authentication: login, logout, session, password reset."},
{"name": "users", "description": "User management: CRUD, user-tenant assignments."},
{"name": "roles", "description": "Role management and permission definitions."},
{"name": "groups", "description": "User groups for contact assignment and filtering."},
{"name": "tenants", "description": "Tenant management and tenant switching."},
{"name": "notifications", "description": "User notifications and notification preferences."},
{"name": "contacts", "description": "Contact CRUD, contact persons, FTS search, export, soft-delete."},
{"name": "contact-folders", "description": "Contact folder management for organization."},
{"name": "entity-history", "description": "Audit trail and entity change history."},
{"name": "import-export", "description": "Bulk import and export of contacts and data."},
{"name": "plugins", "description": "Plugin management: list, install, activate, deactivate."},
{"name": "ai-copilot", "description": "AI copilot: chat, suggestions, conversation history."},
{"name": "workflows", "description": "Workflow definitions, instances, and execution."},
{"name": "user-preferences", "description": "Per-user preference settings."},
{"name": "currencies", "description": "Currency management for multi-currency support."},
{"name": "taxes", "description": "Tax rate management (VAT, sales tax)."},
{"name": "sequences", "description": "Number sequence management for invoices, quotes, etc."},
{"name": "system-settings", "description": "Tenant-level system settings (company info, theme)."},
{"name": "attachments", "description": "File attachments for contacts, companies, and entities."},
{"name": "addresses", "description": "Address management for contacts and companies."},
{"name": "bank_accounts", "description": "Bank account management for tenant."},
{"name": "audit", "description": "Audit log queries and compliance reporting."},
{"name": "automation", "description": "Automation engine: agents, automations, cron jobs, execution logs."},
{"name": "agents", "description": "AI agent definitions and agent runner endpoints."},
{"name": "dms", "description": "Document Management System: files, folders, sources, sharing."},
{"name": "mail", "description": "Email integration: IMAP accounts, folders, messages, send."},
{"name": "calendar", "description": "Calendar management: appointments, resources, recurrence."},
{"name": "search", "description": "Unified search across contacts, documents, emails, etc."},
{"name": "reports", "description": "Report generator: templates, rendering, scheduled reports."},
{"name": "entity-links", "description": "Entity linking: connect contacts to DMS documents and other entities."},
{"name": "kommunikation", "description": "Unified messaging: conversations, participants, messages."},
{"name": "ai-proactive", "description": "Proactive AI: insights, alerts, and recommendations."},
{"name": "ai-assistant", "description": "AI assistant: chat completions, tool calling, context awareness."},
{"name": "tags", "description": "Tag management: create, assign, search tags across entities."},
{"name": "permissions", "description": "Permission management: roles, field-level permissions, sharing."},
{"name": "public-share", "description": "Public sharing endpoints — no auth required, token-based access."},
],
)
app.add_middleware(
CORSMiddleware,
allow_origins=settings.cors_origin_list,
allow_credentials=True,
allow_methods=["GET", "POST", "PATCH", "PUT", "DELETE", "OPTIONS"],
allow_headers=["Authorization", "Content-Type", "X-CSRF-Token", "X-Tenant-ID"],
max_age=3600,
)
app.add_middleware(CSRFMiddleware)
app.add_middleware(SecurityHeadersMiddleware)
app.add_middleware(GeneralRateLimitMiddleware)
app.add_middleware(RequestLoggingMiddleware)
app.add_middleware(CircuitBreakerMiddleware)
# ── Global exception handler — catch ALL unhandled exceptions ──
@app.exception_handler(Exception)
async def global_exception_handler(request: Request, exc: Exception):
logger.error(f"Unhandled exception: {exc}", exc_info=True)
record_error(
event="unhandled_exception",
method=request.method,
path=request.url.path,
status_code=500,
error=str(exc),
)
return JSONResponse(
status_code=500,
content={"detail": "Internal server error", "code": "internal_error"},
)
# ── ApiError handler — structured error responses ──
@app.exception_handler(ApiError)
async def api_error_handler(request: Request, exc: ApiError):
return JSONResponse(
status_code=exc.status,
content={'code': exc.code, 'detail': exc.detail, 'field': exc.field}
)
app.include_router(health.router)
app.include_router(metrics.router)
app.include_router(auth.router)
app.include_router(users.router)
app.include_router(roles.router)
app.include_router(groups.router)
app.include_router(tenants.router)
app.include_router(notifications.router)
app.include_router(contacts.router)
app.include_router(contact_folders.router)
app.include_router(contact_folder_permissions.router)
app.include_router(entity_permissions.router)
app.include_router(dashboard.router)
app.include_router(entity_history.router)
app.include_router(import_export.router)
app.include_router(plugins.router)
app.include_router(ai_copilot.router)
app.include_router(workflows.router)
app.include_router(user_preferences.router)
app.include_router(currencies.router)
app.include_router(taxes.router)
app.include_router(sequences.router)
app.include_router(system_settings.router)
app.include_router(attachments.router)
app.include_router(addresses.router)
app.include_router(bank_accounts.router)
app.include_router(audit.router)
app.include_router(backups.router)
app.include_router(owner_transfer.router)
app.include_router(custom_field_definitions.router)
app.include_router(custom_fields.router)
app.include_router(saved_filters.router)
app.include_router(saved_views.router)
app.include_router(webhooks.router)
app.include_router(permission_templates.router)
app.include_router(delegations.router)
app.include_router(policies.router)
app.include_router(errors.router)
app.include_router(guest_auth.router)
app.include_router(guests.router)
app.include_router(workspaces.router)
app.include_router(outbox.router)
app.include_router(api_tokens.router)
# ── Register plugin routes for all built-in plugins ──
# Routes are registered at app creation time so OpenAPI docs are complete.
# Activation status is enforced per-request via require_active_plugin().
import importlib
from app.deps import require_active_plugin
# Discover all built-in plugin modules and register their routes
plugin_modules = [
"app.plugins.builtins.tags",
"app.plugins.builtins.permissions",
"app.plugins.builtins.entity_links",
"app.plugins.builtins.dms",
"app.plugins.builtins.calendar",
"app.plugins.builtins.mail",
"app.plugins.builtins.report_generator",
"app.plugins.builtins.kommunikation",
"app.plugins.builtins.tasks",
"app.plugins.builtins.automation",
"app.plugins.builtins.ai_assistant",
"app.plugins.builtins.ai_proactive",
"app.plugins.builtins.ai_ui_control",
"app.plugins.builtins.mcp_client",
"app.plugins.builtins.mcp_server",
"app.plugins.builtins.system_notif",
"app.plugins.builtins.unified_search",
"app.plugins.builtins.forgejo_error_reporter",
]
for mod_name in plugin_modules:
try:
mod = importlib.import_module(mod_name)
# Find the plugin class and get its manifest routes
for attr_name in dir(mod):
attr = getattr(mod, attr_name)
if isinstance(attr, type) and hasattr(attr, "manifest") and hasattr(attr.manifest, "routes"):
plugin_name = getattr(attr.manifest, "name", mod_name.split(".")[-1])
for route_def in attr.manifest.routes:
try:
router_module = importlib.import_module(route_def.module)
router = getattr(router_module, route_def.router_attr)
# Skip WebSocket routes — no wrapping, no plugin check
from starlette.routing import WebSocketRoute
plugin_dep = Depends(require_active_plugin(plugin_name))
for route in router.routes:
if isinstance(route, WebSocketRoute):
# WebSocket routes also need plugin check — don't skip (P1.9 fix)
if not hasattr(route, 'dependencies'):
route.dependencies = []
route.dependencies.append(plugin_dep)
continue
# Add require_active_plugin to each HTTP route's dependencies
if not hasattr(route, 'dependencies'):
route.dependencies = []
route.dependencies.append(plugin_dep)
app.include_router(router)
except Exception as exc:
logger.error(f"Failed to register route {route_def.module}.{route_def.router_attr}: {exc}")
break
except Exception as exc:
logger.error(f"Failed to register plugin routes for {mod_name}: {exc}")
# ── Serve frontend static files (SPA) ──────────────────────────────
# Mount built frontend assets (JS, CSS, images)
frontend_dist = os.path.join(os.path.dirname(__file__), "..", "frontend", "dist")
if os.path.isdir(frontend_dist):
# Serve static assets at /assets
assets_path = os.path.join(frontend_dist, "assets")
if os.path.isdir(assets_path):
app.mount("/assets", StaticFiles(directory=assets_path), name="assets")
# SPA catch-all: serve index.html for all non-API routes
@app.get("/{full_path:path}")
async def spa_spa(full_path: str):
"""Serve index.html for all non-API routes (SPA fallback)."""
# Don't intercept API routes
if full_path.startswith(("api/", "docs", "openapi", "redoc")):
raise HTTPException(status_code=404, detail="Not Found")
# Block path traversal and system file access
blocked_prefixes = ("var/log/", "error/", "error_log", "var/", "etc/", "proc/", "sys/")
if full_path.startswith(blocked_prefixes) or ".." in full_path:
raise HTTPException(status_code=404, detail="Not Found")
index_path = os.path.join(frontend_dist, "index.html")
if os.path.isfile(index_path):
return FileResponse(index_path)
raise HTTPException(status_code=404, detail="Frontend not built")
return app
app = create_app()