"""Structured logging setup with secret redaction, plus a capped live-log feed for the Settings UI. Never log raw credentials. Any log event field whose key looks secret-shaped gets its value replaced before it leaves the process. Logs also get pushed to a capped Redis list (`app:logs`) so the Settings page can show a live feed of what the application is doing - stdout/Docker logs alone aren't visible from the UI, and this app runs as several separate processes (API, Celery worker, beat), so Redis (already shared infra across all of them) is the simplest common sink. A sink failure here must never break the actual log call or crash the app - every Redis operation is wrapped and swallowed. """ from __future__ import annotations import json import logging import re from datetime import UTC, datetime from typing import TYPE_CHECKING, Any import structlog if TYPE_CHECKING: from app.core.config import Settings _SECRET_KEY_PATTERN = re.compile( r"(key|secret|token|password|authorization|credential)", re.IGNORECASE ) _REDACTED = "***REDACTED***" LOG_STREAM_KEY = "app:logs" LOG_STREAM_MAXLEN = 500 LOG_CATEGORY_LABELS = { "internal_error": "Internal error", "api_error": "API error", "important": "Important", "normal": "Normal", } _NON_CONTEXT_KEYS = {"level", "timestamp", "logger", "event", "important"} _sync_redis_client: Any = None def _redact_secrets(_logger: Any, _method_name: str, event_dict: dict) -> dict: for key in list(event_dict.keys()): if _SECRET_KEY_PATTERN.search(key): event_dict[key] = _REDACTED return event_dict def _category_for(level: str, event_dict: dict) -> str: if level in ("error", "critical"): return "internal_error" if level == "warning": return "api_error" if level == "info" and event_dict.get("important") is True: return "important" return "normal" def _get_sync_redis_client(redis_url: str): global _sync_redis_client if _sync_redis_client is None: import redis as redis_sync _sync_redis_client = redis_sync.Redis.from_url(redis_url) return _sync_redis_client def _make_capture_processor(redis_url: str): def _capture_for_ui(_logger: Any, method_name: str, event_dict: dict) -> dict: level = event_dict.get("level", method_name) if level == "debug": return event_dict try: record = { "ts": event_dict.get("timestamp") or datetime.now(UTC).isoformat(), "level": level, "category": _category_for(level, event_dict), "logger": event_dict.get("logger", "app"), "event": str(event_dict.get("event", "")), "context": {k: v for k, v in event_dict.items() if k not in _NON_CONTEXT_KEYS}, } client = _get_sync_redis_client(redis_url) pipe = client.pipeline() pipe.lpush(LOG_STREAM_KEY, json.dumps(record, default=str)) pipe.ltrim(LOG_STREAM_KEY, 0, LOG_STREAM_MAXLEN - 1) pipe.execute() except Exception: # noqa: BLE001 - a broken log sink must never break the app pass return event_dict return _capture_for_ui def configure_logging(settings: Settings) -> None: logging.basicConfig(level=settings.log_level, format="%(message)s") structlog.configure( processors=[ structlog.contextvars.merge_contextvars, structlog.stdlib.add_log_level, structlog.processors.TimeStamper(fmt="iso"), _redact_secrets, _make_capture_processor(settings.redis_url), structlog.processors.StackInfoRenderer(), structlog.processors.format_exc_info, structlog.processors.JSONRenderer(), ], wrapper_class=structlog.make_filtering_bound_logger( logging.getLevelName(settings.log_level) ), context_class=dict, logger_factory=structlog.PrintLoggerFactory(), cache_logger_on_first_use=True, ) def get_logger(name: str) -> structlog.stdlib.BoundLogger: # `.bind(logger=name)` (rather than `structlog.stdlib.add_logger_name`, # which only works with `structlog.stdlib.LoggerFactory`) is what makes # the module name available to `_capture_for_ui` below - this app uses # `PrintLoggerFactory`, whose loggers have no `.name` attribute. return structlog.get_logger().bind(logger=name) async def get_recent_logs(settings: Settings, limit: int = 100) -> list[dict]: """Most-recent-first log entries from the capped Redis feed, for the Settings page's Logging box.""" import redis.asyncio as redis_asyncio client = redis_asyncio.from_url(settings.redis_url) try: raw_entries = await client.lrange(LOG_STREAM_KEY, 0, limit - 1) finally: await client.aclose() entries: list[dict] = [] for raw in raw_entries: try: entries.append(json.loads(raw)) except (json.JSONDecodeError, TypeError): continue return entries