CIAgent/apps/api/app/core/logging.py

"""Structured logging setup with secret redaction, plus a capped live-log
feed for the Settings UI.

Never log raw credentials. Any log event field whose key looks secret-shaped
gets its value replaced before it leaves the process.

Logs also get pushed to a capped Redis list (`app:logs`) so the Settings
page can show a live feed of what the application is doing - stdout/Docker
logs alone aren't visible from the UI, and this app runs as several
separate processes (API, Celery worker, beat), so Redis (already shared
infra across all of them) is the simplest common sink. A sink failure here
must never break the actual log call or crash the app - every Redis
operation is wrapped and swallowed.
"""

from __future__ import annotations

import json
import logging
import re
from datetime import UTC, datetime
from typing import TYPE_CHECKING, Any

import structlog

if TYPE_CHECKING:
    from app.core.config import Settings

_SECRET_KEY_PATTERN = re.compile(
    r"(key|secret|token|password|authorization|credential)", re.IGNORECASE
)
_REDACTED = "***REDACTED***"

LOG_STREAM_KEY = "app:logs"
LOG_STREAM_MAXLEN = 500

LOG_CATEGORY_LABELS = {
    "internal_error": "Internal error",
    "api_error": "API error",
    "important": "Important",
    "normal": "Normal",
}

_NON_CONTEXT_KEYS = {"level", "timestamp", "logger", "event", "important"}

_sync_redis_client: Any = None


def _redact_secrets(_logger: Any, _method_name: str, event_dict: dict) -> dict:
    for key in list(event_dict.keys()):
        if _SECRET_KEY_PATTERN.search(key):
            event_dict[key] = _REDACTED
    return event_dict


def _category_for(level: str, event_dict: dict) -> str:
    if level in ("error", "critical"):
        return "internal_error"
    if level == "warning":
        return "api_error"
    if level == "info" and event_dict.get("important") is True:
        return "important"
    return "normal"


def _get_sync_redis_client(redis_url: str):
    global _sync_redis_client
    if _sync_redis_client is None:
        import redis as redis_sync

        _sync_redis_client = redis_sync.Redis.from_url(redis_url)
    return _sync_redis_client


def _make_capture_processor(redis_url: str):
    def _capture_for_ui(_logger: Any, method_name: str, event_dict: dict) -> dict:
        level = event_dict.get("level", method_name)
        if level == "debug":
            return event_dict
        try:
            record = {
                "ts": event_dict.get("timestamp") or datetime.now(UTC).isoformat(),
                "level": level,
                "category": _category_for(level, event_dict),
                "logger": event_dict.get("logger", "app"),
                "event": str(event_dict.get("event", "")),
                "context": {k: v for k, v in event_dict.items() if k not in _NON_CONTEXT_KEYS},
            }
            client = _get_sync_redis_client(redis_url)
            pipe = client.pipeline()
            pipe.lpush(LOG_STREAM_KEY, json.dumps(record, default=str))
            pipe.ltrim(LOG_STREAM_KEY, 0, LOG_STREAM_MAXLEN - 1)
            pipe.execute()
        except Exception:  # noqa: BLE001 - a broken log sink must never break the app
            pass
        return event_dict

    return _capture_for_ui


def configure_logging(settings: Settings) -> None:
    logging.basicConfig(level=settings.log_level, format="%(message)s")

    structlog.configure(
        processors=[
            structlog.contextvars.merge_contextvars,
            structlog.stdlib.add_log_level,
            structlog.processors.TimeStamper(fmt="iso"),
            _redact_secrets,
            _make_capture_processor(settings.redis_url),
            structlog.processors.StackInfoRenderer(),
            structlog.processors.format_exc_info,
            structlog.processors.JSONRenderer(),
        ],
        wrapper_class=structlog.make_filtering_bound_logger(
            logging.getLevelName(settings.log_level)
        ),
        context_class=dict,
        logger_factory=structlog.PrintLoggerFactory(),
        cache_logger_on_first_use=True,
    )


def get_logger(name: str) -> structlog.stdlib.BoundLogger:
    # `.bind(logger=name)` (rather than `structlog.stdlib.add_logger_name`,
    # which only works with `structlog.stdlib.LoggerFactory`) is what makes
    # the module name available to `_capture_for_ui` below - this app uses
    # `PrintLoggerFactory`, whose loggers have no `.name` attribute.
    return structlog.get_logger().bind(logger=name)


async def get_recent_logs(settings: Settings, limit: int = 100) -> list[dict]:
    """Most-recent-first log entries from the capped Redis feed, for the
    Settings page's Logging box."""
    import redis.asyncio as redis_asyncio

    client = redis_asyncio.from_url(settings.redis_url)
    try:
        raw_entries = await client.lrange(LOG_STREAM_KEY, 0, limit - 1)
    finally:
        await client.aclose()

    entries: list[dict] = []
    for raw in raw_entries:
        try:
            entries.append(json.loads(raw))
        except (json.JSONDecodeError, TypeError):
            continue
    return entries