Initial commit: CI Agent competitive-intelligence monitoring app

FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring,
source collection, LLM-based change analysis, enrichment, and account
security (Turnstile, escalating lockout, email verification).

apps/api/app/core/logging.py

@@ -0,0 +1,149 @@
+"""Structured logging setup with secret redaction, plus a capped live-log
+feed for the Settings UI.
+
+Never log raw credentials. Any log event field whose key looks secret-shaped
+gets its value replaced before it leaves the process.
+
+Logs also get pushed to a capped Redis list (`app:logs`) so the Settings
+page can show a live feed of what the application is doing - stdout/Docker
+logs alone aren't visible from the UI, and this app runs as several
+separate processes (API, Celery worker, beat), so Redis (already shared
+infra across all of them) is the simplest common sink. A sink failure here
+must never break the actual log call or crash the app - every Redis
+operation is wrapped and swallowed.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING, Any
+
+import structlog
+
+if TYPE_CHECKING:
+    from app.core.config import Settings
+
+_SECRET_KEY_PATTERN = re.compile(
+    r"(key|secret|token|password|authorization|credential)", re.IGNORECASE
+)
+_REDACTED = "***REDACTED***"
+
+LOG_STREAM_KEY = "app:logs"
+LOG_STREAM_MAXLEN = 500
+
+LOG_CATEGORY_LABELS = {
+    "internal_error": "Internal error",
+    "api_error": "API error",
+    "important": "Important",
+    "normal": "Normal",
+}
+
+_NON_CONTEXT_KEYS = {"level", "timestamp", "logger", "event", "important"}
+
+_sync_redis_client: Any = None
+
+
+def _redact_secrets(_logger: Any, _method_name: str, event_dict: dict) -> dict:
+    for key in list(event_dict.keys()):
+        if _SECRET_KEY_PATTERN.search(key):
+            event_dict[key] = _REDACTED
+    return event_dict
+
+
+def _category_for(level: str, event_dict: dict) -> str:
+    if level in ("error", "critical"):
+        return "internal_error"
+    if level == "warning":
+        return "api_error"
+    if level == "info" and event_dict.get("important") is True:
+        return "important"
+    return "normal"
+
+
+def _get_sync_redis_client(redis_url: str):
+    global _sync_redis_client
+    if _sync_redis_client is None:
+        import redis as redis_sync
+
+        _sync_redis_client = redis_sync.Redis.from_url(redis_url)
+    return _sync_redis_client
+
+
+def _make_capture_processor(redis_url: str):
+    def _capture_for_ui(_logger: Any, method_name: str, event_dict: dict) -> dict:
+        level = event_dict.get("level", method_name)
+        if level == "debug":
+            return event_dict
+        try:
+            record = {
+                "ts": event_dict.get("timestamp") or datetime.now(UTC).isoformat(),
+                "level": level,
+                "category": _category_for(level, event_dict),
+                "logger": event_dict.get("logger", "app"),
+                "event": str(event_dict.get("event", "")),
+                "context": {k: v for k, v in event_dict.items() if k not in _NON_CONTEXT_KEYS},
+            }
+            client = _get_sync_redis_client(redis_url)
+            pipe = client.pipeline()
+            pipe.lpush(LOG_STREAM_KEY, json.dumps(record, default=str))
+            pipe.ltrim(LOG_STREAM_KEY, 0, LOG_STREAM_MAXLEN - 1)
+            pipe.execute()
+        except Exception:  # noqa: BLE001 - a broken log sink must never break the app
+            pass
+        return event_dict
+
+    return _capture_for_ui
+
+
+def configure_logging(settings: Settings) -> None:
+    logging.basicConfig(level=settings.log_level, format="%(message)s")
+
+    structlog.configure(
+        processors=[
+            structlog.contextvars.merge_contextvars,
+            structlog.stdlib.add_log_level,
+            structlog.processors.TimeStamper(fmt="iso"),
+            _redact_secrets,
+            _make_capture_processor(settings.redis_url),
+            structlog.processors.StackInfoRenderer(),
+            structlog.processors.format_exc_info,
+            structlog.processors.JSONRenderer(),
+        ],
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(settings.log_level)
+        ),
+        context_class=dict,
+        logger_factory=structlog.PrintLoggerFactory(),
+        cache_logger_on_first_use=True,
+    )
+
+
+def get_logger(name: str) -> structlog.stdlib.BoundLogger:
+    # `.bind(logger=name)` (rather than `structlog.stdlib.add_logger_name`,
+    # which only works with `structlog.stdlib.LoggerFactory`) is what makes
+    # the module name available to `_capture_for_ui` below - this app uses
+    # `PrintLoggerFactory`, whose loggers have no `.name` attribute.
+    return structlog.get_logger().bind(logger=name)
+
+
+async def get_recent_logs(settings: Settings, limit: int = 100) -> list[dict]:
+    """Most-recent-first log entries from the capped Redis feed, for the
+    Settings page's Logging box."""
+    import redis.asyncio as redis_asyncio
+
+    client = redis_asyncio.from_url(settings.redis_url)
+    try:
+        raw_entries = await client.lrange(LOG_STREAM_KEY, 0, limit - 1)
+    finally:
+        await client.aclose()
+
+    entries: list[dict] = []
+    for raw in raw_entries:
+        try:
+            entries.append(json.loads(raw))
+        except (json.JSONDecodeError, TypeError):
+            continue
+    return entries