Initial commit: CI Agent competitive-intelligence monitoring app

FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification).
2026-08-05 10:48:20 -04:00
commit 1a4c80958f
365 changed files with 43541 additions and 0 deletions
@@ -0,0 +1,60 @@
+"""Celery application: broker/backend config, queue routing, and the Beat
+schedule. Task modules are plain sync functions that bridge into the async
+service layer via `asyncio.run` (see tasks/base.py) - Celery's worker model
+is synchronous/prefork, while the rest of the app is async SQLAlchemy.
+"""
+
+from __future__ import annotations
+
+from celery import Celery
+from celery.schedules import crontab
+
+from app.core.config import get_settings
+from app.core.logging import configure_logging
+
+settings = get_settings()
+# The worker process never went through main.py's bootstrap, so without
+# this call every logger.* call here ran through structlog's unconfigured
+# default - no secret redaction, no capture into the Settings-page log feed.
+# This module is imported once per worker process (Celery's `include=`
+# above), so this runs exactly where main.py's equivalent call does.
+configure_logging(settings)
+
+celery_app = Celery(
+    "ci_agent",
+    broker=settings.redis_url,
+    backend=settings.redis_url,
+    include=[
+        "app.tasks.collection",
+        "app.tasks.scheduler",
+        "app.tasks.maintenance",
+        "app.tasks.enrichment",
+    ],
+)
+
+celery_app.conf.update(
+    task_always_eager=settings.celery_task_always_eager,
+    task_eager_propagates=settings.celery_task_always_eager,
+    task_default_queue="default",
+    task_routes={
+        "app.tasks.collection.*": {"queue": "collection"},
+        "app.tasks.scheduler.*": {"queue": "default"},
+        "app.tasks.analysis.*": {"queue": "analysis"},
+        "app.tasks.notifications.*": {"queue": "notifications"},
+        "app.tasks.maintenance.*": {"queue": "maintenance"},
+        "app.tasks.enrichment.*": {"queue": "enrichment"},
+    },
+    beat_schedule={
+        "sync-monitoring-schedules": {
+            "task": "app.tasks.scheduler.sync_schedules",
+            "schedule": 60.0,
+        },
+        "purge-expired-data": {
+            "task": "app.tasks.maintenance.purge_expired_data",
+            "schedule": crontab(hour=3, minute=0),
+        },
+    },
+    timezone="UTC",
+    worker_hijack_root_logger=False,
+    task_track_started=True,
+)