saksham/CIAgent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit: CI Agent competitive-intelligence monitoring app

Browse Source 
FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring,
source collection, LLM-based change analysis, enrichment, and account
security (Turnstile, escalating lockout, email verification).
This commit is contained in:
saksham
2026-08-05 10:48:20 -04:00
commit 1a4c80958f
365 changed files with 43541 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/app/tasks/__init__.py
View File
apps/api/app/tasks/base.py
+33
View File
@@ -0,0 +1,33 @@
"""Bridges Celery's sync task functions into the app's async service layer.
In production, a Celery worker process has no asyncio event loop running
when a task executes, so a plain `asyncio.run(...)` is enough. But with
`CELERY_TASK_ALWAYS_EAGER=true` (tests, and `.delay()` called from inside an
async FastAPI route handler), the task body runs synchronously *inside* the
caller's already-running event loop, and `asyncio.run()` refuses to nest.
`run_async_task` handles both: it uses `asyncio.run()` directly when no loop
is running, and falls back to a dedicated thread with its own loop when one is.
"""
from __future__ import annotations
import asyncio
import contextvars
from collections.abc import Coroutine
from concurrent.futures import ThreadPoolExecutor
def run_async_task[T](coro: Coroutine[object, object, T]) -> T:
try:
asyncio.get_running_loop()
except RuntimeError:
return asyncio.run(coro)
# ThreadPoolExecutor does not copy contextvars into the worker thread by
# default, which would silently drop the structlog correlation id
# (request_id/run_id/task_id - see core/logging.py) bound by the caller.
# Capturing the current context explicitly and running the executor call
# through it keeps those log fields intact even on this fallback path.
ctx = contextvars.copy_context()
with ThreadPoolExecutor(max_workers=1) as executor:
return executor.submit(ctx.run, asyncio.run, coro).result()
apps/api/app/tasks/celery_app.py
+60
View File
@@ -0,0 +1,60 @@
"""Celery application: broker/backend config, queue routing, and the Beat
schedule. Task modules are plain sync functions that bridge into the async
service layer via `asyncio.run` (see tasks/base.py) - Celery's worker model
is synchronous/prefork, while the rest of the app is async SQLAlchemy.
"""
from __future__ import annotations
from celery import Celery
from celery.schedules import crontab
from app.core.config import get_settings
from app.core.logging import configure_logging
settings = get_settings()
# The worker process never went through main.py's bootstrap, so without
# this call every logger.* call here ran through structlog's unconfigured
# default - no secret redaction, no capture into the Settings-page log feed.
# This module is imported once per worker process (Celery's `include=`
# above), so this runs exactly where main.py's equivalent call does.
configure_logging(settings)
celery_app = Celery(
"ci_agent",
broker=settings.redis_url,
backend=settings.redis_url,
include=[
"app.tasks.collection",
"app.tasks.scheduler",
"app.tasks.maintenance",
"app.tasks.enrichment",
],
)
celery_app.conf.update(
task_always_eager=settings.celery_task_always_eager,
task_eager_propagates=settings.celery_task_always_eager,
task_default_queue="default",
task_routes={
"app.tasks.collection.*": {"queue": "collection"},
"app.tasks.scheduler.*": {"queue": "default"},
"app.tasks.analysis.*": {"queue": "analysis"},
"app.tasks.notifications.*": {"queue": "notifications"},
"app.tasks.maintenance.*": {"queue": "maintenance"},
"app.tasks.enrichment.*": {"queue": "enrichment"},
},
beat_schedule={
"sync-monitoring-schedules": {
"task": "app.tasks.scheduler.sync_schedules",
"schedule": 60.0,
},
"purge-expired-data": {
"task": "app.tasks.maintenance.purge_expired_data",
"schedule": crontab(hour=3, minute=0),
},
},
timezone="UTC",
worker_hijack_root_logger=False,
task_track_started=True,
)
apps/api/app/tasks/collection.py
+265
View File
@@ -0,0 +1,265 @@
"""Celery task that executes one MonitoringRun: discovers sources on a
company's first run, collects every active source, and records progress on
the MonitoringRun row as it goes so the frontend can poll live status.
"""
from __future__ import annotations
import uuid
from datetime import UTC, datetime
from structlog.contextvars import bound_contextvars
from app.analysis.llm.factory import get_llm_provider
from app.core.config import get_settings
from app.core.logging import get_logger
from app.db.session import get_sessionmaker
from app.models.enums import MonitoringRunStatus, MonitoringRunTrigger, ReportType, SourceStatus
from app.repositories.company_repository import CompanyRepository
from app.repositories.monitoring_run_repository import MonitoringRunRepository
from app.repositories.report_repository import ReportRepository
from app.repositories.source_repository import SnapshotRepository, SourceRepository
from app.services import (
alert_service,
change_detection_service,
collection_service,
report_service,
user_api_key_service,
)
from app.services.scheduling import validate_and_compute_next_run
from app.tasks.base import run_async_task
from app.tasks.celery_app import celery_app
logger = get_logger(__name__)
@celery_app.task(bind=True, name="app.tasks.collection.run_monitoring", max_retries=2)
def run_monitoring(self, run_id: str) -> None:
run_async_task(_run_monitoring_async(run_id, self.request.id))
async def _run_monitoring_async(run_id: str, task_id: str | None) -> None:
with bound_contextvars(run_id=run_id, task_id=task_id):
await _run_monitoring(run_id, task_id)
async def _run_monitoring(run_id: str, task_id: str | None) -> None:
settings = get_settings()
session_factory = get_sessionmaker()
async with session_factory() as db:
run_repo = MonitoringRunRepository(db)
run = await run_repo.get(uuid.UUID(run_id))
if run is None:
logger.warning("monitoring_run_not_found", run_id=run_id)
return
company_repo = CompanyRepository(db)
company = await company_repo.get_by_id(run.company_id)
if company is None:
await run_repo.mark_finished(
run, status=MonitoringRunStatus.FAILED, error_summary="Company no longer exists"
)
return
await run_repo.set_worker_task_id(run, task_id)
await run_repo.mark_running(run)
logger.info(
"monitoring_run_started",
run_id=run_id,
company_id=str(company.id),
trigger=run.trigger_type.value,
)
settings = await user_api_key_service.get_effective_settings(db, company.user_id, settings)
llm = get_llm_provider(settings)
source_repo = SourceRepository(db)
snapshot_repo = SnapshotRepository(db)
sources = await source_repo.list_for_company(company.id)
if not sources:
try:
await collection_service.discover_sources_for_company(db, company, settings)
except Exception as exc: # pragma: no cover - defensive, discovery is best-effort
logger.warning(
"discovery_failed_during_run", company_id=str(company.id), error=str(exc)
)
sources = await source_repo.list_for_company(company.id)
now = datetime.now(UTC)
config = company.monitor_configuration
if run.trigger_type == MonitoringRunTrigger.SCHEDULED:
# Only the sources actually due right now - a source with a
# slower override (e.g. patents checked monthly) sits out a run
# that only a faster sibling source (e.g. news checked daily)
# triggered. See SourceRepository.list_due_for_company.
company_next_run = config.next_run if config is not None else None
active_sources = await source_repo.list_due_for_company(
company.id, now, company_next_run
)
else:
# Manual "Run now" is always a full on-demand check of every
# active source, regardless of any source's individual cadence.
active_sources = [s for s in sources if s.active]
items_collected = 0
successful = 0
failed = 0
changes_detected = 0
errors: list[str] = []
for source in active_sources:
try:
result = await collection_service.collect_source(
db, settings, source, company, monitoring_run_id=run.id
)
items_collected += len(result.documents)
# Advance this source's own clock only if it has an override
# - a source with no override has no next_check of its own
# (it rides the company's next_run instead, advanced below).
# Guarded separately from the collection result above: a
# scheduling computation issue must never get reported as a
# collection failure for a source that actually succeeded.
if source.frequency_type is not None:
try:
tz_name = config.timezone if config is not None else "UTC"
source.next_check = validate_and_compute_next_run(
frequency_type=source.frequency_type,
interval_minutes=source.interval_minutes,
cron_expression=source.cron_expression,
tz_name=tz_name,
minimum_interval_minutes=settings.minimum_monitoring_interval_minutes,
from_time=now,
)
except Exception as exc: # pragma: no cover - defensive
logger.warning(
"source_next_check_computation_failed",
source_id=str(source.id),
error=str(exc),
)
if result.status == SourceStatus.ACTIVE:
successful += 1
else:
failed += 1
if result.error:
errors.append(f"{source.name}: {result.error}")
if result.documents:
current_snapshot = await snapshot_repo.latest_for_source(source.id)
if current_snapshot is not None:
change = await change_detection_service.detect_change_for_source(
db, source, company, current_snapshot, run.id
)
if change is not None:
changes_detected += 1
logger.info(
"change_detected",
source_id=str(source.id),
change_type=change.change_type.value,
severity=change.severity.value,
significance=change.significance_score,
confidence=change.confidence_score,
)
try:
alert = await alert_service.create_alert_for_change(
db, settings, llm, change, company
)
if alert is not None:
logger.info(
"alert_created",
alert_id=str(alert.id),
severity=alert.severity.value,
)
except Exception as exc: # pragma: no cover - defensive, alerting failure shouldn't fail the run
logger.error(
"alert_creation_failed",
change_id=str(change.id),
error=str(exc),
)
except (
Exception
) as exc: # pragma: no cover - defensive, one bad source shouldn't kill the run
failed += 1
errors.append(f"{source.name}: {exc}")
logger.error(
"source_collection_unexpected_error", source_id=str(source.id), error=str(exc)
)
await run_repo.update_progress(
run,
sources_attempted=successful + failed,
sources_successful=successful,
sources_failed=failed,
items_collected=items_collected,
changes_detected=changes_detected,
)
if not active_sources or failed == 0:
status = MonitoringRunStatus.SUCCESSFUL
elif successful == 0:
status = MonitoringRunStatus.FAILED
else:
status = MonitoringRunStatus.PARTIAL
# Generate a baseline report the first time this company has any
# evidence at all, and an update report whenever a run actually
# detected something new - never regenerate on a no-op run (keeps
# LLM usage proportional to real activity, not to schedule cadence).
report_repo = ReportRepository(db)
existing_report_count = await report_repo.count_for_company(company.id)
report_type = None
if existing_report_count == 0 and items_collected > 0:
report_type = ReportType.BASELINE
elif changes_detected > 0:
report_type = ReportType.UPDATE
if report_type is not None:
try:
await report_service.generate_and_persist_report(
db, settings, llm, company, report_type=report_type, monitoring_run_id=run.id
)
logger.info(
"report_generated", company_id=str(company.id), report_type=report_type.value
)
except (
Exception
) as exc: # pragma: no cover - defensive, report failure shouldn't fail the run
logger.error("report_generation_failed", company_id=str(company.id), error=str(exc))
# last_run reflects any check; next_run (the schedule cadence) only
# advances for runs the schedule itself fired - a manual "run now"
# must not disrupt the next scheduled run (see spec + ARCHITECTURE.md).
# Applied unconditionally when config exists - it also carries each
# source's own next_check advance from the collection loop above.
if config is not None:
config.last_run = datetime.now(UTC)
if run.trigger_type == MonitoringRunTrigger.SCHEDULED:
config.next_run = validate_and_compute_next_run(
frequency_type=config.frequency_type,
interval_minutes=config.interval_minutes,
cron_expression=config.cron_expression,
tz_name=config.timezone,
minimum_interval_minutes=settings.minimum_monitoring_interval_minutes,
)
# Marked finished last, and deliberately in the same commit as the
# report/config work above: the frontend polls run.status to decide
# when to stop showing "Running..." - flipping it to a terminal
# status any earlier would let that indicator (and the Latest
# report/Sources/Monitoring history/Snapshots/Overview tabs it
# gates) go stale while report generation and schedule bookkeeping
# are still in flight.
await run_repo.mark_finished(
run, status=status, error_summary="; ".join(errors[:5]) or None
)
logger.info(
"monitoring_run_finished",
run_id=run_id,
status=status.value,
sources_successful=successful,
sources_failed=failed,
items_collected=items_collected,
)
apps/api/app/tasks/enrichment.py
+55
View File
@@ -0,0 +1,55 @@
"""Celery task that runs company enrichment exactly once, right after a
company is created (see company_service.create_company, which only ever
enqueues this when NINJAPEAR_API_KEY is configured). Generous time limits
since some NinjaPear endpoints are documented as taking up to 5 minutes,
and a single company can trigger the company-level calls plus several
capped per-leadership-member lookups in series.
"""
from __future__ import annotations
import uuid
from structlog.contextvars import bound_contextvars
from app.core.config import get_settings
from app.core.logging import get_logger
from app.db.session import get_sessionmaker
from app.enrichment.factory import get_enrichment_provider
from app.repositories.company_repository import CompanyRepository
from app.services import user_api_key_service
from app.services.enrichment_service import enrich_company as enrich_company_service
from app.tasks.base import run_async_task
from app.tasks.celery_app import celery_app
logger = get_logger(__name__)
@celery_app.task(
bind=True,
name="app.tasks.enrichment.enrich_company",
max_retries=1,
soft_time_limit=1500,
time_limit=1600,
)
def enrich_company(self, company_id: str) -> None:
with bound_contextvars(task_id=self.request.id, company_id=company_id):
run_async_task(_enrich_company_async(company_id))
async def _enrich_company_async(company_id: str) -> None:
settings = get_settings()
session_factory = get_sessionmaker()
async with session_factory() as db:
company = await CompanyRepository(db).get_by_id(uuid.UUID(company_id))
if company is None:
logger.warning("enrichment_company_not_found", company_id=company_id)
return
settings = await user_api_key_service.get_effective_settings(db, company.user_id, settings)
provider = get_enrichment_provider(settings)
logger.info(
"company_enrichment_started", company_id=company_id, provider=provider.provider_name
)
await enrich_company_service(db, settings, provider, company)
apps/api/app/tasks/maintenance.py
+43
View File
@@ -0,0 +1,43 @@
"""Celery Beat-triggered housekeeping. Currently just the data-retention
purge (DATA_RETENTION_DAYS) - only SourceDocument rows are ever deleted
here; see SourceDocumentRepository.delete_older_than for why that's the
only safe target in the current FK graph.
"""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from structlog.contextvars import bound_contextvars
from app.core.config import get_settings
from app.core.logging import get_logger
from app.db.session import get_sessionmaker
from app.repositories.source_repository import SourceDocumentRepository
from app.tasks.base import run_async_task
from app.tasks.celery_app import celery_app
logger = get_logger(__name__)
@celery_app.task(bind=True, name="app.tasks.maintenance.purge_expired_data")
def purge_expired_data(self) -> None:
with bound_contextvars(task_id=self.request.id):
run_async_task(_purge_expired_data_async())
async def _purge_expired_data_async() -> None:
settings = get_settings()
cutoff = datetime.now(UTC) - timedelta(days=settings.data_retention_days)
session_factory = get_sessionmaker()
async with session_factory() as db:
deleted = await SourceDocumentRepository(db).delete_older_than(cutoff)
await db.commit()
logger.info(
"data_retention_purge_completed",
deleted_source_documents=deleted,
retention_days=settings.data_retention_days,
cutoff=cutoff.isoformat(),
)
apps/api/app/tasks/scheduler.py
+73
View File
@@ -0,0 +1,73 @@
"""Celery Beat-triggered task. Runs every minute (see celery_app.py's
beat_schedule) and dynamically discovers which companies are due for a
check by querying MonitorConfiguration/Source directly - adding a new
company never requires touching this task or any static schedule config.
A company is due either because its own default MonitorConfiguration.next_run
has arrived, or because at least one of its sources has its own faster
frequency override that's independently due (see
SourceRepository.list_due_for_company) - e.g. a "check news daily" source
inside a company whose overall default cadence is weekly. Either way,
run_monitoring only actually collects the sources that are themselves due
for a SCHEDULED trigger - see tasks/collection.py.
"""
from __future__ import annotations
from datetime import UTC, datetime
from structlog.contextvars import bound_contextvars
from app.core.logging import get_logger
from app.db.session import get_sessionmaker
from app.models.enums import MonitoringRunTrigger
from app.repositories.company_repository import MonitorConfigurationRepository
from app.repositories.monitoring_run_repository import MonitoringRunRepository
from app.repositories.source_repository import SourceRepository
from app.tasks.base import run_async_task
from app.tasks.celery_app import celery_app
logger = get_logger(__name__)
@celery_app.task(bind=True, name="app.tasks.scheduler.sync_schedules")
def sync_schedules(self) -> None:
with bound_contextvars(task_id=self.request.id):
run_async_task(_sync_schedules_async())
async def _sync_schedules_async() -> None:
from app.tasks.collection import (
run_monitoring,
) # local import: avoids a circular import at module load
session_factory = get_sessionmaker()
async with session_factory() as db:
monitor_repo = MonitorConfigurationRepository(db)
source_repo = SourceRepository(db)
run_repo = MonitoringRunRepository(db)
now = datetime.now(UTC)
enabled_configs = await monitor_repo.list_enabled()
enqueued = 0
for config in enabled_configs:
is_due = await source_repo.company_has_due_work(config.company_id, now, config.next_run)
if not is_due:
continue
# Idempotency: don't double-enqueue a company that's still
# mid-run from a previous tick or a manual "run now".
active_run = await run_repo.get_active_for_company(config.company_id)
if active_run is not None:
continue
run = await run_repo.create(
company_id=config.company_id, trigger_type=MonitoringRunTrigger.SCHEDULED
)
await db.commit()
run_monitoring.delay(str(run.id))
enqueued += 1
if enqueued:
logger.info("schedule_sync_enqueued_runs", count=enqueued)