saksham
1a4c80958f
FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification).
266 lines
12 KiB
Python
266 lines
12 KiB
Python
"""Celery task that executes one MonitoringRun: discovers sources on a
|
|
company's first run, collects every active source, and records progress on
|
|
the MonitoringRun row as it goes so the frontend can poll live status.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import uuid
|
|
from datetime import UTC, datetime
|
|
|
|
from structlog.contextvars import bound_contextvars
|
|
|
|
from app.analysis.llm.factory import get_llm_provider
|
|
from app.core.config import get_settings
|
|
from app.core.logging import get_logger
|
|
from app.db.session import get_sessionmaker
|
|
from app.models.enums import MonitoringRunStatus, MonitoringRunTrigger, ReportType, SourceStatus
|
|
from app.repositories.company_repository import CompanyRepository
|
|
from app.repositories.monitoring_run_repository import MonitoringRunRepository
|
|
from app.repositories.report_repository import ReportRepository
|
|
from app.repositories.source_repository import SnapshotRepository, SourceRepository
|
|
from app.services import (
|
|
alert_service,
|
|
change_detection_service,
|
|
collection_service,
|
|
report_service,
|
|
user_api_key_service,
|
|
)
|
|
from app.services.scheduling import validate_and_compute_next_run
|
|
from app.tasks.base import run_async_task
|
|
from app.tasks.celery_app import celery_app
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
|
|
@celery_app.task(bind=True, name="app.tasks.collection.run_monitoring", max_retries=2)
|
|
def run_monitoring(self, run_id: str) -> None:
|
|
run_async_task(_run_monitoring_async(run_id, self.request.id))
|
|
|
|
|
|
async def _run_monitoring_async(run_id: str, task_id: str | None) -> None:
|
|
with bound_contextvars(run_id=run_id, task_id=task_id):
|
|
await _run_monitoring(run_id, task_id)
|
|
|
|
|
|
async def _run_monitoring(run_id: str, task_id: str | None) -> None:
|
|
settings = get_settings()
|
|
session_factory = get_sessionmaker()
|
|
|
|
async with session_factory() as db:
|
|
run_repo = MonitoringRunRepository(db)
|
|
run = await run_repo.get(uuid.UUID(run_id))
|
|
if run is None:
|
|
logger.warning("monitoring_run_not_found", run_id=run_id)
|
|
return
|
|
|
|
company_repo = CompanyRepository(db)
|
|
company = await company_repo.get_by_id(run.company_id)
|
|
if company is None:
|
|
await run_repo.mark_finished(
|
|
run, status=MonitoringRunStatus.FAILED, error_summary="Company no longer exists"
|
|
)
|
|
return
|
|
|
|
await run_repo.set_worker_task_id(run, task_id)
|
|
await run_repo.mark_running(run)
|
|
logger.info(
|
|
"monitoring_run_started",
|
|
run_id=run_id,
|
|
company_id=str(company.id),
|
|
trigger=run.trigger_type.value,
|
|
)
|
|
|
|
settings = await user_api_key_service.get_effective_settings(db, company.user_id, settings)
|
|
llm = get_llm_provider(settings)
|
|
source_repo = SourceRepository(db)
|
|
snapshot_repo = SnapshotRepository(db)
|
|
sources = await source_repo.list_for_company(company.id)
|
|
|
|
if not sources:
|
|
try:
|
|
await collection_service.discover_sources_for_company(db, company, settings)
|
|
except Exception as exc: # pragma: no cover - defensive, discovery is best-effort
|
|
logger.warning(
|
|
"discovery_failed_during_run", company_id=str(company.id), error=str(exc)
|
|
)
|
|
sources = await source_repo.list_for_company(company.id)
|
|
|
|
now = datetime.now(UTC)
|
|
config = company.monitor_configuration
|
|
if run.trigger_type == MonitoringRunTrigger.SCHEDULED:
|
|
# Only the sources actually due right now - a source with a
|
|
# slower override (e.g. patents checked monthly) sits out a run
|
|
# that only a faster sibling source (e.g. news checked daily)
|
|
# triggered. See SourceRepository.list_due_for_company.
|
|
company_next_run = config.next_run if config is not None else None
|
|
active_sources = await source_repo.list_due_for_company(
|
|
company.id, now, company_next_run
|
|
)
|
|
else:
|
|
# Manual "Run now" is always a full on-demand check of every
|
|
# active source, regardless of any source's individual cadence.
|
|
active_sources = [s for s in sources if s.active]
|
|
|
|
items_collected = 0
|
|
successful = 0
|
|
failed = 0
|
|
changes_detected = 0
|
|
errors: list[str] = []
|
|
|
|
for source in active_sources:
|
|
try:
|
|
result = await collection_service.collect_source(
|
|
db, settings, source, company, monitoring_run_id=run.id
|
|
)
|
|
items_collected += len(result.documents)
|
|
|
|
# Advance this source's own clock only if it has an override
|
|
# - a source with no override has no next_check of its own
|
|
# (it rides the company's next_run instead, advanced below).
|
|
# Guarded separately from the collection result above: a
|
|
# scheduling computation issue must never get reported as a
|
|
# collection failure for a source that actually succeeded.
|
|
if source.frequency_type is not None:
|
|
try:
|
|
tz_name = config.timezone if config is not None else "UTC"
|
|
source.next_check = validate_and_compute_next_run(
|
|
frequency_type=source.frequency_type,
|
|
interval_minutes=source.interval_minutes,
|
|
cron_expression=source.cron_expression,
|
|
tz_name=tz_name,
|
|
minimum_interval_minutes=settings.minimum_monitoring_interval_minutes,
|
|
from_time=now,
|
|
)
|
|
except Exception as exc: # pragma: no cover - defensive
|
|
logger.warning(
|
|
"source_next_check_computation_failed",
|
|
source_id=str(source.id),
|
|
error=str(exc),
|
|
)
|
|
|
|
if result.status == SourceStatus.ACTIVE:
|
|
successful += 1
|
|
else:
|
|
failed += 1
|
|
if result.error:
|
|
errors.append(f"{source.name}: {result.error}")
|
|
|
|
if result.documents:
|
|
current_snapshot = await snapshot_repo.latest_for_source(source.id)
|
|
if current_snapshot is not None:
|
|
change = await change_detection_service.detect_change_for_source(
|
|
db, source, company, current_snapshot, run.id
|
|
)
|
|
if change is not None:
|
|
changes_detected += 1
|
|
logger.info(
|
|
"change_detected",
|
|
source_id=str(source.id),
|
|
change_type=change.change_type.value,
|
|
severity=change.severity.value,
|
|
significance=change.significance_score,
|
|
confidence=change.confidence_score,
|
|
)
|
|
try:
|
|
alert = await alert_service.create_alert_for_change(
|
|
db, settings, llm, change, company
|
|
)
|
|
if alert is not None:
|
|
logger.info(
|
|
"alert_created",
|
|
alert_id=str(alert.id),
|
|
severity=alert.severity.value,
|
|
)
|
|
except Exception as exc: # pragma: no cover - defensive, alerting failure shouldn't fail the run
|
|
logger.error(
|
|
"alert_creation_failed",
|
|
change_id=str(change.id),
|
|
error=str(exc),
|
|
)
|
|
except (
|
|
Exception
|
|
) as exc: # pragma: no cover - defensive, one bad source shouldn't kill the run
|
|
failed += 1
|
|
errors.append(f"{source.name}: {exc}")
|
|
logger.error(
|
|
"source_collection_unexpected_error", source_id=str(source.id), error=str(exc)
|
|
)
|
|
|
|
await run_repo.update_progress(
|
|
run,
|
|
sources_attempted=successful + failed,
|
|
sources_successful=successful,
|
|
sources_failed=failed,
|
|
items_collected=items_collected,
|
|
changes_detected=changes_detected,
|
|
)
|
|
|
|
if not active_sources or failed == 0:
|
|
status = MonitoringRunStatus.SUCCESSFUL
|
|
elif successful == 0:
|
|
status = MonitoringRunStatus.FAILED
|
|
else:
|
|
status = MonitoringRunStatus.PARTIAL
|
|
|
|
# Generate a baseline report the first time this company has any
|
|
# evidence at all, and an update report whenever a run actually
|
|
# detected something new - never regenerate on a no-op run (keeps
|
|
# LLM usage proportional to real activity, not to schedule cadence).
|
|
report_repo = ReportRepository(db)
|
|
existing_report_count = await report_repo.count_for_company(company.id)
|
|
report_type = None
|
|
if existing_report_count == 0 and items_collected > 0:
|
|
report_type = ReportType.BASELINE
|
|
elif changes_detected > 0:
|
|
report_type = ReportType.UPDATE
|
|
|
|
if report_type is not None:
|
|
try:
|
|
await report_service.generate_and_persist_report(
|
|
db, settings, llm, company, report_type=report_type, monitoring_run_id=run.id
|
|
)
|
|
logger.info(
|
|
"report_generated", company_id=str(company.id), report_type=report_type.value
|
|
)
|
|
except (
|
|
Exception
|
|
) as exc: # pragma: no cover - defensive, report failure shouldn't fail the run
|
|
logger.error("report_generation_failed", company_id=str(company.id), error=str(exc))
|
|
|
|
# last_run reflects any check; next_run (the schedule cadence) only
|
|
# advances for runs the schedule itself fired - a manual "run now"
|
|
# must not disrupt the next scheduled run (see spec + ARCHITECTURE.md).
|
|
# Applied unconditionally when config exists - it also carries each
|
|
# source's own next_check advance from the collection loop above.
|
|
if config is not None:
|
|
config.last_run = datetime.now(UTC)
|
|
if run.trigger_type == MonitoringRunTrigger.SCHEDULED:
|
|
config.next_run = validate_and_compute_next_run(
|
|
frequency_type=config.frequency_type,
|
|
interval_minutes=config.interval_minutes,
|
|
cron_expression=config.cron_expression,
|
|
tz_name=config.timezone,
|
|
minimum_interval_minutes=settings.minimum_monitoring_interval_minutes,
|
|
)
|
|
|
|
# Marked finished last, and deliberately in the same commit as the
|
|
# report/config work above: the frontend polls run.status to decide
|
|
# when to stop showing "Running..." - flipping it to a terminal
|
|
# status any earlier would let that indicator (and the Latest
|
|
# report/Sources/Monitoring history/Snapshots/Overview tabs it
|
|
# gates) go stale while report generation and schedule bookkeeping
|
|
# are still in flight.
|
|
await run_repo.mark_finished(
|
|
run, status=status, error_summary="; ".join(errors[:5]) or None
|
|
)
|
|
|
|
logger.info(
|
|
"monitoring_run_finished",
|
|
run_id=run_id,
|
|
status=status.value,
|
|
sources_successful=successful,
|
|
sources_failed=failed,
|
|
items_collected=items_collected,
|
|
)
|