"""Celery task that executes one MonitoringRun: discovers sources on a company's first run, collects every active source, and records progress on the MonitoringRun row as it goes so the frontend can poll live status. """ from __future__ import annotations import uuid from datetime import UTC, datetime from structlog.contextvars import bound_contextvars from app.analysis.llm.factory import get_llm_provider from app.core.config import get_settings from app.core.logging import get_logger from app.db.session import get_sessionmaker from app.models.enums import MonitoringRunStatus, MonitoringRunTrigger, ReportType, SourceStatus from app.repositories.company_repository import CompanyRepository from app.repositories.monitoring_run_repository import MonitoringRunRepository from app.repositories.report_repository import ReportRepository from app.repositories.source_repository import SnapshotRepository, SourceRepository from app.services import ( alert_service, change_detection_service, collection_service, report_service, user_api_key_service, ) from app.services.scheduling import validate_and_compute_next_run from app.tasks.base import run_async_task from app.tasks.celery_app import celery_app logger = get_logger(__name__) @celery_app.task(bind=True, name="app.tasks.collection.run_monitoring", max_retries=2) def run_monitoring(self, run_id: str) -> None: run_async_task(_run_monitoring_async(run_id, self.request.id)) async def _run_monitoring_async(run_id: str, task_id: str | None) -> None: with bound_contextvars(run_id=run_id, task_id=task_id): await _run_monitoring(run_id, task_id) async def _run_monitoring(run_id: str, task_id: str | None) -> None: settings = get_settings() session_factory = get_sessionmaker() async with session_factory() as db: run_repo = MonitoringRunRepository(db) run = await run_repo.get(uuid.UUID(run_id)) if run is None: logger.warning("monitoring_run_not_found", run_id=run_id) return company_repo = CompanyRepository(db) company = await company_repo.get_by_id(run.company_id) if company is None: await run_repo.mark_finished( run, status=MonitoringRunStatus.FAILED, error_summary="Company no longer exists" ) return await run_repo.set_worker_task_id(run, task_id) await run_repo.mark_running(run) logger.info( "monitoring_run_started", run_id=run_id, company_id=str(company.id), trigger=run.trigger_type.value, ) settings = await user_api_key_service.get_effective_settings(db, company.user_id, settings) llm = get_llm_provider(settings) source_repo = SourceRepository(db) snapshot_repo = SnapshotRepository(db) sources = await source_repo.list_for_company(company.id) if not sources: try: await collection_service.discover_sources_for_company(db, company, settings) except Exception as exc: # pragma: no cover - defensive, discovery is best-effort logger.warning( "discovery_failed_during_run", company_id=str(company.id), error=str(exc) ) sources = await source_repo.list_for_company(company.id) now = datetime.now(UTC) config = company.monitor_configuration if run.trigger_type == MonitoringRunTrigger.SCHEDULED: # Only the sources actually due right now - a source with a # slower override (e.g. patents checked monthly) sits out a run # that only a faster sibling source (e.g. news checked daily) # triggered. See SourceRepository.list_due_for_company. company_next_run = config.next_run if config is not None else None active_sources = await source_repo.list_due_for_company( company.id, now, company_next_run ) else: # Manual "Run now" is always a full on-demand check of every # active source, regardless of any source's individual cadence. active_sources = [s for s in sources if s.active] items_collected = 0 successful = 0 failed = 0 changes_detected = 0 errors: list[str] = [] for source in active_sources: try: result = await collection_service.collect_source( db, settings, source, company, monitoring_run_id=run.id ) items_collected += len(result.documents) # Advance this source's own clock only if it has an override # - a source with no override has no next_check of its own # (it rides the company's next_run instead, advanced below). # Guarded separately from the collection result above: a # scheduling computation issue must never get reported as a # collection failure for a source that actually succeeded. if source.frequency_type is not None: try: tz_name = config.timezone if config is not None else "UTC" source.next_check = validate_and_compute_next_run( frequency_type=source.frequency_type, interval_minutes=source.interval_minutes, cron_expression=source.cron_expression, tz_name=tz_name, minimum_interval_minutes=settings.minimum_monitoring_interval_minutes, from_time=now, ) except Exception as exc: # pragma: no cover - defensive logger.warning( "source_next_check_computation_failed", source_id=str(source.id), error=str(exc), ) if result.status == SourceStatus.ACTIVE: successful += 1 else: failed += 1 if result.error: errors.append(f"{source.name}: {result.error}") if result.documents: current_snapshot = await snapshot_repo.latest_for_source(source.id) if current_snapshot is not None: change = await change_detection_service.detect_change_for_source( db, source, company, current_snapshot, run.id ) if change is not None: changes_detected += 1 logger.info( "change_detected", source_id=str(source.id), change_type=change.change_type.value, severity=change.severity.value, significance=change.significance_score, confidence=change.confidence_score, ) try: alert = await alert_service.create_alert_for_change( db, settings, llm, change, company ) if alert is not None: logger.info( "alert_created", alert_id=str(alert.id), severity=alert.severity.value, ) except Exception as exc: # pragma: no cover - defensive, alerting failure shouldn't fail the run logger.error( "alert_creation_failed", change_id=str(change.id), error=str(exc), ) except ( Exception ) as exc: # pragma: no cover - defensive, one bad source shouldn't kill the run failed += 1 errors.append(f"{source.name}: {exc}") logger.error( "source_collection_unexpected_error", source_id=str(source.id), error=str(exc) ) await run_repo.update_progress( run, sources_attempted=successful + failed, sources_successful=successful, sources_failed=failed, items_collected=items_collected, changes_detected=changes_detected, ) if not active_sources or failed == 0: status = MonitoringRunStatus.SUCCESSFUL elif successful == 0: status = MonitoringRunStatus.FAILED else: status = MonitoringRunStatus.PARTIAL # Generate a baseline report the first time this company has any # evidence at all, and an update report whenever a run actually # detected something new - never regenerate on a no-op run (keeps # LLM usage proportional to real activity, not to schedule cadence). report_repo = ReportRepository(db) existing_report_count = await report_repo.count_for_company(company.id) report_type = None if existing_report_count == 0 and items_collected > 0: report_type = ReportType.BASELINE elif changes_detected > 0: report_type = ReportType.UPDATE if report_type is not None: try: await report_service.generate_and_persist_report( db, settings, llm, company, report_type=report_type, monitoring_run_id=run.id ) logger.info( "report_generated", company_id=str(company.id), report_type=report_type.value ) except ( Exception ) as exc: # pragma: no cover - defensive, report failure shouldn't fail the run logger.error("report_generation_failed", company_id=str(company.id), error=str(exc)) # last_run reflects any check; next_run (the schedule cadence) only # advances for runs the schedule itself fired - a manual "run now" # must not disrupt the next scheduled run (see spec + ARCHITECTURE.md). # Applied unconditionally when config exists - it also carries each # source's own next_check advance from the collection loop above. if config is not None: config.last_run = datetime.now(UTC) if run.trigger_type == MonitoringRunTrigger.SCHEDULED: config.next_run = validate_and_compute_next_run( frequency_type=config.frequency_type, interval_minutes=config.interval_minutes, cron_expression=config.cron_expression, tz_name=config.timezone, minimum_interval_minutes=settings.minimum_monitoring_interval_minutes, ) # Marked finished last, and deliberately in the same commit as the # report/config work above: the frontend polls run.status to decide # when to stop showing "Running..." - flipping it to a terminal # status any earlier would let that indicator (and the Latest # report/Sources/Monitoring history/Snapshots/Overview tabs it # gates) go stale while report generation and schedule bookkeeping # are still in flight. await run_repo.mark_finished( run, status=status, error_summary="; ".join(errors[:5]) or None ) logger.info( "monitoring_run_finished", run_id=run_id, status=status.value, sources_successful=successful, sources_failed=failed, items_collected=items_collected, )