Initial commit: CI Agent competitive-intelligence monitoring app

FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring,
source collection, LLM-based change analysis, enrichment, and account
security (Turnstile, escalating lockout, email verification).

apps/api/app/collectors/base.py

@@ -0,0 +1,96 @@
+"""Collector interface. Every source type (website, RSS, SEC EDGAR, GitHub,
+custom URL, job postings, and the fixture-backed patent/review adapters)
+implements this same `SourceCollector` protocol, so `tasks/collection.py`
+(Phase 5) can treat them uniformly.
+
+Collectors never talk to the database - they take plain dataclasses in and
+return plain dataclasses out. Persisting `CollectedDocument`s into
+`SourceDocument` rows is the caller's job (a service function, not the
+collector), which keeps collectors trivially unit-testable against fixtures.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Protocol
+
+from app.models.enums import SourceStatus, SourceType
+
+
+@dataclass(frozen=True)
+class CompanyContext:
+    """Read-only view of a Company, passed into collectors instead of the ORM object."""
+
+    id: str
+    name: str
+    official_website: str | None
+    monitoring_focus: str | None
+    aliases: list[str] = field(default_factory=list)
+    competitors: list[str] = field(default_factory=list)
+    # From NinjaPear enrichment (app/models/company_enrichment.py), when it
+    # ran and found a leadership team - empty otherwise (no key, still
+    # pending, or no leadership data). Used by PatentSourceCollector to
+    # search USPTO by inventor name, since that endpoint has no queryable
+    # company/assignee field at all - see collectors/patents.py.
+    leadership_names: list[str] = field(default_factory=list)
+    # The owning user's effective USPTO key (their own, or the server's
+    # global one) - None means "use the server's global settings.uspto_api_key
+    # directly", for call sites that never resolved a per-user override
+    # (e.g. discovery preview paths outside a monitoring run).
+    uspto_api_key: str | None = None
+
+
+@dataclass(frozen=True)
+class DiscoveredSource:
+    source_type: SourceType
+    name: str
+    base_url: str | None
+    configuration_metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class SourceConfig:
+    id: str
+    source_type: SourceType
+    name: str
+    base_url: str | None
+    configuration_metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class CollectedDocument:
+    url: str
+    canonical_url: str
+    title: str | None
+    author: str | None
+    publication_date: datetime | None
+    retrieved_date: datetime
+    content_text: str
+    content_hash: str
+    metadata: dict[str, Any] = field(default_factory=dict)
+    language: str | None = None
+    http_status: int | None = None
+    extraction_method: str = "unknown"
+    trust_score: float = 0.7
+
+
+@dataclass
+class CollectionResult:
+    status: SourceStatus
+    documents: list[CollectedDocument] = field(default_factory=list)
+    error: str | None = None
+    pages_attempted: int = 0
+
+
+class SourceCollector(Protocol):
+    source_type: SourceType
+
+    async def discover(self, company: CompanyContext) -> list[DiscoveredSource]:
+        """Suggest sources for a newly added company. May return an empty
+        list if this collector type can't be auto-discovered (e.g. patents)."""
+        ...
+
+    async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult:
+        """Fetch and extract current content for a configured source."""
+        ...