"""Collector interface. Every source type (website, RSS, SEC EDGAR, GitHub, custom URL, job postings, and the fixture-backed patent/review adapters) implements this same `SourceCollector` protocol, so `tasks/collection.py` (Phase 5) can treat them uniformly. Collectors never talk to the database - they take plain dataclasses in and return plain dataclasses out. Persisting `CollectedDocument`s into `SourceDocument` rows is the caller's job (a service function, not the collector), which keeps collectors trivially unit-testable against fixtures. """ from __future__ import annotations from dataclasses import dataclass, field from datetime import datetime from typing import Any, Protocol from app.models.enums import SourceStatus, SourceType @dataclass(frozen=True) class CompanyContext: """Read-only view of a Company, passed into collectors instead of the ORM object.""" id: str name: str official_website: str | None monitoring_focus: str | None aliases: list[str] = field(default_factory=list) competitors: list[str] = field(default_factory=list) # From NinjaPear enrichment (app/models/company_enrichment.py), when it # ran and found a leadership team - empty otherwise (no key, still # pending, or no leadership data). Used by PatentSourceCollector to # search USPTO by inventor name, since that endpoint has no queryable # company/assignee field at all - see collectors/patents.py. leadership_names: list[str] = field(default_factory=list) # The owning user's effective USPTO key (their own, or the server's # global one) - None means "use the server's global settings.uspto_api_key # directly", for call sites that never resolved a per-user override # (e.g. discovery preview paths outside a monitoring run). uspto_api_key: str | None = None @dataclass(frozen=True) class DiscoveredSource: source_type: SourceType name: str base_url: str | None configuration_metadata: dict[str, Any] = field(default_factory=dict) @dataclass(frozen=True) class SourceConfig: id: str source_type: SourceType name: str base_url: str | None configuration_metadata: dict[str, Any] = field(default_factory=dict) @dataclass(frozen=True) class CollectedDocument: url: str canonical_url: str title: str | None author: str | None publication_date: datetime | None retrieved_date: datetime content_text: str content_hash: str metadata: dict[str, Any] = field(default_factory=dict) language: str | None = None http_status: int | None = None extraction_method: str = "unknown" trust_score: float = 0.7 @dataclass class CollectionResult: status: SourceStatus documents: list[CollectedDocument] = field(default_factory=list) error: str | None = None pages_attempted: int = 0 class SourceCollector(Protocol): source_type: SourceType async def discover(self, company: CompanyContext) -> list[DiscoveredSource]: """Suggest sources for a newly added company. May return an empty list if this collector type can't be auto-discovered (e.g. patents).""" ... async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult: """Fetch and extract current content for a configured source.""" ...