saksham
1a4c80958f
FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification).
97 lines
3.3 KiB
Python
97 lines
3.3 KiB
Python
"""Collector interface. Every source type (website, RSS, SEC EDGAR, GitHub,
|
|
custom URL, job postings, and the fixture-backed patent/review adapters)
|
|
implements this same `SourceCollector` protocol, so `tasks/collection.py`
|
|
(Phase 5) can treat them uniformly.
|
|
|
|
Collectors never talk to the database - they take plain dataclasses in and
|
|
return plain dataclasses out. Persisting `CollectedDocument`s into
|
|
`SourceDocument` rows is the caller's job (a service function, not the
|
|
collector), which keeps collectors trivially unit-testable against fixtures.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime
|
|
from typing import Any, Protocol
|
|
|
|
from app.models.enums import SourceStatus, SourceType
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CompanyContext:
|
|
"""Read-only view of a Company, passed into collectors instead of the ORM object."""
|
|
|
|
id: str
|
|
name: str
|
|
official_website: str | None
|
|
monitoring_focus: str | None
|
|
aliases: list[str] = field(default_factory=list)
|
|
competitors: list[str] = field(default_factory=list)
|
|
# From NinjaPear enrichment (app/models/company_enrichment.py), when it
|
|
# ran and found a leadership team - empty otherwise (no key, still
|
|
# pending, or no leadership data). Used by PatentSourceCollector to
|
|
# search USPTO by inventor name, since that endpoint has no queryable
|
|
# company/assignee field at all - see collectors/patents.py.
|
|
leadership_names: list[str] = field(default_factory=list)
|
|
# The owning user's effective USPTO key (their own, or the server's
|
|
# global one) - None means "use the server's global settings.uspto_api_key
|
|
# directly", for call sites that never resolved a per-user override
|
|
# (e.g. discovery preview paths outside a monitoring run).
|
|
uspto_api_key: str | None = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DiscoveredSource:
|
|
source_type: SourceType
|
|
name: str
|
|
base_url: str | None
|
|
configuration_metadata: dict[str, Any] = field(default_factory=dict)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class SourceConfig:
|
|
id: str
|
|
source_type: SourceType
|
|
name: str
|
|
base_url: str | None
|
|
configuration_metadata: dict[str, Any] = field(default_factory=dict)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CollectedDocument:
|
|
url: str
|
|
canonical_url: str
|
|
title: str | None
|
|
author: str | None
|
|
publication_date: datetime | None
|
|
retrieved_date: datetime
|
|
content_text: str
|
|
content_hash: str
|
|
metadata: dict[str, Any] = field(default_factory=dict)
|
|
language: str | None = None
|
|
http_status: int | None = None
|
|
extraction_method: str = "unknown"
|
|
trust_score: float = 0.7
|
|
|
|
|
|
@dataclass
|
|
class CollectionResult:
|
|
status: SourceStatus
|
|
documents: list[CollectedDocument] = field(default_factory=list)
|
|
error: str | None = None
|
|
pages_attempted: int = 0
|
|
|
|
|
|
class SourceCollector(Protocol):
|
|
source_type: SourceType
|
|
|
|
async def discover(self, company: CompanyContext) -> list[DiscoveredSource]:
|
|
"""Suggest sources for a newly added company. May return an empty
|
|
list if this collector type can't be auto-discovered (e.g. patents)."""
|
|
...
|
|
|
|
async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult:
|
|
"""Fetch and extract current content for a configured source."""
|
|
...
|