CIAgent/apps/api/app/collectors/sec_edgar.py

"""SEC EDGAR collector for US public companies.

No API key required, but SEC asks that callers identify themselves with a
descriptive User-Agent (see `SCRAPER_USER_AGENT` in .env.example) and stay
within its rate limits - the shared `safe_fetch` per-domain delay covers
that. We store filing *metadata* (form type, date, accession number, link)
rather than parsing full filing bodies, which is out of scope for this pass.
"""

from __future__ import annotations

import json
from datetime import UTC, datetime
from xml.etree import ElementTree

import httpx

from app.collectors.base import (
    CollectedDocument,
    CollectionResult,
    CompanyContext,
    DiscoveredSource,
    SourceConfig,
)
from app.collectors.extraction import compute_content_hash, normalize_whitespace
from app.core.config import get_settings
from app.core.http import FetchError, SsrfBlockedError, fetch_with_retries
from app.core.logging import get_logger
from app.models.enums import SourceStatus, SourceType

logger = get_logger(__name__)

_RELEVANT_FORMS = {"10-K", "10-Q", "8-K"}
_SEARCH_URL = "https://www.sec.gov/cgi-bin/browse-edgar"
_SUBMISSIONS_URL = "https://data.sec.gov/submissions/CIK{cik}.json"


class SecEdgarCollector:
    source_type = SourceType.SEC_EDGAR

    async def discover(self, company: CompanyContext) -> list[DiscoveredSource]:
        settings = get_settings()
        cik = await self._lookup_cik(company.name, settings)
        if cik is None:
            return []
        return [
            DiscoveredSource(
                source_type=SourceType.SEC_EDGAR,
                name=f"{company.name} — SEC EDGAR filings",
                base_url=f"https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={cik}",
                configuration_metadata={"cik": cik},
            )
        ]

    async def _lookup_cik(self, company_name: str, settings) -> str | None:
        params = {
            "action": "getcompany",
            "company": company_name,
            "type": "10-K",
            "dateb": "",
            "owner": "include",
            "count": "5",
            "output": "atom",
        }
        url = str(httpx.URL(_SEARCH_URL, params=params))
        try:
            result = await fetch_with_retries(url, settings=settings, max_attempts=2)
        except (SsrfBlockedError, FetchError, httpx.HTTPError) as exc:
            logger.warning("sec_edgar_lookup_failed", company=company_name, error=str(exc))
            return None
        if result.status_code != 200:
            return None
        try:
            root = ElementTree.fromstring(result.content)
        except ElementTree.ParseError:
            return None

        ns = {"a": "http://www.w3.org/2005/Atom"}
        for entry in root.findall(".//a:entry", ns):
            cik_elem = entry.find("a:content", ns)
            title_elem = entry.find("a:title", ns)
            if cik_elem is None or title_elem is None:
                continue
            # The atom feed embeds "CIK=0000320193" style text in <content>.
            text = "".join(cik_elem.itertext())
            if "CIK=" in text:
                cik = text.split("CIK=")[1].split("&")[0].strip()
                return cik.zfill(10)
        return None

    async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult:
        cik = source.configuration_metadata.get("cik")
        if not cik:
            return CollectionResult(status=SourceStatus.FAILED, error="No CIK configured")

        settings = get_settings()
        url = _SUBMISSIONS_URL.format(cik=str(cik).zfill(10))
        try:
            result = await fetch_with_retries(url, settings=settings)
        except SsrfBlockedError as exc:
            return CollectionResult(status=SourceStatus.BLOCKED_BY_POLICY, error=str(exc))
        except (FetchError, httpx.HTTPError) as exc:
            return CollectionResult(status=SourceStatus.FAILED, error=str(exc))

        if result.status_code == 404:
            return CollectionResult(status=SourceStatus.FAILED, error="CIK not found on EDGAR")
        if result.status_code >= 400:
            return CollectionResult(status=SourceStatus.FAILED, error=f"HTTP {result.status_code}")

        try:
            payload = json.loads(result.text)
        except json.JSONDecodeError:
            return CollectionResult(status=SourceStatus.FAILED, error="Malformed EDGAR response")

        recent = payload.get("filings", {}).get("recent", {})
        forms = recent.get("form", [])
        dates = recent.get("filingDate", [])
        accessions = recent.get("accessionNumber", [])
        primary_docs = recent.get("primaryDocument", [])
        company_name = payload.get("name", company.name)

        documents: list[CollectedDocument] = []
        for i, form in enumerate(forms):
            if form not in _RELEVANT_FORMS:
                continue
            if len(documents) >= 10:
                break
            accession = accessions[i].replace("-", "") if i < len(accessions) else ""
            primary_doc = primary_docs[i] if i < len(primary_docs) else ""
            filing_date = dates[i] if i < len(dates) else ""
            filing_url = (
                f"https://www.sec.gov/Archives/edgar/data/{int(cik)}/{accession}/{primary_doc}"
                if accession and primary_doc
                else url
            )
            text = normalize_whitespace(
                f"{company_name} filed a {form} with the SEC on {filing_date}.\n"
                f"Accession number: {accessions[i] if i < len(accessions) else 'unknown'}.\n"
                f"Filing document: {filing_url}"
            )
            documents.append(
                CollectedDocument(
                    url=filing_url,
                    canonical_url=filing_url,
                    title=f"{company_name} {form} ({filing_date})",
                    author="SEC EDGAR",
                    publication_date=(
                        datetime.strptime(filing_date, "%Y-%m-%d").replace(tzinfo=UTC)
                        if filing_date
                        else None
                    ),
                    retrieved_date=datetime.now(UTC),
                    content_text=text,
                    content_hash=compute_content_hash(text),
                    metadata={
                        "form": form,
                        "accession_number": accessions[i] if i < len(accessions) else None,
                    },
                    extraction_method="sec_edgar_metadata",
                    http_status=result.status_code,
                    trust_score=0.95,
                )
            )

        # Zero relevant filings isn't a failure - the company may simply have
        # none in its recent filing history.
        return CollectionResult(status=SourceStatus.ACTIVE, documents=documents, pages_attempted=1)