Initial commit: CI Agent competitive-intelligence monitoring app
FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification).
This commit is contained in:
@@ -0,0 +1,167 @@
|
||||
"""SEC EDGAR collector for US public companies.
|
||||
|
||||
No API key required, but SEC asks that callers identify themselves with a
|
||||
descriptive User-Agent (see `SCRAPER_USER_AGENT` in .env.example) and stay
|
||||
within its rate limits - the shared `safe_fetch` per-domain delay covers
|
||||
that. We store filing *metadata* (form type, date, accession number, link)
|
||||
rather than parsing full filing bodies, which is out of scope for this pass.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from xml.etree import ElementTree
|
||||
|
||||
import httpx
|
||||
|
||||
from app.collectors.base import (
|
||||
CollectedDocument,
|
||||
CollectionResult,
|
||||
CompanyContext,
|
||||
DiscoveredSource,
|
||||
SourceConfig,
|
||||
)
|
||||
from app.collectors.extraction import compute_content_hash, normalize_whitespace
|
||||
from app.core.config import get_settings
|
||||
from app.core.http import FetchError, SsrfBlockedError, fetch_with_retries
|
||||
from app.core.logging import get_logger
|
||||
from app.models.enums import SourceStatus, SourceType
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
_RELEVANT_FORMS = {"10-K", "10-Q", "8-K"}
|
||||
_SEARCH_URL = "https://www.sec.gov/cgi-bin/browse-edgar"
|
||||
_SUBMISSIONS_URL = "https://data.sec.gov/submissions/CIK{cik}.json"
|
||||
|
||||
|
||||
class SecEdgarCollector:
|
||||
source_type = SourceType.SEC_EDGAR
|
||||
|
||||
async def discover(self, company: CompanyContext) -> list[DiscoveredSource]:
|
||||
settings = get_settings()
|
||||
cik = await self._lookup_cik(company.name, settings)
|
||||
if cik is None:
|
||||
return []
|
||||
return [
|
||||
DiscoveredSource(
|
||||
source_type=SourceType.SEC_EDGAR,
|
||||
name=f"{company.name} — SEC EDGAR filings",
|
||||
base_url=f"https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={cik}",
|
||||
configuration_metadata={"cik": cik},
|
||||
)
|
||||
]
|
||||
|
||||
async def _lookup_cik(self, company_name: str, settings) -> str | None:
|
||||
params = {
|
||||
"action": "getcompany",
|
||||
"company": company_name,
|
||||
"type": "10-K",
|
||||
"dateb": "",
|
||||
"owner": "include",
|
||||
"count": "5",
|
||||
"output": "atom",
|
||||
}
|
||||
url = str(httpx.URL(_SEARCH_URL, params=params))
|
||||
try:
|
||||
result = await fetch_with_retries(url, settings=settings, max_attempts=2)
|
||||
except (SsrfBlockedError, FetchError, httpx.HTTPError) as exc:
|
||||
logger.warning("sec_edgar_lookup_failed", company=company_name, error=str(exc))
|
||||
return None
|
||||
if result.status_code != 200:
|
||||
return None
|
||||
try:
|
||||
root = ElementTree.fromstring(result.content)
|
||||
except ElementTree.ParseError:
|
||||
return None
|
||||
|
||||
ns = {"a": "http://www.w3.org/2005/Atom"}
|
||||
for entry in root.findall(".//a:entry", ns):
|
||||
cik_elem = entry.find("a:content", ns)
|
||||
title_elem = entry.find("a:title", ns)
|
||||
if cik_elem is None or title_elem is None:
|
||||
continue
|
||||
# The atom feed embeds "CIK=0000320193" style text in <content>.
|
||||
text = "".join(cik_elem.itertext())
|
||||
if "CIK=" in text:
|
||||
cik = text.split("CIK=")[1].split("&")[0].strip()
|
||||
return cik.zfill(10)
|
||||
return None
|
||||
|
||||
async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult:
|
||||
cik = source.configuration_metadata.get("cik")
|
||||
if not cik:
|
||||
return CollectionResult(status=SourceStatus.FAILED, error="No CIK configured")
|
||||
|
||||
settings = get_settings()
|
||||
url = _SUBMISSIONS_URL.format(cik=str(cik).zfill(10))
|
||||
try:
|
||||
result = await fetch_with_retries(url, settings=settings)
|
||||
except SsrfBlockedError as exc:
|
||||
return CollectionResult(status=SourceStatus.BLOCKED_BY_POLICY, error=str(exc))
|
||||
except (FetchError, httpx.HTTPError) as exc:
|
||||
return CollectionResult(status=SourceStatus.FAILED, error=str(exc))
|
||||
|
||||
if result.status_code == 404:
|
||||
return CollectionResult(status=SourceStatus.FAILED, error="CIK not found on EDGAR")
|
||||
if result.status_code >= 400:
|
||||
return CollectionResult(status=SourceStatus.FAILED, error=f"HTTP {result.status_code}")
|
||||
|
||||
try:
|
||||
payload = json.loads(result.text)
|
||||
except json.JSONDecodeError:
|
||||
return CollectionResult(status=SourceStatus.FAILED, error="Malformed EDGAR response")
|
||||
|
||||
recent = payload.get("filings", {}).get("recent", {})
|
||||
forms = recent.get("form", [])
|
||||
dates = recent.get("filingDate", [])
|
||||
accessions = recent.get("accessionNumber", [])
|
||||
primary_docs = recent.get("primaryDocument", [])
|
||||
company_name = payload.get("name", company.name)
|
||||
|
||||
documents: list[CollectedDocument] = []
|
||||
for i, form in enumerate(forms):
|
||||
if form not in _RELEVANT_FORMS:
|
||||
continue
|
||||
if len(documents) >= 10:
|
||||
break
|
||||
accession = accessions[i].replace("-", "") if i < len(accessions) else ""
|
||||
primary_doc = primary_docs[i] if i < len(primary_docs) else ""
|
||||
filing_date = dates[i] if i < len(dates) else ""
|
||||
filing_url = (
|
||||
f"https://www.sec.gov/Archives/edgar/data/{int(cik)}/{accession}/{primary_doc}"
|
||||
if accession and primary_doc
|
||||
else url
|
||||
)
|
||||
text = normalize_whitespace(
|
||||
f"{company_name} filed a {form} with the SEC on {filing_date}.\n"
|
||||
f"Accession number: {accessions[i] if i < len(accessions) else 'unknown'}.\n"
|
||||
f"Filing document: {filing_url}"
|
||||
)
|
||||
documents.append(
|
||||
CollectedDocument(
|
||||
url=filing_url,
|
||||
canonical_url=filing_url,
|
||||
title=f"{company_name} {form} ({filing_date})",
|
||||
author="SEC EDGAR",
|
||||
publication_date=(
|
||||
datetime.strptime(filing_date, "%Y-%m-%d").replace(tzinfo=UTC)
|
||||
if filing_date
|
||||
else None
|
||||
),
|
||||
retrieved_date=datetime.now(UTC),
|
||||
content_text=text,
|
||||
content_hash=compute_content_hash(text),
|
||||
metadata={
|
||||
"form": form,
|
||||
"accession_number": accessions[i] if i < len(accessions) else None,
|
||||
},
|
||||
extraction_method="sec_edgar_metadata",
|
||||
http_status=result.status_code,
|
||||
trust_score=0.95,
|
||||
)
|
||||
)
|
||||
|
||||
# Zero relevant filings isn't a failure - the company may simply have
|
||||
# none in its recent filing history.
|
||||
return CollectionResult(status=SourceStatus.ACTIVE, documents=documents, pages_attempted=1)
|
||||
Reference in New Issue
Block a user