Initial commit: CI Agent competitive-intelligence monitoring app
FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification).
This commit is contained in:
@@ -0,0 +1,96 @@
|
||||
"""Collector interface. Every source type (website, RSS, SEC EDGAR, GitHub,
|
||||
custom URL, job postings, and the fixture-backed patent/review adapters)
|
||||
implements this same `SourceCollector` protocol, so `tasks/collection.py`
|
||||
(Phase 5) can treat them uniformly.
|
||||
|
||||
Collectors never talk to the database - they take plain dataclasses in and
|
||||
return plain dataclasses out. Persisting `CollectedDocument`s into
|
||||
`SourceDocument` rows is the caller's job (a service function, not the
|
||||
collector), which keeps collectors trivially unit-testable against fixtures.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Any, Protocol
|
||||
|
||||
from app.models.enums import SourceStatus, SourceType
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CompanyContext:
|
||||
"""Read-only view of a Company, passed into collectors instead of the ORM object."""
|
||||
|
||||
id: str
|
||||
name: str
|
||||
official_website: str | None
|
||||
monitoring_focus: str | None
|
||||
aliases: list[str] = field(default_factory=list)
|
||||
competitors: list[str] = field(default_factory=list)
|
||||
# From NinjaPear enrichment (app/models/company_enrichment.py), when it
|
||||
# ran and found a leadership team - empty otherwise (no key, still
|
||||
# pending, or no leadership data). Used by PatentSourceCollector to
|
||||
# search USPTO by inventor name, since that endpoint has no queryable
|
||||
# company/assignee field at all - see collectors/patents.py.
|
||||
leadership_names: list[str] = field(default_factory=list)
|
||||
# The owning user's effective USPTO key (their own, or the server's
|
||||
# global one) - None means "use the server's global settings.uspto_api_key
|
||||
# directly", for call sites that never resolved a per-user override
|
||||
# (e.g. discovery preview paths outside a monitoring run).
|
||||
uspto_api_key: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DiscoveredSource:
|
||||
source_type: SourceType
|
||||
name: str
|
||||
base_url: str | None
|
||||
configuration_metadata: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceConfig:
|
||||
id: str
|
||||
source_type: SourceType
|
||||
name: str
|
||||
base_url: str | None
|
||||
configuration_metadata: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CollectedDocument:
|
||||
url: str
|
||||
canonical_url: str
|
||||
title: str | None
|
||||
author: str | None
|
||||
publication_date: datetime | None
|
||||
retrieved_date: datetime
|
||||
content_text: str
|
||||
content_hash: str
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
language: str | None = None
|
||||
http_status: int | None = None
|
||||
extraction_method: str = "unknown"
|
||||
trust_score: float = 0.7
|
||||
|
||||
|
||||
@dataclass
|
||||
class CollectionResult:
|
||||
status: SourceStatus
|
||||
documents: list[CollectedDocument] = field(default_factory=list)
|
||||
error: str | None = None
|
||||
pages_attempted: int = 0
|
||||
|
||||
|
||||
class SourceCollector(Protocol):
|
||||
source_type: SourceType
|
||||
|
||||
async def discover(self, company: CompanyContext) -> list[DiscoveredSource]:
|
||||
"""Suggest sources for a newly added company. May return an empty
|
||||
list if this collector type can't be auto-discovered (e.g. patents)."""
|
||||
...
|
||||
|
||||
async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult:
|
||||
"""Fetch and extract current content for a configured source."""
|
||||
...
|
||||
@@ -0,0 +1,87 @@
|
||||
"""User-supplied custom URL collector - fetches, extracts, and monitors a
|
||||
single public URL the user explicitly added."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import httpx
|
||||
|
||||
from app.collectors.base import (
|
||||
CollectedDocument,
|
||||
CollectionResult,
|
||||
CompanyContext,
|
||||
DiscoveredSource,
|
||||
SourceConfig,
|
||||
)
|
||||
from app.collectors.extraction import (
|
||||
canonicalize_url,
|
||||
compute_content_hash,
|
||||
extract_readable_text,
|
||||
extract_title,
|
||||
)
|
||||
from app.collectors.robots import is_allowed
|
||||
from app.core.config import get_settings
|
||||
from app.core.http import FetchError, SsrfBlockedError, fetch_with_retries
|
||||
from app.models.enums import SourceStatus, SourceType
|
||||
|
||||
|
||||
class CustomUrlCollector:
|
||||
source_type = SourceType.CUSTOM_URL
|
||||
|
||||
async def discover(self, company: CompanyContext) -> list[DiscoveredSource]:
|
||||
return [] # Custom URLs are always user-supplied, never auto-discovered.
|
||||
|
||||
async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult:
|
||||
if not source.base_url:
|
||||
return CollectionResult(status=SourceStatus.FAILED, error="No URL configured")
|
||||
|
||||
settings = get_settings()
|
||||
try:
|
||||
if not await is_allowed(source.base_url, settings=settings):
|
||||
return CollectionResult(
|
||||
status=SourceStatus.BLOCKED_BY_POLICY,
|
||||
error="Disallowed by robots.txt",
|
||||
pages_attempted=1,
|
||||
)
|
||||
|
||||
result = await fetch_with_retries(source.base_url, settings=settings)
|
||||
except SsrfBlockedError as exc:
|
||||
return CollectionResult(
|
||||
status=SourceStatus.BLOCKED_BY_POLICY, error=str(exc), pages_attempted=1
|
||||
)
|
||||
except (FetchError, httpx.HTTPError) as exc:
|
||||
return CollectionResult(status=SourceStatus.FAILED, error=str(exc), pages_attempted=1)
|
||||
|
||||
if result.status_code in (401, 403):
|
||||
return CollectionResult(
|
||||
status=SourceStatus.AUTH_REQUIRED,
|
||||
error=f"HTTP {result.status_code}",
|
||||
pages_attempted=1,
|
||||
)
|
||||
if result.status_code >= 400:
|
||||
return CollectionResult(
|
||||
status=SourceStatus.FAILED, error=f"HTTP {result.status_code}", pages_attempted=1
|
||||
)
|
||||
|
||||
text, method = extract_readable_text(result.text, source.base_url)
|
||||
if not text:
|
||||
return CollectionResult(
|
||||
status=SourceStatus.FAILED, error="No extractable content", pages_attempted=1
|
||||
)
|
||||
|
||||
document = CollectedDocument(
|
||||
url=source.base_url,
|
||||
canonical_url=canonicalize_url(result.final_url),
|
||||
title=extract_title(result.text),
|
||||
author=None,
|
||||
publication_date=None,
|
||||
retrieved_date=datetime.now(UTC),
|
||||
content_text=text,
|
||||
content_hash=compute_content_hash(text),
|
||||
metadata={"http_status": result.status_code},
|
||||
extraction_method=method,
|
||||
http_status=result.status_code,
|
||||
trust_score=0.7,
|
||||
)
|
||||
return CollectionResult(status=SourceStatus.ACTIVE, documents=[document], pages_attempted=1)
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Shared content extraction/normalization helpers used by every collector.
|
||||
|
||||
Centralizing this (rather than letting each collector roll its own) is what
|
||||
makes cross-collector dedup and hashing behave consistently.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
import trafilatura
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
_WHITESPACE_RE = re.compile(r"[ \t\f\v]+")
|
||||
_BLANK_LINES_RE = re.compile(r"\n{3,}")
|
||||
|
||||
# Query params that vary per-request/session but don't change page meaning -
|
||||
# stripped so the same logical page always canonicalizes identically.
|
||||
_NOISE_QUERY_PREFIXES = ("utm_", "fbclid", "gclid", "mc_", "_hs")
|
||||
|
||||
|
||||
def normalize_whitespace(text: str) -> str:
|
||||
text = text.replace("\r\n", "\n").replace("\r", "\n")
|
||||
text = _WHITESPACE_RE.sub(" ", text)
|
||||
lines = [line.strip() for line in text.split("\n")]
|
||||
text = "\n".join(lines)
|
||||
return _BLANK_LINES_RE.sub("\n\n", text).strip()
|
||||
|
||||
|
||||
def compute_content_hash(text: str) -> str:
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def canonicalize_url(url: str) -> str:
|
||||
parts = urlsplit(url)
|
||||
query_pairs = [
|
||||
pair
|
||||
for pair in parts.query.split("&")
|
||||
if pair and not pair.split("=")[0].startswith(_NOISE_QUERY_PREFIXES)
|
||||
]
|
||||
path = parts.path.rstrip("/") or "/"
|
||||
return urlunsplit((parts.scheme.lower(), parts.netloc.lower(), path, "&".join(query_pairs), ""))
|
||||
|
||||
|
||||
def extract_readable_text(html: str, url: str) -> tuple[str, str]:
|
||||
"""Returns (text, extraction_method). Prefers trafilatura (boilerplate
|
||||
removal tuned for articles/press releases); falls back to a plain
|
||||
BeautifulSoup text extraction for pages trafilatura can't parse (e.g.
|
||||
thin job listing pages)."""
|
||||
extracted = trafilatura.extract(
|
||||
html,
|
||||
url=url,
|
||||
include_comments=False,
|
||||
include_tables=True,
|
||||
favor_precision=True,
|
||||
)
|
||||
if extracted and extracted.strip():
|
||||
return normalize_whitespace(extracted), "trafilatura"
|
||||
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
for tag in soup(["script", "style", "nav", "footer", "header", "noscript"]):
|
||||
tag.decompose()
|
||||
text = soup.get_text(separator="\n")
|
||||
return normalize_whitespace(text), "beautifulsoup_fallback"
|
||||
|
||||
|
||||
def extract_title(html: str) -> str | None:
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
if soup.title and soup.title.string:
|
||||
return soup.title.string.strip()
|
||||
h1 = soup.find("h1")
|
||||
if h1:
|
||||
return h1.get_text(strip=True)
|
||||
return None
|
||||
@@ -0,0 +1,147 @@
|
||||
"""GitHub collector - public organization/repository metadata via the
|
||||
public REST API. `GITHUB_TOKEN` is optional and only raises the rate limit;
|
||||
nothing here requires authentication.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import httpx
|
||||
|
||||
from app.collectors.base import (
|
||||
CollectedDocument,
|
||||
CollectionResult,
|
||||
CompanyContext,
|
||||
DiscoveredSource,
|
||||
SourceConfig,
|
||||
)
|
||||
from app.collectors.extraction import compute_content_hash, normalize_whitespace
|
||||
from app.core.config import Settings, get_settings
|
||||
from app.core.http import FetchError, SsrfBlockedError, fetch_with_retries
|
||||
from app.core.logging import get_logger
|
||||
from app.models.enums import SourceStatus, SourceType
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
_API_BASE = "https://api.github.com"
|
||||
|
||||
|
||||
def _auth_headers(settings: Settings) -> dict[str, str]:
|
||||
headers = {"Accept": "application/vnd.github+json"}
|
||||
if settings.github_token:
|
||||
headers["Authorization"] = f"Bearer {settings.github_token}"
|
||||
return headers
|
||||
|
||||
|
||||
class GithubCollector:
|
||||
source_type = SourceType.GITHUB
|
||||
|
||||
async def discover(self, company: CompanyContext) -> list[DiscoveredSource]:
|
||||
settings = get_settings()
|
||||
org_login = await self._find_org(company.name, settings)
|
||||
if org_login is None:
|
||||
return []
|
||||
return [
|
||||
DiscoveredSource(
|
||||
source_type=SourceType.GITHUB,
|
||||
name=f"{company.name} — GitHub",
|
||||
base_url=f"https://github.com/{org_login}",
|
||||
configuration_metadata={"org": org_login},
|
||||
)
|
||||
]
|
||||
|
||||
async def _find_org(self, company_name: str, settings: Settings) -> str | None:
|
||||
url = str(
|
||||
httpx.URL(
|
||||
f"{_API_BASE}/search/users",
|
||||
params={"q": f"{company_name} type:org", "per_page": 1},
|
||||
)
|
||||
)
|
||||
try:
|
||||
result = await fetch_with_retries(
|
||||
url, settings=settings, max_attempts=2, extra_headers=_auth_headers(settings)
|
||||
)
|
||||
except (SsrfBlockedError, FetchError, httpx.HTTPError) as exc:
|
||||
logger.warning("github_org_search_failed", company=company_name, error=str(exc))
|
||||
return None
|
||||
if result.status_code != 200:
|
||||
return None
|
||||
try:
|
||||
payload = json.loads(result.text)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
items = payload.get("items", [])
|
||||
return items[0]["login"] if items else None
|
||||
|
||||
async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult:
|
||||
org = source.configuration_metadata.get("org")
|
||||
if not org:
|
||||
return CollectionResult(status=SourceStatus.FAILED, error="No GitHub org configured")
|
||||
|
||||
settings = get_settings()
|
||||
url = str(
|
||||
httpx.URL(f"{_API_BASE}/orgs/{org}/repos", params={"sort": "pushed", "per_page": 15})
|
||||
)
|
||||
try:
|
||||
result = await fetch_with_retries(
|
||||
url, settings=settings, extra_headers=_auth_headers(settings)
|
||||
)
|
||||
except SsrfBlockedError as exc:
|
||||
return CollectionResult(status=SourceStatus.BLOCKED_BY_POLICY, error=str(exc))
|
||||
except (FetchError, httpx.HTTPError) as exc:
|
||||
return CollectionResult(status=SourceStatus.FAILED, error=str(exc))
|
||||
|
||||
if result.status_code == 404:
|
||||
return CollectionResult(
|
||||
status=SourceStatus.FAILED, error=f"GitHub org not found: {org}"
|
||||
)
|
||||
if result.status_code == 403:
|
||||
return CollectionResult(
|
||||
status=SourceStatus.RATE_LIMITED, error="GitHub API rate limited"
|
||||
)
|
||||
if result.status_code >= 400:
|
||||
return CollectionResult(status=SourceStatus.FAILED, error=f"HTTP {result.status_code}")
|
||||
|
||||
try:
|
||||
repos = json.loads(result.text)
|
||||
except json.JSONDecodeError:
|
||||
return CollectionResult(status=SourceStatus.FAILED, error="Malformed GitHub response")
|
||||
|
||||
documents: list[CollectedDocument] = []
|
||||
for repo in repos:
|
||||
text = normalize_whitespace(
|
||||
f"{repo.get('full_name')}\n"
|
||||
f"{repo.get('description') or ''}\n"
|
||||
f"Language: {repo.get('language') or 'unknown'}\n"
|
||||
f"Stars: {repo.get('stargazers_count', 0)}\n"
|
||||
f"Last pushed: {repo.get('pushed_at')}"
|
||||
)
|
||||
documents.append(
|
||||
CollectedDocument(
|
||||
url=repo.get("html_url"),
|
||||
canonical_url=repo.get("html_url"),
|
||||
title=repo.get("full_name"),
|
||||
author=org,
|
||||
publication_date=(
|
||||
datetime.fromisoformat(repo["pushed_at"].replace("Z", "+00:00"))
|
||||
if repo.get("pushed_at")
|
||||
else None
|
||||
),
|
||||
retrieved_date=datetime.now(UTC),
|
||||
content_text=text,
|
||||
content_hash=compute_content_hash(text),
|
||||
metadata={
|
||||
"stars": repo.get("stargazers_count"),
|
||||
"language": repo.get("language"),
|
||||
"archived": repo.get("archived"),
|
||||
},
|
||||
extraction_method="github_api",
|
||||
http_status=result.status_code,
|
||||
trust_score=0.75,
|
||||
)
|
||||
)
|
||||
|
||||
status = SourceStatus.ACTIVE
|
||||
return CollectionResult(status=status, documents=documents, pages_attempted=1)
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Federal contracts collector via USASpending.gov's public Award Search
|
||||
API - free, keyless, no registration (a fixed, trusted, first-party
|
||||
integration endpoint like Brave/Twilio, so this calls httpx directly rather
|
||||
than through `safe_fetch`, which exists to guard arbitrary/user-supplied
|
||||
collector targets, not our own known API integrations).
|
||||
|
||||
Offered for every company regardless of type, same as SecEdgarCollector -
|
||||
a private company simply returns zero awards, which is a normal empty
|
||||
result, not a failure.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import httpx
|
||||
|
||||
from app.collectors.base import (
|
||||
CollectedDocument,
|
||||
CollectionResult,
|
||||
CompanyContext,
|
||||
DiscoveredSource,
|
||||
SourceConfig,
|
||||
)
|
||||
from app.collectors.extraction import compute_content_hash, normalize_whitespace
|
||||
from app.core.logging import get_logger
|
||||
from app.models.enums import SourceStatus, SourceType
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
_SEARCH_URL = "https://api.usaspending.gov/api/v2/search/spending_by_award/"
|
||||
_AWARD_TYPE_CODES = ["A", "B", "C", "D"] # contracts (definitive/BPA/purchase order/delivery order)
|
||||
|
||||
|
||||
class GovContractCollector:
|
||||
source_type = SourceType.GOV_CONTRACT
|
||||
|
||||
async def discover(self, company: CompanyContext) -> list[DiscoveredSource]:
|
||||
return [
|
||||
DiscoveredSource(
|
||||
source_type=SourceType.GOV_CONTRACT,
|
||||
name=f"{company.name} — Federal Contracts",
|
||||
base_url=None,
|
||||
configuration_metadata={"recipient_search_text": company.name},
|
||||
)
|
||||
]
|
||||
|
||||
async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult:
|
||||
recipient = source.configuration_metadata.get("recipient_search_text") or company.name
|
||||
|
||||
body = {
|
||||
"filters": {
|
||||
"recipient_search_text": [recipient],
|
||||
"award_type_codes": _AWARD_TYPE_CODES,
|
||||
},
|
||||
"fields": [
|
||||
"Award ID",
|
||||
"Recipient Name",
|
||||
"Award Amount",
|
||||
"Start Date",
|
||||
"Awarding Agency",
|
||||
"Description",
|
||||
],
|
||||
"sort": "Award Amount",
|
||||
"order": "desc",
|
||||
"page": 1,
|
||||
"limit": 25,
|
||||
}
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=20) as client:
|
||||
response = await client.post(_SEARCH_URL, json=body)
|
||||
except httpx.HTTPError as exc:
|
||||
return CollectionResult(status=SourceStatus.FAILED, error=str(exc))
|
||||
|
||||
if response.status_code >= 400:
|
||||
return CollectionResult(
|
||||
status=SourceStatus.FAILED,
|
||||
error=f"USASpending API error {response.status_code}: {response.text[:200]}",
|
||||
pages_attempted=1,
|
||||
)
|
||||
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError:
|
||||
return CollectionResult(
|
||||
status=SourceStatus.FAILED,
|
||||
error="Malformed USASpending response",
|
||||
pages_attempted=1,
|
||||
)
|
||||
|
||||
results = payload.get("results", [])
|
||||
documents: list[CollectedDocument] = []
|
||||
for award in results:
|
||||
award_id = award.get("Award ID", "unknown")
|
||||
agency = award.get("Awarding Agency", "Unknown agency")
|
||||
amount = award.get("Award Amount")
|
||||
amount_display = (
|
||||
f"${amount:,.0f}" if isinstance(amount, (int, float)) else "unknown amount"
|
||||
)
|
||||
start_date = award.get("Start Date", "")
|
||||
description = award.get("Description") or ""
|
||||
|
||||
text = normalize_whitespace(
|
||||
f"{award.get('Recipient Name', recipient)} was awarded federal contract "
|
||||
f"{award_id} by {agency} for {amount_display}, starting {start_date}. "
|
||||
f"{description}"
|
||||
)
|
||||
documents.append(
|
||||
CollectedDocument(
|
||||
url=_SEARCH_URL,
|
||||
canonical_url=_SEARCH_URL,
|
||||
title=f"{agency}: {award_id} — {amount_display}",
|
||||
author="USASpending.gov",
|
||||
publication_date=(
|
||||
datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=UTC)
|
||||
if start_date
|
||||
else None
|
||||
),
|
||||
retrieved_date=datetime.now(UTC),
|
||||
content_text=text,
|
||||
content_hash=compute_content_hash(text),
|
||||
metadata={"award_id": award_id, "awarding_agency": agency},
|
||||
extraction_method="usaspending_api",
|
||||
http_status=response.status_code,
|
||||
trust_score=0.8,
|
||||
)
|
||||
)
|
||||
|
||||
# Zero awards isn't a failure - most companies never win a federal
|
||||
# contract, same non-error empty-result handling as SecEdgarCollector.
|
||||
return CollectionResult(status=SourceStatus.ACTIVE, documents=documents, pages_attempted=1)
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Job posting collector: generic heuristic extraction from a company's own
|
||||
careers page. Board-specific APIs (LinkedIn, Indeed, etc.) are not
|
||||
implemented - most require paid access or prohibit automated collection in
|
||||
their terms; see KNOWN_LIMITATIONS.md.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from app.collectors.base import (
|
||||
CollectedDocument,
|
||||
CollectionResult,
|
||||
CompanyContext,
|
||||
DiscoveredSource,
|
||||
SourceConfig,
|
||||
)
|
||||
from app.collectors.extraction import (
|
||||
canonicalize_url,
|
||||
compute_content_hash,
|
||||
extract_readable_text,
|
||||
extract_title,
|
||||
normalize_whitespace,
|
||||
)
|
||||
from app.collectors.robots import is_allowed
|
||||
from app.core.config import get_settings
|
||||
from app.core.http import FetchError, SsrfBlockedError, fetch_with_retries
|
||||
from app.models.enums import SourceStatus, SourceType
|
||||
|
||||
_JOB_LINK_KEYWORDS = ("job", "career", "position", "opening", "role", "vacan")
|
||||
_MAX_LISTINGS = 50
|
||||
|
||||
|
||||
class JobPostingCollector:
|
||||
source_type = SourceType.JOB_POSTING
|
||||
|
||||
async def discover(self, company: CompanyContext) -> list[DiscoveredSource]:
|
||||
if not company.official_website:
|
||||
return []
|
||||
careers_url = urljoin(company.official_website, "/careers")
|
||||
return [
|
||||
DiscoveredSource(
|
||||
source_type=SourceType.JOB_POSTING,
|
||||
name=f"{company.name} — Careers",
|
||||
base_url=careers_url,
|
||||
)
|
||||
]
|
||||
|
||||
async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult:
|
||||
if not source.base_url:
|
||||
return CollectionResult(status=SourceStatus.FAILED, error="No careers URL configured")
|
||||
|
||||
settings = get_settings()
|
||||
try:
|
||||
if not await is_allowed(source.base_url, settings=settings):
|
||||
return CollectionResult(
|
||||
status=SourceStatus.BLOCKED_BY_POLICY,
|
||||
error="Disallowed by robots.txt",
|
||||
pages_attempted=1,
|
||||
)
|
||||
result = await fetch_with_retries(source.base_url, settings=settings)
|
||||
except SsrfBlockedError as exc:
|
||||
return CollectionResult(
|
||||
status=SourceStatus.BLOCKED_BY_POLICY, error=str(exc), pages_attempted=1
|
||||
)
|
||||
except (FetchError, httpx.HTTPError) as exc:
|
||||
return CollectionResult(status=SourceStatus.FAILED, error=str(exc), pages_attempted=1)
|
||||
|
||||
if result.status_code in (401, 403):
|
||||
return CollectionResult(
|
||||
status=SourceStatus.AUTH_REQUIRED,
|
||||
error=f"HTTP {result.status_code}",
|
||||
pages_attempted=1,
|
||||
)
|
||||
if result.status_code >= 400:
|
||||
return CollectionResult(
|
||||
status=SourceStatus.FAILED, error=f"HTTP {result.status_code}", pages_attempted=1
|
||||
)
|
||||
|
||||
listings = self._extract_job_listings(result.text, result.final_url)
|
||||
|
||||
if not listings:
|
||||
# Non-standard careers page (e.g. a third-party ATS iframe) -
|
||||
# fall back to the whole page as one document rather than
|
||||
# reporting a failure for a page that did load successfully.
|
||||
text, method = extract_readable_text(result.text, source.base_url)
|
||||
if not text:
|
||||
return CollectionResult(
|
||||
status=SourceStatus.FAILED, error="No extractable content", pages_attempted=1
|
||||
)
|
||||
document = CollectedDocument(
|
||||
url=source.base_url,
|
||||
canonical_url=canonicalize_url(result.final_url),
|
||||
title=extract_title(result.text),
|
||||
author=None,
|
||||
publication_date=None,
|
||||
retrieved_date=datetime.now(UTC),
|
||||
content_text=text,
|
||||
content_hash=compute_content_hash(text),
|
||||
metadata={"extraction": "fallback_whole_page"},
|
||||
extraction_method=method,
|
||||
http_status=result.status_code,
|
||||
trust_score=0.55,
|
||||
)
|
||||
return CollectionResult(
|
||||
status=SourceStatus.ACTIVE, documents=[document], pages_attempted=1
|
||||
)
|
||||
|
||||
documents = [
|
||||
CollectedDocument(
|
||||
url=link,
|
||||
canonical_url=canonicalize_url(link),
|
||||
title=title,
|
||||
author=None,
|
||||
publication_date=None,
|
||||
retrieved_date=datetime.now(UTC),
|
||||
content_text=normalize_whitespace(f"{title}\n{snippet}"),
|
||||
content_hash=compute_content_hash(normalize_whitespace(f"{title}\n{snippet}")),
|
||||
metadata={"source_page": source.base_url},
|
||||
extraction_method="job_link_heuristic",
|
||||
http_status=result.status_code,
|
||||
trust_score=0.65,
|
||||
)
|
||||
for title, link, snippet in listings
|
||||
]
|
||||
return CollectionResult(status=SourceStatus.ACTIVE, documents=documents, pages_attempted=1)
|
||||
|
||||
def _extract_job_listings(self, html: str, base_url: str) -> list[tuple[str, str, str]]:
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
listings: list[tuple[str, str, str]] = []
|
||||
seen_links: set[str] = set()
|
||||
|
||||
for anchor in soup.find_all("a", href=True):
|
||||
href = anchor["href"]
|
||||
text = anchor.get_text(strip=True)
|
||||
if not text or len(text) < 4 or len(text) > 150:
|
||||
continue
|
||||
if not any(keyword in href.lower() for keyword in _JOB_LINK_KEYWORDS):
|
||||
continue
|
||||
|
||||
link = urljoin(base_url, href)
|
||||
if link in seen_links:
|
||||
continue
|
||||
seen_links.add(link)
|
||||
|
||||
parent = anchor.find_parent()
|
||||
snippet = parent.get_text(" ", strip=True)[:300] if parent else ""
|
||||
listings.append((text, link, snippet))
|
||||
if len(listings) >= _MAX_LISTINGS:
|
||||
break
|
||||
|
||||
return listings
|
||||
@@ -0,0 +1,245 @@
|
||||
"""Patent source collector.
|
||||
|
||||
USPTO's PatentsView data migrated into the Open Data Portal (ODP) in March
|
||||
2026; a free API key is available via account registration at
|
||||
data.uspto.gov/apis/getting-started (see `Settings.uspto_api_key`). Without
|
||||
a key configured (the default), this collector never fabricates patent
|
||||
data - it truthfully reports `DISABLED` with an explanation, same as
|
||||
before this integration existed, and a fixture adapter remains available
|
||||
for local development/testing.
|
||||
|
||||
With a key configured, `collect()` calls the real ODP Patent Application
|
||||
Search API - by INVENTOR NAME, not company name. Confirmed live (2026-08)
|
||||
by inspecting a real response's full field list, including one from a
|
||||
query that returned 110k+ real results: there is no assignee/company field
|
||||
anywhere in this endpoint's data model. Company-name search here always
|
||||
returns "no matching records," even for assignees with thousands of real
|
||||
patents - it isn't a wrong-field-name bug, the field doesn't exist on this
|
||||
dataset. USPTO's Patent Application Search reliably supports inventor-name
|
||||
and application-number lookups only.
|
||||
|
||||
So `collect()` instead searches by each of the company's known leadership
|
||||
names (from NinjaPear enrichment, see `CompanyContext.leadership_names` /
|
||||
`enrichment_service.py`) and treats a match as a heuristic company signal,
|
||||
not a verified one - there is still no way to confirm a given patent
|
||||
actually belongs to the monitored company rather than, say, a same-named
|
||||
person, or work the person did at a prior employer. Every resulting
|
||||
document is trust-scored lower (0.5, vs. a hypothetical verified-assignee
|
||||
match) and its content explicitly says which leadership name it matched
|
||||
on, so the report LLM's confidence labeling reflects this rather than
|
||||
treating it as confirmed fact. With no leadership names available (no
|
||||
NinjaPear key, enrichment still pending, or it returned no leadership
|
||||
data), this reports an honest empty result without making a network call
|
||||
- there's nothing meaningful to search USPTO for.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
from app.collectors.base import (
|
||||
CollectedDocument,
|
||||
CollectionResult,
|
||||
CompanyContext,
|
||||
DiscoveredSource,
|
||||
SourceConfig,
|
||||
)
|
||||
from app.collectors.extraction import compute_content_hash, normalize_whitespace
|
||||
from app.core.config import get_settings
|
||||
from app.core.logging import get_logger
|
||||
from app.models.enums import SourceStatus, SourceType
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
_FIXTURES_DIR = Path(__file__).resolve().parents[2] / "tests" / "fixtures" / "patents"
|
||||
_SEARCH_URL = "https://api.uspto.gov/api/v1/patent/applications/search"
|
||||
_MAX_INVENTOR_SEARCHES = 5
|
||||
_MAX_DOCUMENTS = 25
|
||||
|
||||
|
||||
class PatentSourceCollector:
|
||||
source_type = SourceType.PATENT
|
||||
|
||||
async def discover(self, company: CompanyContext) -> list[DiscoveredSource]:
|
||||
# company.uspto_api_key, when set by the caller (see
|
||||
# collection_service.to_company_context), is the resolved effective
|
||||
# key for whichever user owns this company - their own if they've
|
||||
# set one, else the server's global default. None means no
|
||||
# per-user resolution happened for this call path, so fall back to
|
||||
# the global settings directly.
|
||||
api_key = company.uspto_api_key or get_settings().uspto_api_key
|
||||
if not api_key:
|
||||
return [] # No live discovery without a configured provider.
|
||||
return [
|
||||
DiscoveredSource(
|
||||
source_type=SourceType.PATENT,
|
||||
name=f"{company.name} — Patent Filings",
|
||||
base_url=None,
|
||||
configuration_metadata={"assignee": company.name},
|
||||
)
|
||||
]
|
||||
|
||||
async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult:
|
||||
api_key = company.uspto_api_key or get_settings().uspto_api_key
|
||||
if api_key:
|
||||
return await self._collect_live(company.name, company.leadership_names, api_key)
|
||||
|
||||
fixture_key = source.configuration_metadata.get("fixture_key")
|
||||
if not fixture_key:
|
||||
return CollectionResult(
|
||||
status=SourceStatus.DISABLED,
|
||||
error=(
|
||||
"No patent data provider is configured. This collector implements the "
|
||||
"SourceCollector interface for a live integration (USPTO Open Data Portal) "
|
||||
"but does not fabricate results without a configured USPTO_API_KEY."
|
||||
),
|
||||
)
|
||||
|
||||
fixture_path = _FIXTURES_DIR / f"{fixture_key}.json"
|
||||
if not fixture_path.exists():
|
||||
return CollectionResult(
|
||||
status=SourceStatus.DISABLED,
|
||||
error=f"No fixture found for {fixture_key!r} and no live provider is configured.",
|
||||
)
|
||||
|
||||
payload = json.loads(fixture_path.read_text(encoding="utf-8"))
|
||||
documents: list[CollectedDocument] = []
|
||||
for entry in payload.get("patents", []):
|
||||
text = normalize_whitespace(f"{entry['title']}\n\n{entry.get('abstract', '')}")
|
||||
documents.append(
|
||||
CollectedDocument(
|
||||
url=entry.get("url", fixture_path.as_uri()),
|
||||
canonical_url=entry.get("url", fixture_path.as_uri()),
|
||||
title=entry["title"],
|
||||
author=entry.get("assignee"),
|
||||
publication_date=(
|
||||
datetime.fromisoformat(entry["filed_date"]).replace(tzinfo=UTC)
|
||||
if entry.get("filed_date")
|
||||
else None
|
||||
),
|
||||
retrieved_date=datetime.now(UTC),
|
||||
content_text=text,
|
||||
content_hash=compute_content_hash(text),
|
||||
metadata={
|
||||
"is_fixture": True,
|
||||
"data_source": "fixture",
|
||||
"fixture_key": fixture_key,
|
||||
},
|
||||
extraction_method="fixture",
|
||||
trust_score=0.5,
|
||||
)
|
||||
)
|
||||
return CollectionResult(status=SourceStatus.ACTIVE, documents=documents, pages_attempted=1)
|
||||
|
||||
async def _collect_live(
|
||||
self, company_name: str, leadership_names: list[str], api_key: str
|
||||
) -> CollectionResult:
|
||||
if not leadership_names:
|
||||
# No names to search USPTO's inventor index with - see the
|
||||
# module docstring for why company-name search doesn't work on
|
||||
# this endpoint at all. Honest empty result, no network call.
|
||||
return CollectionResult(status=SourceStatus.ACTIVE, documents=[], pages_attempted=0)
|
||||
|
||||
documents: list[CollectedDocument] = []
|
||||
seen_app_numbers: set[str] = set()
|
||||
pages_attempted = 0
|
||||
errors: list[str] = []
|
||||
|
||||
async with httpx.AsyncClient(timeout=20) as client:
|
||||
for inventor_name in leadership_names[:_MAX_INVENTOR_SEARCHES]:
|
||||
pages_attempted += 1
|
||||
body = {
|
||||
"q": f'applicationMetaData.inventorBag.inventorNameText:"{inventor_name}"',
|
||||
"pagination": {"limit": _MAX_DOCUMENTS},
|
||||
"sort": [{"field": "applicationMetaData.filingDate", "order": "desc"}],
|
||||
}
|
||||
try:
|
||||
response = await client.post(
|
||||
_SEARCH_URL, json=body, headers={"x-api-key": api_key}
|
||||
)
|
||||
except httpx.HTTPError as exc:
|
||||
errors.append(f"{inventor_name}: {exc}")
|
||||
continue
|
||||
|
||||
if response.status_code == 404:
|
||||
# USPTO returns 404 for "no matching records" rather
|
||||
# than 200 with an empty array - a real, expected
|
||||
# outcome for most names, not a failure.
|
||||
continue
|
||||
if response.status_code >= 400:
|
||||
errors.append(f"{inventor_name}: USPTO API error {response.status_code}")
|
||||
continue
|
||||
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError:
|
||||
errors.append(f"{inventor_name}: malformed USPTO response")
|
||||
continue
|
||||
|
||||
entries = payload.get("patentFileWrapperDataBag") or payload.get("results") or []
|
||||
for entry in entries:
|
||||
metadata = entry.get("applicationMetaData") or {}
|
||||
app_number = entry.get("applicationNumberText") or entry.get(
|
||||
"applicationNumber"
|
||||
)
|
||||
if not app_number or app_number in seen_app_numbers:
|
||||
continue
|
||||
seen_app_numbers.add(app_number)
|
||||
|
||||
title = metadata.get("inventionTitle") or "Untitled patent filing"
|
||||
filing_date = metadata.get("filingDate")
|
||||
abstract = metadata.get("abstractText") or ""
|
||||
text = normalize_whitespace(
|
||||
f"{title}\n\nInventor match: {inventor_name} (leadership-name "
|
||||
f"heuristic, not a verified {company_name} assignee - USPTO's "
|
||||
"application search has no queryable assignee/company field).\n\n"
|
||||
f"{abstract}"
|
||||
)
|
||||
documents.append(
|
||||
CollectedDocument(
|
||||
url=(
|
||||
f"{_SEARCH_URL}?applicationNumber={app_number}"
|
||||
if app_number
|
||||
else _SEARCH_URL
|
||||
),
|
||||
canonical_url=_SEARCH_URL,
|
||||
title=title,
|
||||
author=inventor_name,
|
||||
publication_date=(
|
||||
datetime.fromisoformat(filing_date).replace(tzinfo=UTC)
|
||||
if filing_date
|
||||
else None
|
||||
),
|
||||
retrieved_date=datetime.now(UTC),
|
||||
content_text=text,
|
||||
content_hash=compute_content_hash(text),
|
||||
metadata={
|
||||
"application_number": app_number,
|
||||
"data_source": "uspto_odp",
|
||||
"matched_inventor_name": inventor_name,
|
||||
"match_type": "leadership_name_heuristic",
|
||||
},
|
||||
extraction_method="uspto_odp_api",
|
||||
http_status=response.status_code,
|
||||
# Lower than a verified-assignee match would be
|
||||
# (was 0.9) - this is a heuristic name match,
|
||||
# not confirmed company ownership.
|
||||
trust_score=0.5,
|
||||
)
|
||||
)
|
||||
if len(documents) >= _MAX_DOCUMENTS:
|
||||
break
|
||||
|
||||
if errors and not documents:
|
||||
return CollectionResult(
|
||||
status=SourceStatus.FAILED,
|
||||
error="; ".join(errors[:3]),
|
||||
pages_attempted=pages_attempted,
|
||||
)
|
||||
return CollectionResult(
|
||||
status=SourceStatus.ACTIVE, documents=documents, pages_attempted=pages_attempted
|
||||
)
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Maps SourceType -> collector instance. The single place Phase 5's Celery
|
||||
task (and this phase's tests) resolve a collector from a Source row."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.collectors.base import SourceCollector
|
||||
from app.collectors.custom_url import CustomUrlCollector
|
||||
from app.collectors.github import GithubCollector
|
||||
from app.collectors.gov_contracts import GovContractCollector
|
||||
from app.collectors.jobs import JobPostingCollector
|
||||
from app.collectors.patents import PatentSourceCollector
|
||||
from app.collectors.reviews import ReviewSourceCollector
|
||||
from app.collectors.rss import RssCollector
|
||||
from app.collectors.sec_edgar import SecEdgarCollector
|
||||
from app.collectors.website import WebsiteCollector
|
||||
from app.models.enums import SourceType
|
||||
|
||||
_COLLECTORS: dict[SourceType, SourceCollector] = {
|
||||
SourceType.WEBSITE: WebsiteCollector(),
|
||||
SourceType.RSS: RssCollector(),
|
||||
SourceType.CUSTOM_URL: CustomUrlCollector(),
|
||||
SourceType.SEC_EDGAR: SecEdgarCollector(),
|
||||
SourceType.GITHUB: GithubCollector(),
|
||||
SourceType.JOB_POSTING: JobPostingCollector(),
|
||||
SourceType.PATENT: PatentSourceCollector(),
|
||||
SourceType.REVIEW: ReviewSourceCollector(),
|
||||
SourceType.GOV_CONTRACT: GovContractCollector(),
|
||||
}
|
||||
|
||||
|
||||
def get_collector(source_type: SourceType) -> SourceCollector:
|
||||
return _COLLECTORS[source_type]
|
||||
|
||||
|
||||
def all_collectors() -> list[SourceCollector]:
|
||||
return list(_COLLECTORS.values())
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Customer review source collector.
|
||||
|
||||
Most review platforms (G2, Trustpilot, Glassdoor, etc.) either prohibit
|
||||
automated scraping in their terms or require a paid API. This collector
|
||||
implements the `SourceCollector` interface and a documented fixture adapter
|
||||
for local development/testing; it never scrapes a review site directly and
|
||||
never fabricates review data when no permitted live provider is configured.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from app.collectors.base import (
|
||||
CollectedDocument,
|
||||
CollectionResult,
|
||||
CompanyContext,
|
||||
DiscoveredSource,
|
||||
SourceConfig,
|
||||
)
|
||||
from app.collectors.extraction import compute_content_hash, normalize_whitespace
|
||||
from app.models.enums import SourceStatus, SourceType
|
||||
|
||||
_FIXTURES_DIR = Path(__file__).resolve().parents[2] / "tests" / "fixtures" / "reviews"
|
||||
|
||||
|
||||
class ReviewSourceCollector:
|
||||
source_type = SourceType.REVIEW
|
||||
|
||||
async def discover(self, company: CompanyContext) -> list[DiscoveredSource]:
|
||||
return []
|
||||
|
||||
async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult:
|
||||
fixture_key = source.configuration_metadata.get("fixture_key")
|
||||
if not fixture_key:
|
||||
return CollectionResult(
|
||||
status=SourceStatus.DISABLED,
|
||||
error=(
|
||||
"No review data provider is configured. Most review platforms prohibit "
|
||||
"automated scraping in their terms; this collector implements the "
|
||||
"SourceCollector interface for a future permitted API integration but "
|
||||
"does not fabricate results without one."
|
||||
),
|
||||
)
|
||||
|
||||
fixture_path = _FIXTURES_DIR / f"{fixture_key}.json"
|
||||
if not fixture_path.exists():
|
||||
return CollectionResult(
|
||||
status=SourceStatus.DISABLED,
|
||||
error=f"No fixture found for {fixture_key!r} and no live provider is configured.",
|
||||
)
|
||||
|
||||
payload = json.loads(fixture_path.read_text(encoding="utf-8"))
|
||||
documents: list[CollectedDocument] = []
|
||||
for entry in payload.get("reviews", []):
|
||||
text = normalize_whitespace(
|
||||
f"Rating: {entry.get('rating', 'n/a')}/5\n\n{entry.get('body', '')}"
|
||||
)
|
||||
documents.append(
|
||||
CollectedDocument(
|
||||
url=entry.get("url", fixture_path.as_uri()),
|
||||
canonical_url=entry.get("url", fixture_path.as_uri()),
|
||||
title=entry.get("title") or "Customer review",
|
||||
author=entry.get("author"),
|
||||
publication_date=(
|
||||
datetime.fromisoformat(entry["date"]).replace(tzinfo=UTC)
|
||||
if entry.get("date")
|
||||
else None
|
||||
),
|
||||
retrieved_date=datetime.now(UTC),
|
||||
content_text=text,
|
||||
content_hash=compute_content_hash(text),
|
||||
metadata={
|
||||
"is_fixture": True,
|
||||
"data_source": "fixture",
|
||||
"fixture_key": fixture_key,
|
||||
"rating": entry.get("rating"),
|
||||
},
|
||||
extraction_method="fixture",
|
||||
trust_score=0.4,
|
||||
)
|
||||
)
|
||||
return CollectionResult(status=SourceStatus.ACTIVE, documents=documents, pages_attempted=1)
|
||||
@@ -0,0 +1,54 @@
|
||||
"""robots.txt compliance check - see SECURITY.md rule 1."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from urllib.parse import urljoin, urlparse
|
||||
from urllib.robotparser import RobotFileParser
|
||||
|
||||
from app.core.config import Settings, get_settings
|
||||
from app.core.http import SsrfBlockedError, safe_fetch
|
||||
from app.core.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
_CACHE_TTL_SECONDS = 3600
|
||||
_cache: dict[str, tuple[float, RobotFileParser]] = {}
|
||||
|
||||
|
||||
async def _get_parser(base_url: str, settings: Settings) -> RobotFileParser:
|
||||
parsed = urlparse(base_url)
|
||||
origin = f"{parsed.scheme}://{parsed.netloc}"
|
||||
cached = _cache.get(origin)
|
||||
now = time.monotonic()
|
||||
if cached and now - cached[0] < _CACHE_TTL_SECONDS:
|
||||
return cached[1]
|
||||
|
||||
parser = RobotFileParser()
|
||||
robots_url = urljoin(origin, "/robots.txt")
|
||||
try:
|
||||
result = await safe_fetch(robots_url, settings=settings)
|
||||
if result.status_code == 200:
|
||||
parser.parse(result.text.splitlines())
|
||||
else:
|
||||
# No robots.txt or inaccessible -> "allow all" per convention.
|
||||
parser.parse([])
|
||||
except SsrfBlockedError:
|
||||
parser.parse([])
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.warning("robots_txt_fetch_failed", url=robots_url, error=str(exc))
|
||||
parser.parse([])
|
||||
|
||||
_cache[origin] = (now, parser)
|
||||
return parser
|
||||
|
||||
|
||||
async def is_allowed(url: str, *, settings: Settings | None = None) -> bool:
|
||||
settings = settings or get_settings()
|
||||
parser = await _get_parser(url, settings)
|
||||
return parser.can_fetch(settings.scraper_user_agent, url)
|
||||
|
||||
|
||||
def clear_cache() -> None:
|
||||
"""Test helper - the module-level cache would otherwise leak between tests."""
|
||||
_cache.clear()
|
||||
@@ -0,0 +1,111 @@
|
||||
"""RSS/Atom feed collector."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time as time_module
|
||||
from datetime import UTC, datetime
|
||||
from urllib.parse import quote
|
||||
|
||||
import feedparser
|
||||
import httpx
|
||||
|
||||
from app.collectors.base import (
|
||||
CollectedDocument,
|
||||
CollectionResult,
|
||||
CompanyContext,
|
||||
DiscoveredSource,
|
||||
SourceConfig,
|
||||
)
|
||||
from app.collectors.extraction import canonicalize_url, compute_content_hash, normalize_whitespace
|
||||
from app.core.config import get_settings
|
||||
from app.core.http import FetchError, SsrfBlockedError, fetch_with_retries
|
||||
from app.core.logging import get_logger
|
||||
from app.models.enums import SourceStatus, SourceType
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class RssCollector:
|
||||
source_type = SourceType.RSS
|
||||
|
||||
async def discover(self, company: CompanyContext) -> list[DiscoveredSource]:
|
||||
# Google News' search RSS endpoint needs no API key and reliably
|
||||
# exists for any query - unlike a company's own press-room feed
|
||||
# (which would need a search provider to locate), this one URL
|
||||
# formula works for every company and already aggregates wire-
|
||||
# service releases (PRNewswire/BusinessWire/GlobeNewswire) as
|
||||
# they're published, so a dedicated wire-specific collector isn't
|
||||
# needed on top of it. Users can still add any other feed manually
|
||||
# (see custom_url.py's sibling "add any public URL" path).
|
||||
query_url = (
|
||||
f"https://news.google.com/rss/search?q={quote(company.name)}&hl=en-US&gl=US&ceid=US:en"
|
||||
)
|
||||
return [
|
||||
DiscoveredSource(
|
||||
source_type=SourceType.RSS,
|
||||
name=f"{company.name} — Google News",
|
||||
base_url=query_url,
|
||||
)
|
||||
]
|
||||
|
||||
async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult:
|
||||
if not source.base_url:
|
||||
return CollectionResult(status=SourceStatus.FAILED, error="No feed URL configured")
|
||||
|
||||
settings = get_settings()
|
||||
try:
|
||||
result = await fetch_with_retries(source.base_url, settings=settings)
|
||||
except SsrfBlockedError as exc:
|
||||
return CollectionResult(status=SourceStatus.BLOCKED_BY_POLICY, error=str(exc))
|
||||
except (FetchError, httpx.HTTPError) as exc:
|
||||
return CollectionResult(status=SourceStatus.FAILED, error=str(exc))
|
||||
|
||||
if result.status_code >= 400:
|
||||
return CollectionResult(
|
||||
status=SourceStatus.FAILED, error=f"HTTP {result.status_code}", pages_attempted=1
|
||||
)
|
||||
|
||||
parsed = feedparser.parse(result.content)
|
||||
if parsed.bozo and not parsed.entries:
|
||||
return CollectionResult(
|
||||
status=SourceStatus.FAILED,
|
||||
error=str(parsed.get("bozo_exception", "Unparseable feed")),
|
||||
pages_attempted=1,
|
||||
)
|
||||
|
||||
max_items = source.configuration_metadata.get("max_items", 20)
|
||||
documents: list[CollectedDocument] = []
|
||||
for entry in parsed.entries[:max_items]:
|
||||
link = entry.get("link")
|
||||
if not link:
|
||||
continue
|
||||
summary = entry.get("summary", "") or entry.get("description", "")
|
||||
text = normalize_whitespace(f"{entry.get('title', '')}\n\n{summary}")
|
||||
if not text:
|
||||
continue
|
||||
|
||||
pub_date = None
|
||||
if entry.get("published_parsed"):
|
||||
pub_date = datetime.fromtimestamp(
|
||||
time_module.mktime(entry.published_parsed), tz=UTC
|
||||
)
|
||||
|
||||
documents.append(
|
||||
CollectedDocument(
|
||||
url=link,
|
||||
canonical_url=canonicalize_url(link),
|
||||
title=entry.get("title"),
|
||||
author=entry.get("author"),
|
||||
publication_date=pub_date,
|
||||
retrieved_date=datetime.now(UTC),
|
||||
content_text=text,
|
||||
content_hash=compute_content_hash(text),
|
||||
metadata={"feed_url": source.base_url},
|
||||
extraction_method="feedparser",
|
||||
http_status=result.status_code,
|
||||
trust_score=0.6,
|
||||
)
|
||||
)
|
||||
|
||||
status = SourceStatus.ACTIVE if documents else SourceStatus.FAILED
|
||||
return CollectionResult(status=status, documents=documents, pages_attempted=1)
|
||||
@@ -0,0 +1,167 @@
|
||||
"""SEC EDGAR collector for US public companies.
|
||||
|
||||
No API key required, but SEC asks that callers identify themselves with a
|
||||
descriptive User-Agent (see `SCRAPER_USER_AGENT` in .env.example) and stay
|
||||
within its rate limits - the shared `safe_fetch` per-domain delay covers
|
||||
that. We store filing *metadata* (form type, date, accession number, link)
|
||||
rather than parsing full filing bodies, which is out of scope for this pass.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from xml.etree import ElementTree
|
||||
|
||||
import httpx
|
||||
|
||||
from app.collectors.base import (
|
||||
CollectedDocument,
|
||||
CollectionResult,
|
||||
CompanyContext,
|
||||
DiscoveredSource,
|
||||
SourceConfig,
|
||||
)
|
||||
from app.collectors.extraction import compute_content_hash, normalize_whitespace
|
||||
from app.core.config import get_settings
|
||||
from app.core.http import FetchError, SsrfBlockedError, fetch_with_retries
|
||||
from app.core.logging import get_logger
|
||||
from app.models.enums import SourceStatus, SourceType
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
_RELEVANT_FORMS = {"10-K", "10-Q", "8-K"}
|
||||
_SEARCH_URL = "https://www.sec.gov/cgi-bin/browse-edgar"
|
||||
_SUBMISSIONS_URL = "https://data.sec.gov/submissions/CIK{cik}.json"
|
||||
|
||||
|
||||
class SecEdgarCollector:
|
||||
source_type = SourceType.SEC_EDGAR
|
||||
|
||||
async def discover(self, company: CompanyContext) -> list[DiscoveredSource]:
|
||||
settings = get_settings()
|
||||
cik = await self._lookup_cik(company.name, settings)
|
||||
if cik is None:
|
||||
return []
|
||||
return [
|
||||
DiscoveredSource(
|
||||
source_type=SourceType.SEC_EDGAR,
|
||||
name=f"{company.name} — SEC EDGAR filings",
|
||||
base_url=f"https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={cik}",
|
||||
configuration_metadata={"cik": cik},
|
||||
)
|
||||
]
|
||||
|
||||
async def _lookup_cik(self, company_name: str, settings) -> str | None:
|
||||
params = {
|
||||
"action": "getcompany",
|
||||
"company": company_name,
|
||||
"type": "10-K",
|
||||
"dateb": "",
|
||||
"owner": "include",
|
||||
"count": "5",
|
||||
"output": "atom",
|
||||
}
|
||||
url = str(httpx.URL(_SEARCH_URL, params=params))
|
||||
try:
|
||||
result = await fetch_with_retries(url, settings=settings, max_attempts=2)
|
||||
except (SsrfBlockedError, FetchError, httpx.HTTPError) as exc:
|
||||
logger.warning("sec_edgar_lookup_failed", company=company_name, error=str(exc))
|
||||
return None
|
||||
if result.status_code != 200:
|
||||
return None
|
||||
try:
|
||||
root = ElementTree.fromstring(result.content)
|
||||
except ElementTree.ParseError:
|
||||
return None
|
||||
|
||||
ns = {"a": "http://www.w3.org/2005/Atom"}
|
||||
for entry in root.findall(".//a:entry", ns):
|
||||
cik_elem = entry.find("a:content", ns)
|
||||
title_elem = entry.find("a:title", ns)
|
||||
if cik_elem is None or title_elem is None:
|
||||
continue
|
||||
# The atom feed embeds "CIK=0000320193" style text in <content>.
|
||||
text = "".join(cik_elem.itertext())
|
||||
if "CIK=" in text:
|
||||
cik = text.split("CIK=")[1].split("&")[0].strip()
|
||||
return cik.zfill(10)
|
||||
return None
|
||||
|
||||
async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult:
|
||||
cik = source.configuration_metadata.get("cik")
|
||||
if not cik:
|
||||
return CollectionResult(status=SourceStatus.FAILED, error="No CIK configured")
|
||||
|
||||
settings = get_settings()
|
||||
url = _SUBMISSIONS_URL.format(cik=str(cik).zfill(10))
|
||||
try:
|
||||
result = await fetch_with_retries(url, settings=settings)
|
||||
except SsrfBlockedError as exc:
|
||||
return CollectionResult(status=SourceStatus.BLOCKED_BY_POLICY, error=str(exc))
|
||||
except (FetchError, httpx.HTTPError) as exc:
|
||||
return CollectionResult(status=SourceStatus.FAILED, error=str(exc))
|
||||
|
||||
if result.status_code == 404:
|
||||
return CollectionResult(status=SourceStatus.FAILED, error="CIK not found on EDGAR")
|
||||
if result.status_code >= 400:
|
||||
return CollectionResult(status=SourceStatus.FAILED, error=f"HTTP {result.status_code}")
|
||||
|
||||
try:
|
||||
payload = json.loads(result.text)
|
||||
except json.JSONDecodeError:
|
||||
return CollectionResult(status=SourceStatus.FAILED, error="Malformed EDGAR response")
|
||||
|
||||
recent = payload.get("filings", {}).get("recent", {})
|
||||
forms = recent.get("form", [])
|
||||
dates = recent.get("filingDate", [])
|
||||
accessions = recent.get("accessionNumber", [])
|
||||
primary_docs = recent.get("primaryDocument", [])
|
||||
company_name = payload.get("name", company.name)
|
||||
|
||||
documents: list[CollectedDocument] = []
|
||||
for i, form in enumerate(forms):
|
||||
if form not in _RELEVANT_FORMS:
|
||||
continue
|
||||
if len(documents) >= 10:
|
||||
break
|
||||
accession = accessions[i].replace("-", "") if i < len(accessions) else ""
|
||||
primary_doc = primary_docs[i] if i < len(primary_docs) else ""
|
||||
filing_date = dates[i] if i < len(dates) else ""
|
||||
filing_url = (
|
||||
f"https://www.sec.gov/Archives/edgar/data/{int(cik)}/{accession}/{primary_doc}"
|
||||
if accession and primary_doc
|
||||
else url
|
||||
)
|
||||
text = normalize_whitespace(
|
||||
f"{company_name} filed a {form} with the SEC on {filing_date}.\n"
|
||||
f"Accession number: {accessions[i] if i < len(accessions) else 'unknown'}.\n"
|
||||
f"Filing document: {filing_url}"
|
||||
)
|
||||
documents.append(
|
||||
CollectedDocument(
|
||||
url=filing_url,
|
||||
canonical_url=filing_url,
|
||||
title=f"{company_name} {form} ({filing_date})",
|
||||
author="SEC EDGAR",
|
||||
publication_date=(
|
||||
datetime.strptime(filing_date, "%Y-%m-%d").replace(tzinfo=UTC)
|
||||
if filing_date
|
||||
else None
|
||||
),
|
||||
retrieved_date=datetime.now(UTC),
|
||||
content_text=text,
|
||||
content_hash=compute_content_hash(text),
|
||||
metadata={
|
||||
"form": form,
|
||||
"accession_number": accessions[i] if i < len(accessions) else None,
|
||||
},
|
||||
extraction_method="sec_edgar_metadata",
|
||||
http_status=result.status_code,
|
||||
trust_score=0.95,
|
||||
)
|
||||
)
|
||||
|
||||
# Zero relevant filings isn't a failure - the company may simply have
|
||||
# none in its recent filing history.
|
||||
return CollectionResult(status=SourceStatus.ACTIVE, documents=documents, pages_attempted=1)
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Official website collector: sitemap.xml + heuristic page discovery,
|
||||
robots.txt-respecting, capped crawl depth/page count.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from urllib.parse import urljoin
|
||||
from xml.etree import ElementTree
|
||||
|
||||
import httpx
|
||||
|
||||
from app.collectors.base import (
|
||||
CollectedDocument,
|
||||
CollectionResult,
|
||||
CompanyContext,
|
||||
DiscoveredSource,
|
||||
SourceConfig,
|
||||
)
|
||||
from app.collectors.extraction import (
|
||||
canonicalize_url,
|
||||
compute_content_hash,
|
||||
extract_readable_text,
|
||||
extract_title,
|
||||
)
|
||||
from app.collectors.robots import is_allowed
|
||||
from app.core.config import Settings, get_settings
|
||||
from app.core.http import FetchError, SsrfBlockedError, fetch_with_retries
|
||||
from app.core.logging import get_logger
|
||||
from app.models.enums import SourceStatus, SourceType
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
HEURISTIC_PATHS = [
|
||||
"",
|
||||
"/about",
|
||||
"/about-us",
|
||||
"/products",
|
||||
"/services",
|
||||
"/news",
|
||||
"/press",
|
||||
"/press-releases",
|
||||
"/careers",
|
||||
"/jobs",
|
||||
"/leadership",
|
||||
"/team",
|
||||
"/investors",
|
||||
"/investor-relations",
|
||||
"/sustainability",
|
||||
"/contact",
|
||||
]
|
||||
|
||||
|
||||
class WebsiteCollector:
|
||||
source_type = SourceType.WEBSITE
|
||||
|
||||
async def discover(self, company: CompanyContext) -> list[DiscoveredSource]:
|
||||
if not company.official_website:
|
||||
return []
|
||||
settings = get_settings()
|
||||
pages = await self._discover_pages(company.official_website, settings)
|
||||
return [
|
||||
DiscoveredSource(
|
||||
source_type=SourceType.WEBSITE,
|
||||
name=f"{company.name} — Official Website",
|
||||
base_url=company.official_website,
|
||||
configuration_metadata={"pages": pages},
|
||||
)
|
||||
]
|
||||
|
||||
async def _discover_pages(self, base_url: str, settings: Settings) -> list[str]:
|
||||
pages: list[str] = []
|
||||
|
||||
sitemap_urls = await self._read_sitemap(base_url, settings)
|
||||
pages.extend(sitemap_urls[: settings.max_pages_per_domain])
|
||||
|
||||
if len(pages) < settings.max_pages_per_domain:
|
||||
for path in HEURISTIC_PATHS:
|
||||
candidate = urljoin(base_url, path)
|
||||
if candidate not in pages:
|
||||
pages.append(candidate)
|
||||
if len(pages) >= settings.max_pages_per_domain:
|
||||
break
|
||||
|
||||
return pages[: settings.max_pages_per_domain]
|
||||
|
||||
async def _read_sitemap(self, base_url: str, settings: Settings) -> list[str]:
|
||||
sitemap_url = urljoin(base_url, "/sitemap.xml")
|
||||
try:
|
||||
result = await fetch_with_retries(sitemap_url, settings=settings, max_attempts=1)
|
||||
except (SsrfBlockedError, FetchError, httpx.HTTPError):
|
||||
return []
|
||||
if result.status_code != 200:
|
||||
return []
|
||||
try:
|
||||
root = ElementTree.fromstring(result.content)
|
||||
except ElementTree.ParseError:
|
||||
return []
|
||||
|
||||
ns = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"}
|
||||
urls = [loc.text.strip() for loc in root.findall(".//sm:url/sm:loc", ns) if loc.text]
|
||||
return urls
|
||||
|
||||
async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult:
|
||||
settings = get_settings()
|
||||
pages: list[str] = source.configuration_metadata.get("pages") or (
|
||||
[source.base_url] if source.base_url else []
|
||||
)
|
||||
pages = pages[: settings.max_pages_per_domain]
|
||||
|
||||
documents: list[CollectedDocument] = []
|
||||
seen_hashes: set[str] = set()
|
||||
attempted = 0
|
||||
any_success = False
|
||||
last_error: str | None = None
|
||||
|
||||
for page_url in pages:
|
||||
attempted += 1
|
||||
try:
|
||||
if not await is_allowed(page_url, settings=settings):
|
||||
logger.info("website_collector_robots_disallowed", url=page_url)
|
||||
continue
|
||||
|
||||
result = await fetch_with_retries(page_url, settings=settings)
|
||||
if result.status_code == 401 or result.status_code == 403:
|
||||
last_error = f"HTTP {result.status_code} (auth required) for {page_url}"
|
||||
continue
|
||||
if result.status_code >= 400:
|
||||
last_error = f"HTTP {result.status_code} for {page_url}"
|
||||
continue
|
||||
|
||||
text, method = extract_readable_text(result.text, page_url)
|
||||
if not text:
|
||||
continue
|
||||
content_hash = compute_content_hash(text)
|
||||
if content_hash in seen_hashes:
|
||||
continue
|
||||
seen_hashes.add(content_hash)
|
||||
|
||||
documents.append(
|
||||
CollectedDocument(
|
||||
url=page_url,
|
||||
canonical_url=canonicalize_url(result.final_url),
|
||||
title=extract_title(result.text),
|
||||
author=None,
|
||||
publication_date=None,
|
||||
retrieved_date=datetime.now(UTC),
|
||||
content_text=text,
|
||||
content_hash=content_hash,
|
||||
metadata={"http_status": result.status_code},
|
||||
extraction_method=method,
|
||||
http_status=result.status_code,
|
||||
trust_score=0.85,
|
||||
)
|
||||
)
|
||||
any_success = True
|
||||
except SsrfBlockedError as exc:
|
||||
last_error = str(exc)
|
||||
logger.warning("website_collector_ssrf_blocked", url=page_url, error=str(exc))
|
||||
except (FetchError, httpx.HTTPError) as exc:
|
||||
last_error = str(exc)
|
||||
logger.warning("website_collector_fetch_failed", url=page_url, error=str(exc))
|
||||
|
||||
if not documents:
|
||||
status = SourceStatus.FAILED if attempted > 0 else SourceStatus.ACTIVE
|
||||
return CollectionResult(
|
||||
status=status, documents=[], error=last_error, pages_attempted=attempted
|
||||
)
|
||||
|
||||
status = SourceStatus.ACTIVE if any_success else SourceStatus.FAILED
|
||||
return CollectionResult(
|
||||
status=status,
|
||||
documents=documents,
|
||||
error=last_error if not any_success else None,
|
||||
pages_attempted=attempted,
|
||||
)
|
||||
Reference in New Issue
Block a user