Initial commit: CI Agent competitive-intelligence monitoring app

FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification).
2026-08-05 10:48:20 -04:00
commit 1a4c80958f
365 changed files with 43541 additions and 0 deletions
@@ -0,0 +1,131 @@
+"""Federal contracts collector via USASpending.gov's public Award Search
+API - free, keyless, no registration (a fixed, trusted, first-party
+integration endpoint like Brave/Twilio, so this calls httpx directly rather
+than through `safe_fetch`, which exists to guard arbitrary/user-supplied
+collector targets, not our own known API integrations).
+
+Offered for every company regardless of type, same as SecEdgarCollector -
+a private company simply returns zero awards, which is a normal empty
+result, not a failure.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+import httpx
+
+from app.collectors.base import (
+    CollectedDocument,
+    CollectionResult,
+    CompanyContext,
+    DiscoveredSource,
+    SourceConfig,
+)
+from app.collectors.extraction import compute_content_hash, normalize_whitespace
+from app.core.logging import get_logger
+from app.models.enums import SourceStatus, SourceType
+
+logger = get_logger(__name__)
+
+_SEARCH_URL = "https://api.usaspending.gov/api/v2/search/spending_by_award/"
+_AWARD_TYPE_CODES = ["A", "B", "C", "D"]  # contracts (definitive/BPA/purchase order/delivery order)
+
+
+class GovContractCollector:
+    source_type = SourceType.GOV_CONTRACT
+
+    async def discover(self, company: CompanyContext) -> list[DiscoveredSource]:
+        return [
+            DiscoveredSource(
+                source_type=SourceType.GOV_CONTRACT,
+                name=f"{company.name} — Federal Contracts",
+                base_url=None,
+                configuration_metadata={"recipient_search_text": company.name},
+            )
+        ]
+
+    async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult:
+        recipient = source.configuration_metadata.get("recipient_search_text") or company.name
+
+        body = {
+            "filters": {
+                "recipient_search_text": [recipient],
+                "award_type_codes": _AWARD_TYPE_CODES,
+            },
+            "fields": [
+                "Award ID",
+                "Recipient Name",
+                "Award Amount",
+                "Start Date",
+                "Awarding Agency",
+                "Description",
+            ],
+            "sort": "Award Amount",
+            "order": "desc",
+            "page": 1,
+            "limit": 25,
+        }
+        try:
+            async with httpx.AsyncClient(timeout=20) as client:
+                response = await client.post(_SEARCH_URL, json=body)
+        except httpx.HTTPError as exc:
+            return CollectionResult(status=SourceStatus.FAILED, error=str(exc))
+
+        if response.status_code >= 400:
+            return CollectionResult(
+                status=SourceStatus.FAILED,
+                error=f"USASpending API error {response.status_code}: {response.text[:200]}",
+                pages_attempted=1,
+            )
+
+        try:
+            payload = response.json()
+        except ValueError:
+            return CollectionResult(
+                status=SourceStatus.FAILED,
+                error="Malformed USASpending response",
+                pages_attempted=1,
+            )
+
+        results = payload.get("results", [])
+        documents: list[CollectedDocument] = []
+        for award in results:
+            award_id = award.get("Award ID", "unknown")
+            agency = award.get("Awarding Agency", "Unknown agency")
+            amount = award.get("Award Amount")
+            amount_display = (
+                f"${amount:,.0f}" if isinstance(amount, (int, float)) else "unknown amount"
+            )
+            start_date = award.get("Start Date", "")
+            description = award.get("Description") or ""
+
+            text = normalize_whitespace(
+                f"{award.get('Recipient Name', recipient)} was awarded federal contract "
+                f"{award_id} by {agency} for {amount_display}, starting {start_date}. "
+                f"{description}"
+            )
+            documents.append(
+                CollectedDocument(
+                    url=_SEARCH_URL,
+                    canonical_url=_SEARCH_URL,
+                    title=f"{agency}: {award_id} — {amount_display}",
+                    author="USASpending.gov",
+                    publication_date=(
+                        datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=UTC)
+                        if start_date
+                        else None
+                    ),
+                    retrieved_date=datetime.now(UTC),
+                    content_text=text,
+                    content_hash=compute_content_hash(text),
+                    metadata={"award_id": award_id, "awarding_agency": agency},
+                    extraction_method="usaspending_api",
+                    http_status=response.status_code,
+                    trust_score=0.8,
+                )
+            )
+
+        # Zero awards isn't a failure - most companies never win a federal
+        # contract, same non-error empty-result handling as SecEdgarCollector.
+        return CollectionResult(status=SourceStatus.ACTIVE, documents=documents, pages_attempted=1)