saksham
1a4c80958f
FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification).
132 lines
4.9 KiB
Python
132 lines
4.9 KiB
Python
"""Federal contracts collector via USASpending.gov's public Award Search
|
|
API - free, keyless, no registration (a fixed, trusted, first-party
|
|
integration endpoint like Brave/Twilio, so this calls httpx directly rather
|
|
than through `safe_fetch`, which exists to guard arbitrary/user-supplied
|
|
collector targets, not our own known API integrations).
|
|
|
|
Offered for every company regardless of type, same as SecEdgarCollector -
|
|
a private company simply returns zero awards, which is a normal empty
|
|
result, not a failure.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime
|
|
|
|
import httpx
|
|
|
|
from app.collectors.base import (
|
|
CollectedDocument,
|
|
CollectionResult,
|
|
CompanyContext,
|
|
DiscoveredSource,
|
|
SourceConfig,
|
|
)
|
|
from app.collectors.extraction import compute_content_hash, normalize_whitespace
|
|
from app.core.logging import get_logger
|
|
from app.models.enums import SourceStatus, SourceType
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
_SEARCH_URL = "https://api.usaspending.gov/api/v2/search/spending_by_award/"
|
|
_AWARD_TYPE_CODES = ["A", "B", "C", "D"] # contracts (definitive/BPA/purchase order/delivery order)
|
|
|
|
|
|
class GovContractCollector:
|
|
source_type = SourceType.GOV_CONTRACT
|
|
|
|
async def discover(self, company: CompanyContext) -> list[DiscoveredSource]:
|
|
return [
|
|
DiscoveredSource(
|
|
source_type=SourceType.GOV_CONTRACT,
|
|
name=f"{company.name} — Federal Contracts",
|
|
base_url=None,
|
|
configuration_metadata={"recipient_search_text": company.name},
|
|
)
|
|
]
|
|
|
|
async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult:
|
|
recipient = source.configuration_metadata.get("recipient_search_text") or company.name
|
|
|
|
body = {
|
|
"filters": {
|
|
"recipient_search_text": [recipient],
|
|
"award_type_codes": _AWARD_TYPE_CODES,
|
|
},
|
|
"fields": [
|
|
"Award ID",
|
|
"Recipient Name",
|
|
"Award Amount",
|
|
"Start Date",
|
|
"Awarding Agency",
|
|
"Description",
|
|
],
|
|
"sort": "Award Amount",
|
|
"order": "desc",
|
|
"page": 1,
|
|
"limit": 25,
|
|
}
|
|
try:
|
|
async with httpx.AsyncClient(timeout=20) as client:
|
|
response = await client.post(_SEARCH_URL, json=body)
|
|
except httpx.HTTPError as exc:
|
|
return CollectionResult(status=SourceStatus.FAILED, error=str(exc))
|
|
|
|
if response.status_code >= 400:
|
|
return CollectionResult(
|
|
status=SourceStatus.FAILED,
|
|
error=f"USASpending API error {response.status_code}: {response.text[:200]}",
|
|
pages_attempted=1,
|
|
)
|
|
|
|
try:
|
|
payload = response.json()
|
|
except ValueError:
|
|
return CollectionResult(
|
|
status=SourceStatus.FAILED,
|
|
error="Malformed USASpending response",
|
|
pages_attempted=1,
|
|
)
|
|
|
|
results = payload.get("results", [])
|
|
documents: list[CollectedDocument] = []
|
|
for award in results:
|
|
award_id = award.get("Award ID", "unknown")
|
|
agency = award.get("Awarding Agency", "Unknown agency")
|
|
amount = award.get("Award Amount")
|
|
amount_display = (
|
|
f"${amount:,.0f}" if isinstance(amount, (int, float)) else "unknown amount"
|
|
)
|
|
start_date = award.get("Start Date", "")
|
|
description = award.get("Description") or ""
|
|
|
|
text = normalize_whitespace(
|
|
f"{award.get('Recipient Name', recipient)} was awarded federal contract "
|
|
f"{award_id} by {agency} for {amount_display}, starting {start_date}. "
|
|
f"{description}"
|
|
)
|
|
documents.append(
|
|
CollectedDocument(
|
|
url=_SEARCH_URL,
|
|
canonical_url=_SEARCH_URL,
|
|
title=f"{agency}: {award_id} — {amount_display}",
|
|
author="USASpending.gov",
|
|
publication_date=(
|
|
datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=UTC)
|
|
if start_date
|
|
else None
|
|
),
|
|
retrieved_date=datetime.now(UTC),
|
|
content_text=text,
|
|
content_hash=compute_content_hash(text),
|
|
metadata={"award_id": award_id, "awarding_agency": agency},
|
|
extraction_method="usaspending_api",
|
|
http_status=response.status_code,
|
|
trust_score=0.8,
|
|
)
|
|
)
|
|
|
|
# Zero awards isn't a failure - most companies never win a federal
|
|
# contract, same non-error empty-result handling as SecEdgarCollector.
|
|
return CollectionResult(status=SourceStatus.ACTIVE, documents=documents, pages_attempted=1)
|