Initial commit: CI Agent competitive-intelligence monitoring app

FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification).
2026-08-05 10:48:20 -04:00
commit 1a4c80958f
365 changed files with 43541 additions and 0 deletions
@@ -0,0 +1,85 @@
+"""Customer review source collector.
+
+Most review platforms (G2, Trustpilot, Glassdoor, etc.) either prohibit
+automated scraping in their terms or require a paid API. This collector
+implements the `SourceCollector` interface and a documented fixture adapter
+for local development/testing; it never scrapes a review site directly and
+never fabricates review data when no permitted live provider is configured.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+from pathlib import Path
+
+from app.collectors.base import (
+    CollectedDocument,
+    CollectionResult,
+    CompanyContext,
+    DiscoveredSource,
+    SourceConfig,
+)
+from app.collectors.extraction import compute_content_hash, normalize_whitespace
+from app.models.enums import SourceStatus, SourceType
+
+_FIXTURES_DIR = Path(__file__).resolve().parents[2] / "tests" / "fixtures" / "reviews"
+
+
+class ReviewSourceCollector:
+    source_type = SourceType.REVIEW
+
+    async def discover(self, company: CompanyContext) -> list[DiscoveredSource]:
+        return []
+
+    async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult:
+        fixture_key = source.configuration_metadata.get("fixture_key")
+        if not fixture_key:
+            return CollectionResult(
+                status=SourceStatus.DISABLED,
+                error=(
+                    "No review data provider is configured. Most review platforms prohibit "
+                    "automated scraping in their terms; this collector implements the "
+                    "SourceCollector interface for a future permitted API integration but "
+                    "does not fabricate results without one."
+                ),
+            )
+
+        fixture_path = _FIXTURES_DIR / f"{fixture_key}.json"
+        if not fixture_path.exists():
+            return CollectionResult(
+                status=SourceStatus.DISABLED,
+                error=f"No fixture found for {fixture_key!r} and no live provider is configured.",
+            )
+
+        payload = json.loads(fixture_path.read_text(encoding="utf-8"))
+        documents: list[CollectedDocument] = []
+        for entry in payload.get("reviews", []):
+            text = normalize_whitespace(
+                f"Rating: {entry.get('rating', 'n/a')}/5\n\n{entry.get('body', '')}"
+            )
+            documents.append(
+                CollectedDocument(
+                    url=entry.get("url", fixture_path.as_uri()),
+                    canonical_url=entry.get("url", fixture_path.as_uri()),
+                    title=entry.get("title") or "Customer review",
+                    author=entry.get("author"),
+                    publication_date=(
+                        datetime.fromisoformat(entry["date"]).replace(tzinfo=UTC)
+                        if entry.get("date")
+                        else None
+                    ),
+                    retrieved_date=datetime.now(UTC),
+                    content_text=text,
+                    content_hash=compute_content_hash(text),
+                    metadata={
+                        "is_fixture": True,
+                        "data_source": "fixture",
+                        "fixture_key": fixture_key,
+                        "rating": entry.get("rating"),
+                    },
+                    extraction_method="fixture",
+                    trust_score=0.4,
+                )
+            )
+        return CollectionResult(status=SourceStatus.ACTIVE, documents=documents, pages_attempted=1)