"""Customer review source collector. Most review platforms (G2, Trustpilot, Glassdoor, etc.) either prohibit automated scraping in their terms or require a paid API. This collector implements the `SourceCollector` interface and a documented fixture adapter for local development/testing; it never scrapes a review site directly and never fabricates review data when no permitted live provider is configured. """ from __future__ import annotations import json from datetime import UTC, datetime from pathlib import Path from app.collectors.base import ( CollectedDocument, CollectionResult, CompanyContext, DiscoveredSource, SourceConfig, ) from app.collectors.extraction import compute_content_hash, normalize_whitespace from app.models.enums import SourceStatus, SourceType _FIXTURES_DIR = Path(__file__).resolve().parents[2] / "tests" / "fixtures" / "reviews" class ReviewSourceCollector: source_type = SourceType.REVIEW async def discover(self, company: CompanyContext) -> list[DiscoveredSource]: return [] async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult: fixture_key = source.configuration_metadata.get("fixture_key") if not fixture_key: return CollectionResult( status=SourceStatus.DISABLED, error=( "No review data provider is configured. Most review platforms prohibit " "automated scraping in their terms; this collector implements the " "SourceCollector interface for a future permitted API integration but " "does not fabricate results without one." ), ) fixture_path = _FIXTURES_DIR / f"{fixture_key}.json" if not fixture_path.exists(): return CollectionResult( status=SourceStatus.DISABLED, error=f"No fixture found for {fixture_key!r} and no live provider is configured.", ) payload = json.loads(fixture_path.read_text(encoding="utf-8")) documents: list[CollectedDocument] = [] for entry in payload.get("reviews", []): text = normalize_whitespace( f"Rating: {entry.get('rating', 'n/a')}/5\n\n{entry.get('body', '')}" ) documents.append( CollectedDocument( url=entry.get("url", fixture_path.as_uri()), canonical_url=entry.get("url", fixture_path.as_uri()), title=entry.get("title") or "Customer review", author=entry.get("author"), publication_date=( datetime.fromisoformat(entry["date"]).replace(tzinfo=UTC) if entry.get("date") else None ), retrieved_date=datetime.now(UTC), content_text=text, content_hash=compute_content_hash(text), metadata={ "is_fixture": True, "data_source": "fixture", "fixture_key": fixture_key, "rating": entry.get("rating"), }, extraction_method="fixture", trust_score=0.4, ) ) return CollectionResult(status=SourceStatus.ACTIVE, documents=documents, pages_attempted=1)