saksham
1a4c80958f
FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification).
86 lines
3.4 KiB
Python
86 lines
3.4 KiB
Python
"""Customer review source collector.
|
|
|
|
Most review platforms (G2, Trustpilot, Glassdoor, etc.) either prohibit
|
|
automated scraping in their terms or require a paid API. This collector
|
|
implements the `SourceCollector` interface and a documented fixture adapter
|
|
for local development/testing; it never scrapes a review site directly and
|
|
never fabricates review data when no permitted live provider is configured.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
from app.collectors.base import (
|
|
CollectedDocument,
|
|
CollectionResult,
|
|
CompanyContext,
|
|
DiscoveredSource,
|
|
SourceConfig,
|
|
)
|
|
from app.collectors.extraction import compute_content_hash, normalize_whitespace
|
|
from app.models.enums import SourceStatus, SourceType
|
|
|
|
_FIXTURES_DIR = Path(__file__).resolve().parents[2] / "tests" / "fixtures" / "reviews"
|
|
|
|
|
|
class ReviewSourceCollector:
|
|
source_type = SourceType.REVIEW
|
|
|
|
async def discover(self, company: CompanyContext) -> list[DiscoveredSource]:
|
|
return []
|
|
|
|
async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult:
|
|
fixture_key = source.configuration_metadata.get("fixture_key")
|
|
if not fixture_key:
|
|
return CollectionResult(
|
|
status=SourceStatus.DISABLED,
|
|
error=(
|
|
"No review data provider is configured. Most review platforms prohibit "
|
|
"automated scraping in their terms; this collector implements the "
|
|
"SourceCollector interface for a future permitted API integration but "
|
|
"does not fabricate results without one."
|
|
),
|
|
)
|
|
|
|
fixture_path = _FIXTURES_DIR / f"{fixture_key}.json"
|
|
if not fixture_path.exists():
|
|
return CollectionResult(
|
|
status=SourceStatus.DISABLED,
|
|
error=f"No fixture found for {fixture_key!r} and no live provider is configured.",
|
|
)
|
|
|
|
payload = json.loads(fixture_path.read_text(encoding="utf-8"))
|
|
documents: list[CollectedDocument] = []
|
|
for entry in payload.get("reviews", []):
|
|
text = normalize_whitespace(
|
|
f"Rating: {entry.get('rating', 'n/a')}/5\n\n{entry.get('body', '')}"
|
|
)
|
|
documents.append(
|
|
CollectedDocument(
|
|
url=entry.get("url", fixture_path.as_uri()),
|
|
canonical_url=entry.get("url", fixture_path.as_uri()),
|
|
title=entry.get("title") or "Customer review",
|
|
author=entry.get("author"),
|
|
publication_date=(
|
|
datetime.fromisoformat(entry["date"]).replace(tzinfo=UTC)
|
|
if entry.get("date")
|
|
else None
|
|
),
|
|
retrieved_date=datetime.now(UTC),
|
|
content_text=text,
|
|
content_hash=compute_content_hash(text),
|
|
metadata={
|
|
"is_fixture": True,
|
|
"data_source": "fixture",
|
|
"fixture_key": fixture_key,
|
|
"rating": entry.get("rating"),
|
|
},
|
|
extraction_method="fixture",
|
|
trust_score=0.4,
|
|
)
|
|
)
|
|
return CollectionResult(status=SourceStatus.ACTIVE, documents=documents, pages_attempted=1)
|