saksham
1a4c80958f
FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification).
88 lines
3.1 KiB
Python
88 lines
3.1 KiB
Python
"""User-supplied custom URL collector - fetches, extracts, and monitors a
|
|
single public URL the user explicitly added."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime
|
|
|
|
import httpx
|
|
|
|
from app.collectors.base import (
|
|
CollectedDocument,
|
|
CollectionResult,
|
|
CompanyContext,
|
|
DiscoveredSource,
|
|
SourceConfig,
|
|
)
|
|
from app.collectors.extraction import (
|
|
canonicalize_url,
|
|
compute_content_hash,
|
|
extract_readable_text,
|
|
extract_title,
|
|
)
|
|
from app.collectors.robots import is_allowed
|
|
from app.core.config import get_settings
|
|
from app.core.http import FetchError, SsrfBlockedError, fetch_with_retries
|
|
from app.models.enums import SourceStatus, SourceType
|
|
|
|
|
|
class CustomUrlCollector:
|
|
source_type = SourceType.CUSTOM_URL
|
|
|
|
async def discover(self, company: CompanyContext) -> list[DiscoveredSource]:
|
|
return [] # Custom URLs are always user-supplied, never auto-discovered.
|
|
|
|
async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult:
|
|
if not source.base_url:
|
|
return CollectionResult(status=SourceStatus.FAILED, error="No URL configured")
|
|
|
|
settings = get_settings()
|
|
try:
|
|
if not await is_allowed(source.base_url, settings=settings):
|
|
return CollectionResult(
|
|
status=SourceStatus.BLOCKED_BY_POLICY,
|
|
error="Disallowed by robots.txt",
|
|
pages_attempted=1,
|
|
)
|
|
|
|
result = await fetch_with_retries(source.base_url, settings=settings)
|
|
except SsrfBlockedError as exc:
|
|
return CollectionResult(
|
|
status=SourceStatus.BLOCKED_BY_POLICY, error=str(exc), pages_attempted=1
|
|
)
|
|
except (FetchError, httpx.HTTPError) as exc:
|
|
return CollectionResult(status=SourceStatus.FAILED, error=str(exc), pages_attempted=1)
|
|
|
|
if result.status_code in (401, 403):
|
|
return CollectionResult(
|
|
status=SourceStatus.AUTH_REQUIRED,
|
|
error=f"HTTP {result.status_code}",
|
|
pages_attempted=1,
|
|
)
|
|
if result.status_code >= 400:
|
|
return CollectionResult(
|
|
status=SourceStatus.FAILED, error=f"HTTP {result.status_code}", pages_attempted=1
|
|
)
|
|
|
|
text, method = extract_readable_text(result.text, source.base_url)
|
|
if not text:
|
|
return CollectionResult(
|
|
status=SourceStatus.FAILED, error="No extractable content", pages_attempted=1
|
|
)
|
|
|
|
document = CollectedDocument(
|
|
url=source.base_url,
|
|
canonical_url=canonicalize_url(result.final_url),
|
|
title=extract_title(result.text),
|
|
author=None,
|
|
publication_date=None,
|
|
retrieved_date=datetime.now(UTC),
|
|
content_text=text,
|
|
content_hash=compute_content_hash(text),
|
|
metadata={"http_status": result.status_code},
|
|
extraction_method=method,
|
|
http_status=result.status_code,
|
|
trust_score=0.7,
|
|
)
|
|
return CollectionResult(status=SourceStatus.ACTIVE, documents=[document], pages_attempted=1)
|