Initial commit: CI Agent competitive-intelligence monitoring app

FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring,
source collection, LLM-based change analysis, enrichment, and account
security (Turnstile, escalating lockout, email verification).

apps/api/app/collectors/custom_url.py

@@ -0,0 +1,87 @@
+"""User-supplied custom URL collector - fetches, extracts, and monitors a
+single public URL the user explicitly added."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+import httpx
+
+from app.collectors.base import (
+    CollectedDocument,
+    CollectionResult,
+    CompanyContext,
+    DiscoveredSource,
+    SourceConfig,
+)
+from app.collectors.extraction import (
+    canonicalize_url,
+    compute_content_hash,
+    extract_readable_text,
+    extract_title,
+)
+from app.collectors.robots import is_allowed
+from app.core.config import get_settings
+from app.core.http import FetchError, SsrfBlockedError, fetch_with_retries
+from app.models.enums import SourceStatus, SourceType
+
+
+class CustomUrlCollector:
+    source_type = SourceType.CUSTOM_URL
+
+    async def discover(self, company: CompanyContext) -> list[DiscoveredSource]:
+        return []  # Custom URLs are always user-supplied, never auto-discovered.
+
+    async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult:
+        if not source.base_url:
+            return CollectionResult(status=SourceStatus.FAILED, error="No URL configured")
+
+        settings = get_settings()
+        try:
+            if not await is_allowed(source.base_url, settings=settings):
+                return CollectionResult(
+                    status=SourceStatus.BLOCKED_BY_POLICY,
+                    error="Disallowed by robots.txt",
+                    pages_attempted=1,
+                )
+
+            result = await fetch_with_retries(source.base_url, settings=settings)
+        except SsrfBlockedError as exc:
+            return CollectionResult(
+                status=SourceStatus.BLOCKED_BY_POLICY, error=str(exc), pages_attempted=1
+            )
+        except (FetchError, httpx.HTTPError) as exc:
+            return CollectionResult(status=SourceStatus.FAILED, error=str(exc), pages_attempted=1)
+
+        if result.status_code in (401, 403):
+            return CollectionResult(
+                status=SourceStatus.AUTH_REQUIRED,
+                error=f"HTTP {result.status_code}",
+                pages_attempted=1,
+            )
+        if result.status_code >= 400:
+            return CollectionResult(
+                status=SourceStatus.FAILED, error=f"HTTP {result.status_code}", pages_attempted=1
+            )
+
+        text, method = extract_readable_text(result.text, source.base_url)
+        if not text:
+            return CollectionResult(
+                status=SourceStatus.FAILED, error="No extractable content", pages_attempted=1
+            )
+
+        document = CollectedDocument(
+            url=source.base_url,
+            canonical_url=canonicalize_url(result.final_url),
+            title=extract_title(result.text),
+            author=None,
+            publication_date=None,
+            retrieved_date=datetime.now(UTC),
+            content_text=text,
+            content_hash=compute_content_hash(text),
+            metadata={"http_status": result.status_code},
+            extraction_method=method,
+            http_status=result.status_code,
+            trust_score=0.7,
+        )
+        return CollectionResult(status=SourceStatus.ACTIVE, documents=[document], pages_attempted=1)