Initial commit: CI Agent competitive-intelligence monitoring app

FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring,
source collection, LLM-based change analysis, enrichment, and account
security (Turnstile, escalating lockout, email verification).

apps/api/app/collectors/github.py

@@ -0,0 +1,147 @@
+"""GitHub collector - public organization/repository metadata via the
+public REST API. `GITHUB_TOKEN` is optional and only raises the rate limit;
+nothing here requires authentication.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+
+import httpx
+
+from app.collectors.base import (
+    CollectedDocument,
+    CollectionResult,
+    CompanyContext,
+    DiscoveredSource,
+    SourceConfig,
+)
+from app.collectors.extraction import compute_content_hash, normalize_whitespace
+from app.core.config import Settings, get_settings
+from app.core.http import FetchError, SsrfBlockedError, fetch_with_retries
+from app.core.logging import get_logger
+from app.models.enums import SourceStatus, SourceType
+
+logger = get_logger(__name__)
+
+_API_BASE = "https://api.github.com"
+
+
+def _auth_headers(settings: Settings) -> dict[str, str]:
+    headers = {"Accept": "application/vnd.github+json"}
+    if settings.github_token:
+        headers["Authorization"] = f"Bearer {settings.github_token}"
+    return headers
+
+
+class GithubCollector:
+    source_type = SourceType.GITHUB
+
+    async def discover(self, company: CompanyContext) -> list[DiscoveredSource]:
+        settings = get_settings()
+        org_login = await self._find_org(company.name, settings)
+        if org_login is None:
+            return []
+        return [
+            DiscoveredSource(
+                source_type=SourceType.GITHUB,
+                name=f"{company.name} — GitHub",
+                base_url=f"https://github.com/{org_login}",
+                configuration_metadata={"org": org_login},
+            )
+        ]
+
+    async def _find_org(self, company_name: str, settings: Settings) -> str | None:
+        url = str(
+            httpx.URL(
+                f"{_API_BASE}/search/users",
+                params={"q": f"{company_name} type:org", "per_page": 1},
+            )
+        )
+        try:
+            result = await fetch_with_retries(
+                url, settings=settings, max_attempts=2, extra_headers=_auth_headers(settings)
+            )
+        except (SsrfBlockedError, FetchError, httpx.HTTPError) as exc:
+            logger.warning("github_org_search_failed", company=company_name, error=str(exc))
+            return None
+        if result.status_code != 200:
+            return None
+        try:
+            payload = json.loads(result.text)
+        except json.JSONDecodeError:
+            return None
+        items = payload.get("items", [])
+        return items[0]["login"] if items else None
+
+    async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult:
+        org = source.configuration_metadata.get("org")
+        if not org:
+            return CollectionResult(status=SourceStatus.FAILED, error="No GitHub org configured")
+
+        settings = get_settings()
+        url = str(
+            httpx.URL(f"{_API_BASE}/orgs/{org}/repos", params={"sort": "pushed", "per_page": 15})
+        )
+        try:
+            result = await fetch_with_retries(
+                url, settings=settings, extra_headers=_auth_headers(settings)
+            )
+        except SsrfBlockedError as exc:
+            return CollectionResult(status=SourceStatus.BLOCKED_BY_POLICY, error=str(exc))
+        except (FetchError, httpx.HTTPError) as exc:
+            return CollectionResult(status=SourceStatus.FAILED, error=str(exc))
+
+        if result.status_code == 404:
+            return CollectionResult(
+                status=SourceStatus.FAILED, error=f"GitHub org not found: {org}"
+            )
+        if result.status_code == 403:
+            return CollectionResult(
+                status=SourceStatus.RATE_LIMITED, error="GitHub API rate limited"
+            )
+        if result.status_code >= 400:
+            return CollectionResult(status=SourceStatus.FAILED, error=f"HTTP {result.status_code}")
+
+        try:
+            repos = json.loads(result.text)
+        except json.JSONDecodeError:
+            return CollectionResult(status=SourceStatus.FAILED, error="Malformed GitHub response")
+
+        documents: list[CollectedDocument] = []
+        for repo in repos:
+            text = normalize_whitespace(
+                f"{repo.get('full_name')}\n"
+                f"{repo.get('description') or ''}\n"
+                f"Language: {repo.get('language') or 'unknown'}\n"
+                f"Stars: {repo.get('stargazers_count', 0)}\n"
+                f"Last pushed: {repo.get('pushed_at')}"
+            )
+            documents.append(
+                CollectedDocument(
+                    url=repo.get("html_url"),
+                    canonical_url=repo.get("html_url"),
+                    title=repo.get("full_name"),
+                    author=org,
+                    publication_date=(
+                        datetime.fromisoformat(repo["pushed_at"].replace("Z", "+00:00"))
+                        if repo.get("pushed_at")
+                        else None
+                    ),
+                    retrieved_date=datetime.now(UTC),
+                    content_text=text,
+                    content_hash=compute_content_hash(text),
+                    metadata={
+                        "stars": repo.get("stargazers_count"),
+                        "language": repo.get("language"),
+                        "archived": repo.get("archived"),
+                    },
+                    extraction_method="github_api",
+                    http_status=result.status_code,
+                    trust_score=0.75,
+                )
+            )
+
+        status = SourceStatus.ACTIVE
+        return CollectionResult(status=status, documents=documents, pages_attempted=1)