CIAgent/apps/api/app/collectors/github.py

"""GitHub collector - public organization/repository metadata via the
public REST API. `GITHUB_TOKEN` is optional and only raises the rate limit;
nothing here requires authentication.
"""

from __future__ import annotations

import json
from datetime import UTC, datetime

import httpx

from app.collectors.base import (
    CollectedDocument,
    CollectionResult,
    CompanyContext,
    DiscoveredSource,
    SourceConfig,
)
from app.collectors.extraction import compute_content_hash, normalize_whitespace
from app.core.config import Settings, get_settings
from app.core.http import FetchError, SsrfBlockedError, fetch_with_retries
from app.core.logging import get_logger
from app.models.enums import SourceStatus, SourceType

logger = get_logger(__name__)

_API_BASE = "https://api.github.com"


def _auth_headers(settings: Settings) -> dict[str, str]:
    headers = {"Accept": "application/vnd.github+json"}
    if settings.github_token:
        headers["Authorization"] = f"Bearer {settings.github_token}"
    return headers


class GithubCollector:
    source_type = SourceType.GITHUB

    async def discover(self, company: CompanyContext) -> list[DiscoveredSource]:
        settings = get_settings()
        org_login = await self._find_org(company.name, settings)
        if org_login is None:
            return []
        return [
            DiscoveredSource(
                source_type=SourceType.GITHUB,
                name=f"{company.name} — GitHub",
                base_url=f"https://github.com/{org_login}",
                configuration_metadata={"org": org_login},
            )
        ]

    async def _find_org(self, company_name: str, settings: Settings) -> str | None:
        url = str(
            httpx.URL(
                f"{_API_BASE}/search/users",
                params={"q": f"{company_name} type:org", "per_page": 1},
            )
        )
        try:
            result = await fetch_with_retries(
                url, settings=settings, max_attempts=2, extra_headers=_auth_headers(settings)
            )
        except (SsrfBlockedError, FetchError, httpx.HTTPError) as exc:
            logger.warning("github_org_search_failed", company=company_name, error=str(exc))
            return None
        if result.status_code != 200:
            return None
        try:
            payload = json.loads(result.text)
        except json.JSONDecodeError:
            return None
        items = payload.get("items", [])
        return items[0]["login"] if items else None

    async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult:
        org = source.configuration_metadata.get("org")
        if not org:
            return CollectionResult(status=SourceStatus.FAILED, error="No GitHub org configured")

        settings = get_settings()
        url = str(
            httpx.URL(f"{_API_BASE}/orgs/{org}/repos", params={"sort": "pushed", "per_page": 15})
        )
        try:
            result = await fetch_with_retries(
                url, settings=settings, extra_headers=_auth_headers(settings)
            )
        except SsrfBlockedError as exc:
            return CollectionResult(status=SourceStatus.BLOCKED_BY_POLICY, error=str(exc))
        except (FetchError, httpx.HTTPError) as exc:
            return CollectionResult(status=SourceStatus.FAILED, error=str(exc))

        if result.status_code == 404:
            return CollectionResult(
                status=SourceStatus.FAILED, error=f"GitHub org not found: {org}"
            )
        if result.status_code == 403:
            return CollectionResult(
                status=SourceStatus.RATE_LIMITED, error="GitHub API rate limited"
            )
        if result.status_code >= 400:
            return CollectionResult(status=SourceStatus.FAILED, error=f"HTTP {result.status_code}")

        try:
            repos = json.loads(result.text)
        except json.JSONDecodeError:
            return CollectionResult(status=SourceStatus.FAILED, error="Malformed GitHub response")

        documents: list[CollectedDocument] = []
        for repo in repos:
            text = normalize_whitespace(
                f"{repo.get('full_name')}\n"
                f"{repo.get('description') or ''}\n"
                f"Language: {repo.get('language') or 'unknown'}\n"
                f"Stars: {repo.get('stargazers_count', 0)}\n"
                f"Last pushed: {repo.get('pushed_at')}"
            )
            documents.append(
                CollectedDocument(
                    url=repo.get("html_url"),
                    canonical_url=repo.get("html_url"),
                    title=repo.get("full_name"),
                    author=org,
                    publication_date=(
                        datetime.fromisoformat(repo["pushed_at"].replace("Z", "+00:00"))
                        if repo.get("pushed_at")
                        else None
                    ),
                    retrieved_date=datetime.now(UTC),
                    content_text=text,
                    content_hash=compute_content_hash(text),
                    metadata={
                        "stars": repo.get("stargazers_count"),
                        "language": repo.get("language"),
                        "archived": repo.get("archived"),
                    },
                    extraction_method="github_api",
                    http_status=result.status_code,
                    trust_score=0.75,
                )
            )

        status = SourceStatus.ACTIVE
        return CollectionResult(status=status, documents=documents, pages_attempted=1)