"""GitHub collector - public organization/repository metadata via the public REST API. `GITHUB_TOKEN` is optional and only raises the rate limit; nothing here requires authentication. """ from __future__ import annotations import json from datetime import UTC, datetime import httpx from app.collectors.base import ( CollectedDocument, CollectionResult, CompanyContext, DiscoveredSource, SourceConfig, ) from app.collectors.extraction import compute_content_hash, normalize_whitespace from app.core.config import Settings, get_settings from app.core.http import FetchError, SsrfBlockedError, fetch_with_retries from app.core.logging import get_logger from app.models.enums import SourceStatus, SourceType logger = get_logger(__name__) _API_BASE = "https://api.github.com" def _auth_headers(settings: Settings) -> dict[str, str]: headers = {"Accept": "application/vnd.github+json"} if settings.github_token: headers["Authorization"] = f"Bearer {settings.github_token}" return headers class GithubCollector: source_type = SourceType.GITHUB async def discover(self, company: CompanyContext) -> list[DiscoveredSource]: settings = get_settings() org_login = await self._find_org(company.name, settings) if org_login is None: return [] return [ DiscoveredSource( source_type=SourceType.GITHUB, name=f"{company.name} — GitHub", base_url=f"https://github.com/{org_login}", configuration_metadata={"org": org_login}, ) ] async def _find_org(self, company_name: str, settings: Settings) -> str | None: url = str( httpx.URL( f"{_API_BASE}/search/users", params={"q": f"{company_name} type:org", "per_page": 1}, ) ) try: result = await fetch_with_retries( url, settings=settings, max_attempts=2, extra_headers=_auth_headers(settings) ) except (SsrfBlockedError, FetchError, httpx.HTTPError) as exc: logger.warning("github_org_search_failed", company=company_name, error=str(exc)) return None if result.status_code != 200: return None try: payload = json.loads(result.text) except json.JSONDecodeError: return None items = payload.get("items", []) return items[0]["login"] if items else None async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult: org = source.configuration_metadata.get("org") if not org: return CollectionResult(status=SourceStatus.FAILED, error="No GitHub org configured") settings = get_settings() url = str( httpx.URL(f"{_API_BASE}/orgs/{org}/repos", params={"sort": "pushed", "per_page": 15}) ) try: result = await fetch_with_retries( url, settings=settings, extra_headers=_auth_headers(settings) ) except SsrfBlockedError as exc: return CollectionResult(status=SourceStatus.BLOCKED_BY_POLICY, error=str(exc)) except (FetchError, httpx.HTTPError) as exc: return CollectionResult(status=SourceStatus.FAILED, error=str(exc)) if result.status_code == 404: return CollectionResult( status=SourceStatus.FAILED, error=f"GitHub org not found: {org}" ) if result.status_code == 403: return CollectionResult( status=SourceStatus.RATE_LIMITED, error="GitHub API rate limited" ) if result.status_code >= 400: return CollectionResult(status=SourceStatus.FAILED, error=f"HTTP {result.status_code}") try: repos = json.loads(result.text) except json.JSONDecodeError: return CollectionResult(status=SourceStatus.FAILED, error="Malformed GitHub response") documents: list[CollectedDocument] = [] for repo in repos: text = normalize_whitespace( f"{repo.get('full_name')}\n" f"{repo.get('description') or ''}\n" f"Language: {repo.get('language') or 'unknown'}\n" f"Stars: {repo.get('stargazers_count', 0)}\n" f"Last pushed: {repo.get('pushed_at')}" ) documents.append( CollectedDocument( url=repo.get("html_url"), canonical_url=repo.get("html_url"), title=repo.get("full_name"), author=org, publication_date=( datetime.fromisoformat(repo["pushed_at"].replace("Z", "+00:00")) if repo.get("pushed_at") else None ), retrieved_date=datetime.now(UTC), content_text=text, content_hash=compute_content_hash(text), metadata={ "stars": repo.get("stargazers_count"), "language": repo.get("language"), "archived": repo.get("archived"), }, extraction_method="github_api", http_status=result.status_code, trust_score=0.75, ) ) status = SourceStatus.ACTIVE return CollectionResult(status=status, documents=documents, pages_attempted=1)