CIAgent/apps/api/app/collectors/jobs.py

"""Job posting collector: generic heuristic extraction from a company's own
careers page. Board-specific APIs (LinkedIn, Indeed, etc.) are not
implemented - most require paid access or prohibit automated collection in
their terms; see KNOWN_LIMITATIONS.md.
"""

from __future__ import annotations

from datetime import UTC, datetime
from urllib.parse import urljoin

import httpx
from bs4 import BeautifulSoup

from app.collectors.base import (
    CollectedDocument,
    CollectionResult,
    CompanyContext,
    DiscoveredSource,
    SourceConfig,
)
from app.collectors.extraction import (
    canonicalize_url,
    compute_content_hash,
    extract_readable_text,
    extract_title,
    normalize_whitespace,
)
from app.collectors.robots import is_allowed
from app.core.config import get_settings
from app.core.http import FetchError, SsrfBlockedError, fetch_with_retries
from app.models.enums import SourceStatus, SourceType

_JOB_LINK_KEYWORDS = ("job", "career", "position", "opening", "role", "vacan")
_MAX_LISTINGS = 50


class JobPostingCollector:
    source_type = SourceType.JOB_POSTING

    async def discover(self, company: CompanyContext) -> list[DiscoveredSource]:
        if not company.official_website:
            return []
        careers_url = urljoin(company.official_website, "/careers")
        return [
            DiscoveredSource(
                source_type=SourceType.JOB_POSTING,
                name=f"{company.name} — Careers",
                base_url=careers_url,
            )
        ]

    async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult:
        if not source.base_url:
            return CollectionResult(status=SourceStatus.FAILED, error="No careers URL configured")

        settings = get_settings()
        try:
            if not await is_allowed(source.base_url, settings=settings):
                return CollectionResult(
                    status=SourceStatus.BLOCKED_BY_POLICY,
                    error="Disallowed by robots.txt",
                    pages_attempted=1,
                )
            result = await fetch_with_retries(source.base_url, settings=settings)
        except SsrfBlockedError as exc:
            return CollectionResult(
                status=SourceStatus.BLOCKED_BY_POLICY, error=str(exc), pages_attempted=1
            )
        except (FetchError, httpx.HTTPError) as exc:
            return CollectionResult(status=SourceStatus.FAILED, error=str(exc), pages_attempted=1)

        if result.status_code in (401, 403):
            return CollectionResult(
                status=SourceStatus.AUTH_REQUIRED,
                error=f"HTTP {result.status_code}",
                pages_attempted=1,
            )
        if result.status_code >= 400:
            return CollectionResult(
                status=SourceStatus.FAILED, error=f"HTTP {result.status_code}", pages_attempted=1
            )

        listings = self._extract_job_listings(result.text, result.final_url)

        if not listings:
            # Non-standard careers page (e.g. a third-party ATS iframe) -
            # fall back to the whole page as one document rather than
            # reporting a failure for a page that did load successfully.
            text, method = extract_readable_text(result.text, source.base_url)
            if not text:
                return CollectionResult(
                    status=SourceStatus.FAILED, error="No extractable content", pages_attempted=1
                )
            document = CollectedDocument(
                url=source.base_url,
                canonical_url=canonicalize_url(result.final_url),
                title=extract_title(result.text),
                author=None,
                publication_date=None,
                retrieved_date=datetime.now(UTC),
                content_text=text,
                content_hash=compute_content_hash(text),
                metadata={"extraction": "fallback_whole_page"},
                extraction_method=method,
                http_status=result.status_code,
                trust_score=0.55,
            )
            return CollectionResult(
                status=SourceStatus.ACTIVE, documents=[document], pages_attempted=1
            )

        documents = [
            CollectedDocument(
                url=link,
                canonical_url=canonicalize_url(link),
                title=title,
                author=None,
                publication_date=None,
                retrieved_date=datetime.now(UTC),
                content_text=normalize_whitespace(f"{title}\n{snippet}"),
                content_hash=compute_content_hash(normalize_whitespace(f"{title}\n{snippet}")),
                metadata={"source_page": source.base_url},
                extraction_method="job_link_heuristic",
                http_status=result.status_code,
                trust_score=0.65,
            )
            for title, link, snippet in listings
        ]
        return CollectionResult(status=SourceStatus.ACTIVE, documents=documents, pages_attempted=1)

    def _extract_job_listings(self, html: str, base_url: str) -> list[tuple[str, str, str]]:
        soup = BeautifulSoup(html, "lxml")
        listings: list[tuple[str, str, str]] = []
        seen_links: set[str] = set()

        for anchor in soup.find_all("a", href=True):
            href = anchor["href"]
            text = anchor.get_text(strip=True)
            if not text or len(text) < 4 or len(text) > 150:
                continue
            if not any(keyword in href.lower() for keyword in _JOB_LINK_KEYWORDS):
                continue

            link = urljoin(base_url, href)
            if link in seen_links:
                continue
            seen_links.add(link)

            parent = anchor.find_parent()
            snippet = parent.get_text(" ", strip=True)[:300] if parent else ""
            listings.append((text, link, snippet))
            if len(listings) >= _MAX_LISTINGS:
                break

        return listings