Initial commit: CI Agent competitive-intelligence monitoring app
FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification).
This commit is contained in:
@@ -0,0 +1,156 @@
|
||||
"""Job posting collector: generic heuristic extraction from a company's own
|
||||
careers page. Board-specific APIs (LinkedIn, Indeed, etc.) are not
|
||||
implemented - most require paid access or prohibit automated collection in
|
||||
their terms; see KNOWN_LIMITATIONS.md.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from app.collectors.base import (
|
||||
CollectedDocument,
|
||||
CollectionResult,
|
||||
CompanyContext,
|
||||
DiscoveredSource,
|
||||
SourceConfig,
|
||||
)
|
||||
from app.collectors.extraction import (
|
||||
canonicalize_url,
|
||||
compute_content_hash,
|
||||
extract_readable_text,
|
||||
extract_title,
|
||||
normalize_whitespace,
|
||||
)
|
||||
from app.collectors.robots import is_allowed
|
||||
from app.core.config import get_settings
|
||||
from app.core.http import FetchError, SsrfBlockedError, fetch_with_retries
|
||||
from app.models.enums import SourceStatus, SourceType
|
||||
|
||||
_JOB_LINK_KEYWORDS = ("job", "career", "position", "opening", "role", "vacan")
|
||||
_MAX_LISTINGS = 50
|
||||
|
||||
|
||||
class JobPostingCollector:
|
||||
source_type = SourceType.JOB_POSTING
|
||||
|
||||
async def discover(self, company: CompanyContext) -> list[DiscoveredSource]:
|
||||
if not company.official_website:
|
||||
return []
|
||||
careers_url = urljoin(company.official_website, "/careers")
|
||||
return [
|
||||
DiscoveredSource(
|
||||
source_type=SourceType.JOB_POSTING,
|
||||
name=f"{company.name} — Careers",
|
||||
base_url=careers_url,
|
||||
)
|
||||
]
|
||||
|
||||
async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult:
|
||||
if not source.base_url:
|
||||
return CollectionResult(status=SourceStatus.FAILED, error="No careers URL configured")
|
||||
|
||||
settings = get_settings()
|
||||
try:
|
||||
if not await is_allowed(source.base_url, settings=settings):
|
||||
return CollectionResult(
|
||||
status=SourceStatus.BLOCKED_BY_POLICY,
|
||||
error="Disallowed by robots.txt",
|
||||
pages_attempted=1,
|
||||
)
|
||||
result = await fetch_with_retries(source.base_url, settings=settings)
|
||||
except SsrfBlockedError as exc:
|
||||
return CollectionResult(
|
||||
status=SourceStatus.BLOCKED_BY_POLICY, error=str(exc), pages_attempted=1
|
||||
)
|
||||
except (FetchError, httpx.HTTPError) as exc:
|
||||
return CollectionResult(status=SourceStatus.FAILED, error=str(exc), pages_attempted=1)
|
||||
|
||||
if result.status_code in (401, 403):
|
||||
return CollectionResult(
|
||||
status=SourceStatus.AUTH_REQUIRED,
|
||||
error=f"HTTP {result.status_code}",
|
||||
pages_attempted=1,
|
||||
)
|
||||
if result.status_code >= 400:
|
||||
return CollectionResult(
|
||||
status=SourceStatus.FAILED, error=f"HTTP {result.status_code}", pages_attempted=1
|
||||
)
|
||||
|
||||
listings = self._extract_job_listings(result.text, result.final_url)
|
||||
|
||||
if not listings:
|
||||
# Non-standard careers page (e.g. a third-party ATS iframe) -
|
||||
# fall back to the whole page as one document rather than
|
||||
# reporting a failure for a page that did load successfully.
|
||||
text, method = extract_readable_text(result.text, source.base_url)
|
||||
if not text:
|
||||
return CollectionResult(
|
||||
status=SourceStatus.FAILED, error="No extractable content", pages_attempted=1
|
||||
)
|
||||
document = CollectedDocument(
|
||||
url=source.base_url,
|
||||
canonical_url=canonicalize_url(result.final_url),
|
||||
title=extract_title(result.text),
|
||||
author=None,
|
||||
publication_date=None,
|
||||
retrieved_date=datetime.now(UTC),
|
||||
content_text=text,
|
||||
content_hash=compute_content_hash(text),
|
||||
metadata={"extraction": "fallback_whole_page"},
|
||||
extraction_method=method,
|
||||
http_status=result.status_code,
|
||||
trust_score=0.55,
|
||||
)
|
||||
return CollectionResult(
|
||||
status=SourceStatus.ACTIVE, documents=[document], pages_attempted=1
|
||||
)
|
||||
|
||||
documents = [
|
||||
CollectedDocument(
|
||||
url=link,
|
||||
canonical_url=canonicalize_url(link),
|
||||
title=title,
|
||||
author=None,
|
||||
publication_date=None,
|
||||
retrieved_date=datetime.now(UTC),
|
||||
content_text=normalize_whitespace(f"{title}\n{snippet}"),
|
||||
content_hash=compute_content_hash(normalize_whitespace(f"{title}\n{snippet}")),
|
||||
metadata={"source_page": source.base_url},
|
||||
extraction_method="job_link_heuristic",
|
||||
http_status=result.status_code,
|
||||
trust_score=0.65,
|
||||
)
|
||||
for title, link, snippet in listings
|
||||
]
|
||||
return CollectionResult(status=SourceStatus.ACTIVE, documents=documents, pages_attempted=1)
|
||||
|
||||
def _extract_job_listings(self, html: str, base_url: str) -> list[tuple[str, str, str]]:
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
listings: list[tuple[str, str, str]] = []
|
||||
seen_links: set[str] = set()
|
||||
|
||||
for anchor in soup.find_all("a", href=True):
|
||||
href = anchor["href"]
|
||||
text = anchor.get_text(strip=True)
|
||||
if not text or len(text) < 4 or len(text) > 150:
|
||||
continue
|
||||
if not any(keyword in href.lower() for keyword in _JOB_LINK_KEYWORDS):
|
||||
continue
|
||||
|
||||
link = urljoin(base_url, href)
|
||||
if link in seen_links:
|
||||
continue
|
||||
seen_links.add(link)
|
||||
|
||||
parent = anchor.find_parent()
|
||||
snippet = parent.get_text(" ", strip=True)[:300] if parent else ""
|
||||
listings.append((text, link, snippet))
|
||||
if len(listings) >= _MAX_LISTINGS:
|
||||
break
|
||||
|
||||
return listings
|
||||
Reference in New Issue
Block a user