"""Integration tests: collection_service persisting collector output through real repositories into a real (SQLite) database. No live network - respx mocks every HTTP call and DNS resolution is patched to a fixed public IP.""" from __future__ import annotations import json import uuid from unittest.mock import patch import httpx import pytest import respx from sqlalchemy import select from sqlalchemy.orm import selectinload from app.models.company import Company from app.models.enums import SourceStatus, SourceType from app.models.monitor_configuration import MonitorConfiguration from app.models.snapshot import Snapshot from app.models.source import Source from app.models.source_document import SourceDocument from app.repositories.source_repository import SourceRepository from app.services import collection_service @pytest.fixture(autouse=True) def _no_real_dns(): with patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]): yield async def _make_company( db_session, *, official_website: str | None = "https://example.com" ) -> Company: company = Company( id=uuid.uuid4(), user_id=uuid.uuid4(), name="Acme Corp", slug=f"acme-corp-{uuid.uuid4().hex[:6]}", official_website=official_website, ) db_session.add(company) db_session.add(MonitorConfiguration(company_id=company.id)) await db_session.commit() result = await db_session.execute( select(Company) .where(Company.id == company.id) .options( selectinload(Company.aliases), selectinload(Company.competitors), selectinload(Company.enrichment), ) ) return result.scalar_one() @pytest.mark.asyncio async def test_discover_sources_creates_website_and_job_posting_sources(db_session): company = await _make_company(db_session) with respx.mock: respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404)) respx.get("https://example.com/sitemap.xml").mock(return_value=httpx.Response(404)) respx.get("https://api.github.com/search/users").mock( return_value=httpx.Response(200, text=json.dumps({"items": []})) ) respx.get("https://www.sec.gov/cgi-bin/browse-edgar").mock( return_value=httpx.Response( 200, text='' ) ) created = await collection_service.discover_sources_for_company(db_session, company) types = {s.source_type for s in created} assert SourceType.WEBSITE in types assert SourceType.JOB_POSTING in types assert SourceType.GITHUB not in types # no org match -> not created assert SourceType.SEC_EDGAR not in types # no CIK match -> not created @pytest.mark.asyncio async def test_discover_sources_is_idempotent(db_session): company = await _make_company(db_session) with respx.mock: respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404)) respx.get("https://example.com/sitemap.xml").mock(return_value=httpx.Response(404)) respx.get("https://api.github.com/search/users").mock( return_value=httpx.Response(200, text=json.dumps({"items": []})) ) respx.get("https://www.sec.gov/cgi-bin/browse-edgar").mock( return_value=httpx.Response( 200, text='' ) ) first_batch = await collection_service.discover_sources_for_company(db_session, company) second_batch = await collection_service.discover_sources_for_company(db_session, company) assert len(first_batch) > 0 assert second_batch == [] # already-discovered sources aren't recreated repo = SourceRepository(db_session) all_sources = await repo.list_for_company(company.id) assert len(all_sources) == len(first_batch) @pytest.mark.asyncio async def test_collect_source_persists_documents_and_snapshot(db_session, settings): company = await _make_company(db_session) repo = SourceRepository(db_session) source = await repo.create( company_id=company.id, source_type=SourceType.CUSTOM_URL, name="Pricing page", base_url="https://example.com/pricing", ) await db_session.commit() with respx.mock: respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404)) respx.get("https://example.com/pricing").mock( return_value=httpx.Response( 200, html=( "Pricing" "

Pricing

Base plan is $49 per month.

" "" ), ) ) result = await collection_service.collect_source(db_session, settings, source, company) assert result.status == SourceStatus.ACTIVE docs = ( ( await db_session.execute( select(SourceDocument).where(SourceDocument.source_id == source.id) ) ) .scalars() .all() ) assert len(docs) == 1 assert "$49 per month" in docs[0].content_text snapshots = ( (await db_session.execute(select(Snapshot).where(Snapshot.source_id == source.id))) .scalars() .all() ) assert len(snapshots) == 1 refreshed_source = ( await db_session.execute(select(Source).where(Source.id == source.id)) ).scalar_one() assert refreshed_source.status == SourceStatus.ACTIVE assert refreshed_source.last_successful_check is not None assert refreshed_source.failure_count == 0 @pytest.mark.asyncio async def test_collect_source_deduplicates_unchanged_content_across_runs(db_session, settings): company = await _make_company(db_session) repo = SourceRepository(db_session) source = await repo.create( company_id=company.id, source_type=SourceType.CUSTOM_URL, name="About page", base_url="https://example.com/about", ) await db_session.commit() html = ( "About" "

About

We build electric trucks.

" ) for _ in range(2): with respx.mock: respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404)) respx.get("https://example.com/about").mock(return_value=httpx.Response(200, html=html)) await collection_service.collect_source(db_session, settings, source, company) docs = ( ( await db_session.execute( select(SourceDocument).where(SourceDocument.source_id == source.id) ) ) .scalars() .all() ) assert len(docs) == 1 # second run's identical content was deduplicated @pytest.mark.asyncio async def test_collect_source_marks_failure_and_increments_count(db_session, settings): company = await _make_company(db_session) repo = SourceRepository(db_session) source = await repo.create( company_id=company.id, source_type=SourceType.CUSTOM_URL, name="Broken page", base_url="https://example.com/broken", ) await db_session.commit() with respx.mock: respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404)) respx.get("https://example.com/broken").mock(return_value=httpx.Response(500)) result = await collection_service.collect_source(db_session, settings, source, company) assert result.status == SourceStatus.FAILED refreshed_source = ( await db_session.execute(select(Source).where(Source.id == source.id)) ).scalar_one() assert refreshed_source.failure_count == 1 assert refreshed_source.last_successful_check is None