saksham
1a4c80958f
FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification).
222 lines
7.8 KiB
Python
222 lines
7.8 KiB
Python
"""Integration tests: collection_service persisting collector output through
|
|
real repositories into a real (SQLite) database. No live network - respx
|
|
mocks every HTTP call and DNS resolution is patched to a fixed public IP."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import uuid
|
|
from unittest.mock import patch
|
|
|
|
import httpx
|
|
import pytest
|
|
import respx
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import selectinload
|
|
|
|
from app.models.company import Company
|
|
from app.models.enums import SourceStatus, SourceType
|
|
from app.models.monitor_configuration import MonitorConfiguration
|
|
from app.models.snapshot import Snapshot
|
|
from app.models.source import Source
|
|
from app.models.source_document import SourceDocument
|
|
from app.repositories.source_repository import SourceRepository
|
|
from app.services import collection_service
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _no_real_dns():
|
|
with patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]):
|
|
yield
|
|
|
|
|
|
async def _make_company(
|
|
db_session, *, official_website: str | None = "https://example.com"
|
|
) -> Company:
|
|
company = Company(
|
|
id=uuid.uuid4(),
|
|
user_id=uuid.uuid4(),
|
|
name="Acme Corp",
|
|
slug=f"acme-corp-{uuid.uuid4().hex[:6]}",
|
|
official_website=official_website,
|
|
)
|
|
db_session.add(company)
|
|
db_session.add(MonitorConfiguration(company_id=company.id))
|
|
await db_session.commit()
|
|
result = await db_session.execute(
|
|
select(Company)
|
|
.where(Company.id == company.id)
|
|
.options(
|
|
selectinload(Company.aliases),
|
|
selectinload(Company.competitors),
|
|
selectinload(Company.enrichment),
|
|
)
|
|
)
|
|
return result.scalar_one()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_discover_sources_creates_website_and_job_posting_sources(db_session):
|
|
company = await _make_company(db_session)
|
|
|
|
with respx.mock:
|
|
respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404))
|
|
respx.get("https://example.com/sitemap.xml").mock(return_value=httpx.Response(404))
|
|
respx.get("https://api.github.com/search/users").mock(
|
|
return_value=httpx.Response(200, text=json.dumps({"items": []}))
|
|
)
|
|
respx.get("https://www.sec.gov/cgi-bin/browse-edgar").mock(
|
|
return_value=httpx.Response(
|
|
200, text='<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom"></feed>'
|
|
)
|
|
)
|
|
created = await collection_service.discover_sources_for_company(db_session, company)
|
|
|
|
types = {s.source_type for s in created}
|
|
assert SourceType.WEBSITE in types
|
|
assert SourceType.JOB_POSTING in types
|
|
assert SourceType.GITHUB not in types # no org match -> not created
|
|
assert SourceType.SEC_EDGAR not in types # no CIK match -> not created
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_discover_sources_is_idempotent(db_session):
|
|
company = await _make_company(db_session)
|
|
|
|
with respx.mock:
|
|
respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404))
|
|
respx.get("https://example.com/sitemap.xml").mock(return_value=httpx.Response(404))
|
|
respx.get("https://api.github.com/search/users").mock(
|
|
return_value=httpx.Response(200, text=json.dumps({"items": []}))
|
|
)
|
|
respx.get("https://www.sec.gov/cgi-bin/browse-edgar").mock(
|
|
return_value=httpx.Response(
|
|
200, text='<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom"></feed>'
|
|
)
|
|
)
|
|
first_batch = await collection_service.discover_sources_for_company(db_session, company)
|
|
second_batch = await collection_service.discover_sources_for_company(db_session, company)
|
|
|
|
assert len(first_batch) > 0
|
|
assert second_batch == [] # already-discovered sources aren't recreated
|
|
|
|
repo = SourceRepository(db_session)
|
|
all_sources = await repo.list_for_company(company.id)
|
|
assert len(all_sources) == len(first_batch)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_collect_source_persists_documents_and_snapshot(db_session, settings):
|
|
company = await _make_company(db_session)
|
|
repo = SourceRepository(db_session)
|
|
source = await repo.create(
|
|
company_id=company.id,
|
|
source_type=SourceType.CUSTOM_URL,
|
|
name="Pricing page",
|
|
base_url="https://example.com/pricing",
|
|
)
|
|
await db_session.commit()
|
|
|
|
with respx.mock:
|
|
respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404))
|
|
respx.get("https://example.com/pricing").mock(
|
|
return_value=httpx.Response(
|
|
200,
|
|
html=(
|
|
"<html><head><title>Pricing</title></head><body>"
|
|
"<article><h1>Pricing</h1><p>Base plan is $49 per month.</p></article>"
|
|
"</body></html>"
|
|
),
|
|
)
|
|
)
|
|
result = await collection_service.collect_source(db_session, settings, source, company)
|
|
|
|
assert result.status == SourceStatus.ACTIVE
|
|
|
|
docs = (
|
|
(
|
|
await db_session.execute(
|
|
select(SourceDocument).where(SourceDocument.source_id == source.id)
|
|
)
|
|
)
|
|
.scalars()
|
|
.all()
|
|
)
|
|
assert len(docs) == 1
|
|
assert "$49 per month" in docs[0].content_text
|
|
|
|
snapshots = (
|
|
(await db_session.execute(select(Snapshot).where(Snapshot.source_id == source.id)))
|
|
.scalars()
|
|
.all()
|
|
)
|
|
assert len(snapshots) == 1
|
|
|
|
refreshed_source = (
|
|
await db_session.execute(select(Source).where(Source.id == source.id))
|
|
).scalar_one()
|
|
assert refreshed_source.status == SourceStatus.ACTIVE
|
|
assert refreshed_source.last_successful_check is not None
|
|
assert refreshed_source.failure_count == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_collect_source_deduplicates_unchanged_content_across_runs(db_session, settings):
|
|
company = await _make_company(db_session)
|
|
repo = SourceRepository(db_session)
|
|
source = await repo.create(
|
|
company_id=company.id,
|
|
source_type=SourceType.CUSTOM_URL,
|
|
name="About page",
|
|
base_url="https://example.com/about",
|
|
)
|
|
await db_session.commit()
|
|
|
|
html = (
|
|
"<html><head><title>About</title></head><body>"
|
|
"<article><h1>About</h1><p>We build electric trucks.</p></article></body></html>"
|
|
)
|
|
|
|
for _ in range(2):
|
|
with respx.mock:
|
|
respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404))
|
|
respx.get("https://example.com/about").mock(return_value=httpx.Response(200, html=html))
|
|
await collection_service.collect_source(db_session, settings, source, company)
|
|
|
|
docs = (
|
|
(
|
|
await db_session.execute(
|
|
select(SourceDocument).where(SourceDocument.source_id == source.id)
|
|
)
|
|
)
|
|
.scalars()
|
|
.all()
|
|
)
|
|
assert len(docs) == 1 # second run's identical content was deduplicated
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_collect_source_marks_failure_and_increments_count(db_session, settings):
|
|
company = await _make_company(db_session)
|
|
repo = SourceRepository(db_session)
|
|
source = await repo.create(
|
|
company_id=company.id,
|
|
source_type=SourceType.CUSTOM_URL,
|
|
name="Broken page",
|
|
base_url="https://example.com/broken",
|
|
)
|
|
await db_session.commit()
|
|
|
|
with respx.mock:
|
|
respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404))
|
|
respx.get("https://example.com/broken").mock(return_value=httpx.Response(500))
|
|
result = await collection_service.collect_source(db_session, settings, source, company)
|
|
|
|
assert result.status == SourceStatus.FAILED
|
|
|
|
refreshed_source = (
|
|
await db_session.execute(select(Source).where(Source.id == source.id))
|
|
).scalar_one()
|
|
assert refreshed_source.failure_count == 1
|
|
assert refreshed_source.last_successful_check is None
|