Initial commit: CI Agent competitive-intelligence monitoring app
FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification).
This commit is contained in:
@@ -0,0 +1,221 @@
|
||||
"""Integration tests: collection_service persisting collector output through
|
||||
real repositories into a real (SQLite) database. No live network - respx
|
||||
mocks every HTTP call and DNS resolution is patched to a fixed public IP."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from unittest.mock import patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
from app.models.company import Company
|
||||
from app.models.enums import SourceStatus, SourceType
|
||||
from app.models.monitor_configuration import MonitorConfiguration
|
||||
from app.models.snapshot import Snapshot
|
||||
from app.models.source import Source
|
||||
from app.models.source_document import SourceDocument
|
||||
from app.repositories.source_repository import SourceRepository
|
||||
from app.services import collection_service
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_real_dns():
|
||||
with patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]):
|
||||
yield
|
||||
|
||||
|
||||
async def _make_company(
|
||||
db_session, *, official_website: str | None = "https://example.com"
|
||||
) -> Company:
|
||||
company = Company(
|
||||
id=uuid.uuid4(),
|
||||
user_id=uuid.uuid4(),
|
||||
name="Acme Corp",
|
||||
slug=f"acme-corp-{uuid.uuid4().hex[:6]}",
|
||||
official_website=official_website,
|
||||
)
|
||||
db_session.add(company)
|
||||
db_session.add(MonitorConfiguration(company_id=company.id))
|
||||
await db_session.commit()
|
||||
result = await db_session.execute(
|
||||
select(Company)
|
||||
.where(Company.id == company.id)
|
||||
.options(
|
||||
selectinload(Company.aliases),
|
||||
selectinload(Company.competitors),
|
||||
selectinload(Company.enrichment),
|
||||
)
|
||||
)
|
||||
return result.scalar_one()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_sources_creates_website_and_job_posting_sources(db_session):
|
||||
company = await _make_company(db_session)
|
||||
|
||||
with respx.mock:
|
||||
respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404))
|
||||
respx.get("https://example.com/sitemap.xml").mock(return_value=httpx.Response(404))
|
||||
respx.get("https://api.github.com/search/users").mock(
|
||||
return_value=httpx.Response(200, text=json.dumps({"items": []}))
|
||||
)
|
||||
respx.get("https://www.sec.gov/cgi-bin/browse-edgar").mock(
|
||||
return_value=httpx.Response(
|
||||
200, text='<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom"></feed>'
|
||||
)
|
||||
)
|
||||
created = await collection_service.discover_sources_for_company(db_session, company)
|
||||
|
||||
types = {s.source_type for s in created}
|
||||
assert SourceType.WEBSITE in types
|
||||
assert SourceType.JOB_POSTING in types
|
||||
assert SourceType.GITHUB not in types # no org match -> not created
|
||||
assert SourceType.SEC_EDGAR not in types # no CIK match -> not created
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_sources_is_idempotent(db_session):
|
||||
company = await _make_company(db_session)
|
||||
|
||||
with respx.mock:
|
||||
respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404))
|
||||
respx.get("https://example.com/sitemap.xml").mock(return_value=httpx.Response(404))
|
||||
respx.get("https://api.github.com/search/users").mock(
|
||||
return_value=httpx.Response(200, text=json.dumps({"items": []}))
|
||||
)
|
||||
respx.get("https://www.sec.gov/cgi-bin/browse-edgar").mock(
|
||||
return_value=httpx.Response(
|
||||
200, text='<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom"></feed>'
|
||||
)
|
||||
)
|
||||
first_batch = await collection_service.discover_sources_for_company(db_session, company)
|
||||
second_batch = await collection_service.discover_sources_for_company(db_session, company)
|
||||
|
||||
assert len(first_batch) > 0
|
||||
assert second_batch == [] # already-discovered sources aren't recreated
|
||||
|
||||
repo = SourceRepository(db_session)
|
||||
all_sources = await repo.list_for_company(company.id)
|
||||
assert len(all_sources) == len(first_batch)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_source_persists_documents_and_snapshot(db_session, settings):
|
||||
company = await _make_company(db_session)
|
||||
repo = SourceRepository(db_session)
|
||||
source = await repo.create(
|
||||
company_id=company.id,
|
||||
source_type=SourceType.CUSTOM_URL,
|
||||
name="Pricing page",
|
||||
base_url="https://example.com/pricing",
|
||||
)
|
||||
await db_session.commit()
|
||||
|
||||
with respx.mock:
|
||||
respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404))
|
||||
respx.get("https://example.com/pricing").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
html=(
|
||||
"<html><head><title>Pricing</title></head><body>"
|
||||
"<article><h1>Pricing</h1><p>Base plan is $49 per month.</p></article>"
|
||||
"</body></html>"
|
||||
),
|
||||
)
|
||||
)
|
||||
result = await collection_service.collect_source(db_session, settings, source, company)
|
||||
|
||||
assert result.status == SourceStatus.ACTIVE
|
||||
|
||||
docs = (
|
||||
(
|
||||
await db_session.execute(
|
||||
select(SourceDocument).where(SourceDocument.source_id == source.id)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
assert len(docs) == 1
|
||||
assert "$49 per month" in docs[0].content_text
|
||||
|
||||
snapshots = (
|
||||
(await db_session.execute(select(Snapshot).where(Snapshot.source_id == source.id)))
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
assert len(snapshots) == 1
|
||||
|
||||
refreshed_source = (
|
||||
await db_session.execute(select(Source).where(Source.id == source.id))
|
||||
).scalar_one()
|
||||
assert refreshed_source.status == SourceStatus.ACTIVE
|
||||
assert refreshed_source.last_successful_check is not None
|
||||
assert refreshed_source.failure_count == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_source_deduplicates_unchanged_content_across_runs(db_session, settings):
|
||||
company = await _make_company(db_session)
|
||||
repo = SourceRepository(db_session)
|
||||
source = await repo.create(
|
||||
company_id=company.id,
|
||||
source_type=SourceType.CUSTOM_URL,
|
||||
name="About page",
|
||||
base_url="https://example.com/about",
|
||||
)
|
||||
await db_session.commit()
|
||||
|
||||
html = (
|
||||
"<html><head><title>About</title></head><body>"
|
||||
"<article><h1>About</h1><p>We build electric trucks.</p></article></body></html>"
|
||||
)
|
||||
|
||||
for _ in range(2):
|
||||
with respx.mock:
|
||||
respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404))
|
||||
respx.get("https://example.com/about").mock(return_value=httpx.Response(200, html=html))
|
||||
await collection_service.collect_source(db_session, settings, source, company)
|
||||
|
||||
docs = (
|
||||
(
|
||||
await db_session.execute(
|
||||
select(SourceDocument).where(SourceDocument.source_id == source.id)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
assert len(docs) == 1 # second run's identical content was deduplicated
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_source_marks_failure_and_increments_count(db_session, settings):
|
||||
company = await _make_company(db_session)
|
||||
repo = SourceRepository(db_session)
|
||||
source = await repo.create(
|
||||
company_id=company.id,
|
||||
source_type=SourceType.CUSTOM_URL,
|
||||
name="Broken page",
|
||||
base_url="https://example.com/broken",
|
||||
)
|
||||
await db_session.commit()
|
||||
|
||||
with respx.mock:
|
||||
respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404))
|
||||
respx.get("https://example.com/broken").mock(return_value=httpx.Response(500))
|
||||
result = await collection_service.collect_source(db_session, settings, source, company)
|
||||
|
||||
assert result.status == SourceStatus.FAILED
|
||||
|
||||
refreshed_source = (
|
||||
await db_session.execute(select(Source).where(Source.id == source.id))
|
||||
).scalar_one()
|
||||
assert refreshed_source.failure_count == 1
|
||||
assert refreshed_source.last_successful_check is None
|
||||
Reference in New Issue
Block a user