FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification).
221 lines
8.7 KiB
Python
221 lines
8.7 KiB
Python
"""Monitoring run endpoints, exercised via HTTP with CELERY_TASK_ALWAYS_EAGER
|
|
so `.delay()` runs the task synchronously in-process (see app/tasks/base.py
|
|
for why that needs a threaded asyncio bridge to work from an async route
|
|
handler). No live network: every collector's discovery/collection call is
|
|
respx-mocked."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import uuid
|
|
from unittest.mock import patch
|
|
|
|
import httpx
|
|
import pytest
|
|
import respx
|
|
|
|
|
|
def _register_and_login(client) -> dict[str, str]:
|
|
email = f"user-{uuid.uuid4().hex[:12]}@example.com"
|
|
client.post(
|
|
"/api/v1/auth/register",
|
|
json={"email": email, "password": "correct-horse-1", "display_name": "Test User"},
|
|
)
|
|
tokens = client.post(
|
|
"/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"}
|
|
).json()
|
|
return {"Authorization": f"Bearer {tokens['access_token']}"}
|
|
|
|
|
|
def _create_company(client, headers, **overrides):
|
|
payload = {
|
|
"name": f"Run Test Co {uuid.uuid4().hex[:6]}",
|
|
"frequency_type": "weekly",
|
|
**overrides,
|
|
}
|
|
return client.post("/api/v1/companies", json=payload, headers=headers).json()
|
|
|
|
|
|
def _mock_empty_discovery():
|
|
"""Every discoverable collector type returns nothing (or a harmless
|
|
stub), so a first run doesn't need per-source mocks for whatever
|
|
discovery happens to find. RSS (Google News) and GOV_CONTRACT
|
|
(USASpending) are always discovered unconditionally - see
|
|
discovery_service.py's _PREVIEWABLE_TYPES - so their collect() calls
|
|
need mocking here too, unlike GitHub/SEC which discover.() itself."""
|
|
respx.get("https://api.github.com/search/users").mock(
|
|
return_value=httpx.Response(200, text=json.dumps({"items": []}))
|
|
)
|
|
respx.get("https://www.sec.gov/cgi-bin/browse-edgar").mock(
|
|
return_value=httpx.Response(
|
|
200, text='<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom"></feed>'
|
|
)
|
|
)
|
|
respx.get(url__regex=r"https://news\.google\.com/rss/search.*").mock(
|
|
return_value=httpx.Response(
|
|
200,
|
|
text=(
|
|
'<?xml version="1.0"?><rss version="2.0"><channel><title>Google News</title>'
|
|
"<item><title>Test</title><link>https://example.com/news</link>"
|
|
"<description>Test news item</description></item></channel></rss>"
|
|
),
|
|
)
|
|
)
|
|
respx.post("https://api.usaspending.gov/api/v2/search/spending_by_award/").mock(
|
|
return_value=httpx.Response(200, json={"results": []})
|
|
)
|
|
|
|
|
|
def test_run_now_executes_and_reports_final_status(client):
|
|
headers = _register_and_login(client)
|
|
company = _create_company(client, headers)
|
|
|
|
with patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]):
|
|
with respx.mock:
|
|
_mock_empty_discovery()
|
|
resp = client.post(f"/api/v1/companies/{company['id']}/run", headers=headers)
|
|
|
|
assert resp.status_code == 202
|
|
run = resp.json()
|
|
assert run["trigger_type"] == "manual"
|
|
|
|
# Eager mode means the task already finished by the time .delay() returns.
|
|
detail = client.get(f"/api/v1/runs/{run['id']}", headers=headers).json()
|
|
assert detail["status"] == "successful"
|
|
assert detail["completed_at"] is not None
|
|
|
|
|
|
async def test_run_now_is_idempotent_for_an_active_run(db_session, settings):
|
|
"""A run-now call while one is already queued/running returns that same
|
|
run instead of enqueuing a duplicate. Exercised at the service layer
|
|
directly: under CELERY_TASK_ALWAYS_EAGER a `.delay()` call runs the task
|
|
to completion before returning, so an HTTP-level test can never observe
|
|
an in-flight run to dedupe against."""
|
|
from app.models.company import Company
|
|
from app.models.enums import MonitoringRunTrigger
|
|
from app.models.monitor_configuration import MonitorConfiguration
|
|
from app.repositories.monitoring_run_repository import MonitoringRunRepository
|
|
from app.services import monitoring_service
|
|
|
|
user_id = uuid.uuid4()
|
|
company = Company(
|
|
id=uuid.uuid4(),
|
|
user_id=user_id,
|
|
name="Idempotency Co",
|
|
slug=f"idempotency-co-{uuid.uuid4().hex[:6]}",
|
|
)
|
|
db_session.add(company)
|
|
db_session.add(MonitorConfiguration(company_id=company.id))
|
|
await db_session.commit()
|
|
|
|
run_repo = MonitoringRunRepository(db_session)
|
|
existing = await run_repo.create(
|
|
company_id=company.id, trigger_type=MonitoringRunTrigger.SCHEDULED
|
|
)
|
|
await db_session.commit()
|
|
|
|
with patch("app.tasks.collection.run_monitoring.delay") as mock_delay:
|
|
result = await monitoring_service.enqueue_run_now(db_session, settings, user_id, company.id)
|
|
|
|
assert result.id == existing.id
|
|
mock_delay.assert_not_called()
|
|
|
|
|
|
async def test_run_now_enforces_daily_manual_run_cap(db_session, settings):
|
|
from app.core.errors import RateLimitedError
|
|
from app.models.company import Company
|
|
from app.models.enums import MonitoringRunStatus, MonitoringRunTrigger
|
|
from app.models.monitor_configuration import MonitorConfiguration
|
|
from app.repositories.monitoring_run_repository import MonitoringRunRepository
|
|
from app.services import monitoring_service
|
|
|
|
user_id = uuid.uuid4()
|
|
company = Company(
|
|
id=uuid.uuid4(),
|
|
user_id=user_id,
|
|
name="Rate Limited Co",
|
|
slug=f"rate-limited-co-{uuid.uuid4().hex[:6]}",
|
|
)
|
|
db_session.add(company)
|
|
db_session.add(MonitorConfiguration(company_id=company.id))
|
|
await db_session.commit()
|
|
|
|
run_repo = MonitoringRunRepository(db_session)
|
|
for _ in range(2):
|
|
run = await run_repo.create(company_id=company.id, trigger_type=MonitoringRunTrigger.MANUAL)
|
|
await run_repo.mark_finished(run, status=MonitoringRunStatus.SUCCESSFUL, error_summary=None)
|
|
await db_session.commit()
|
|
|
|
capped_settings = settings.model_copy(update={"max_manual_runs_per_day": 2})
|
|
|
|
with pytest.raises(RateLimitedError):
|
|
await monitoring_service.enqueue_run_now(db_session, capped_settings, user_id, company.id)
|
|
|
|
|
|
def test_run_history_and_single_run_scoped_to_owner(client):
|
|
owner_headers = _register_and_login(client)
|
|
other_headers = _register_and_login(client)
|
|
company = _create_company(client, owner_headers)
|
|
|
|
with patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]):
|
|
with respx.mock:
|
|
_mock_empty_discovery()
|
|
run = client.post(
|
|
f"/api/v1/companies/{company['id']}/run", headers=owner_headers
|
|
).json()
|
|
|
|
assert (
|
|
client.get(f"/api/v1/companies/{company['id']}/runs", headers=other_headers).status_code
|
|
== 404
|
|
)
|
|
assert client.get(f"/api/v1/runs/{run['id']}", headers=other_headers).status_code == 404
|
|
assert client.get(f"/api/v1/runs/{run['id']}", headers=owner_headers).status_code == 200
|
|
|
|
|
|
def test_successful_first_run_generates_a_baseline_report(client):
|
|
headers = _register_and_login(client)
|
|
company = _create_company(client, headers)
|
|
client.post(
|
|
f"/api/v1/companies/{company['id']}/sources",
|
|
json={
|
|
"source_type": "custom_url",
|
|
"name": "Pricing",
|
|
"base_url": "https://example.com/pricing",
|
|
},
|
|
headers=headers,
|
|
)
|
|
|
|
with patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]):
|
|
with respx.mock:
|
|
_mock_empty_discovery()
|
|
respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404))
|
|
respx.get("https://example.com/pricing").mock(
|
|
return_value=httpx.Response(
|
|
200,
|
|
html="<html><head><title>Pricing</title></head><body>"
|
|
"<article><h1>Pricing</h1><p>Plans start at $10/month.</p></article>"
|
|
"</body></html>",
|
|
)
|
|
)
|
|
client.post(f"/api/v1/companies/{company['id']}/run", headers=headers)
|
|
|
|
reports = client.get(f"/api/v1/companies/{company['id']}/reports", headers=headers).json()
|
|
assert len(reports) == 1
|
|
assert reports[0]["report_type"] == "baseline"
|
|
assert reports[0]["model_provider"] == "mock"
|
|
|
|
|
|
def test_manual_run_does_not_disrupt_next_scheduled_run(client):
|
|
headers = _register_and_login(client)
|
|
company = _create_company(client, headers)
|
|
original_next_run = company["monitor_configuration"]["next_run"]
|
|
|
|
with patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]):
|
|
with respx.mock:
|
|
_mock_empty_discovery()
|
|
client.post(f"/api/v1/companies/{company['id']}/run", headers=headers)
|
|
|
|
refreshed = client.get(f"/api/v1/companies/{company['id']}", headers=headers).json()
|
|
assert refreshed["monitor_configuration"]["next_run"] == original_next_run
|
|
assert refreshed["monitor_configuration"]["last_run"] is not None
|