Initial commit: CI Agent competitive-intelligence monitoring app
FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification).
This commit is contained in:
@@ -0,0 +1,24 @@
|
||||
"""Collector tests never touch the real network or real DNS - every
|
||||
hostname resolves to a fixed public IP, and respx intercepts the actual
|
||||
HTTP layer per test."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.collectors import robots as robots_module
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_real_dns():
|
||||
with patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_robots_cache():
|
||||
robots_module.clear_cache()
|
||||
yield
|
||||
robots_module.clear_cache()
|
||||
@@ -0,0 +1,72 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from app.collectors.base import CompanyContext, SourceConfig
|
||||
from app.collectors.custom_url import CustomUrlCollector
|
||||
from app.models.enums import SourceStatus, SourceType
|
||||
|
||||
COMPANY = CompanyContext(id="c1", name="Acme Corp", official_website=None, monitoring_focus=None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_extracts_single_page():
|
||||
source = SourceConfig(
|
||||
id="s1",
|
||||
source_type=SourceType.CUSTOM_URL,
|
||||
name="Pricing page",
|
||||
base_url="https://example.com/pricing",
|
||||
)
|
||||
with respx.mock:
|
||||
respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404))
|
||||
respx.get("https://example.com/pricing").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
html=(
|
||||
"<html><head><title>Pricing</title></head><body>"
|
||||
"<article><h1>Pricing</h1><p>Plans start at $99/month for the base tier.</p>"
|
||||
"</article></body></html>"
|
||||
),
|
||||
)
|
||||
)
|
||||
result = await CustomUrlCollector().collect(source, COMPANY)
|
||||
|
||||
assert result.status == SourceStatus.ACTIVE
|
||||
assert len(result.documents) == 1
|
||||
assert "$99/month" in result.documents[0].content_text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_reports_auth_required_on_403():
|
||||
source = SourceConfig(
|
||||
id="s1",
|
||||
source_type=SourceType.CUSTOM_URL,
|
||||
name="Gated page",
|
||||
base_url="https://example.com/gated",
|
||||
)
|
||||
with respx.mock:
|
||||
respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404))
|
||||
respx.get("https://example.com/gated").mock(return_value=httpx.Response(403))
|
||||
result = await CustomUrlCollector().collect(source, COMPANY)
|
||||
|
||||
assert result.status == SourceStatus.AUTH_REQUIRED
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_respects_robots_disallow():
|
||||
source = SourceConfig(
|
||||
id="s1",
|
||||
source_type=SourceType.CUSTOM_URL,
|
||||
name="Disallowed page",
|
||||
base_url="https://example.com/private",
|
||||
)
|
||||
with respx.mock:
|
||||
respx.get("https://example.com/robots.txt").mock(
|
||||
return_value=httpx.Response(200, text="User-agent: *\nDisallow: /private\n")
|
||||
)
|
||||
result = await CustomUrlCollector().collect(source, COMPANY)
|
||||
|
||||
assert result.status == SourceStatus.BLOCKED_BY_POLICY
|
||||
assert result.documents == []
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Patent/review collectors: documented fixture adapters that must never
|
||||
fabricate data when no provider or fixture is configured."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.collectors.base import CompanyContext, SourceConfig
|
||||
from app.collectors.patents import PatentSourceCollector
|
||||
from app.collectors.reviews import ReviewSourceCollector
|
||||
from app.models.enums import SourceStatus, SourceType
|
||||
|
||||
COMPANY = CompanyContext(
|
||||
id="c1", name="Acme Mobility Systems", official_website=None, monitoring_focus=None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_patent_collector_disabled_without_fixture_key():
|
||||
source = SourceConfig(id="s1", source_type=SourceType.PATENT, name="Patents", base_url=None)
|
||||
result = await PatentSourceCollector().collect(source, COMPANY)
|
||||
assert result.status == SourceStatus.DISABLED
|
||||
assert result.documents == []
|
||||
assert "does not fabricate" in result.error
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_patent_collector_disabled_for_unknown_fixture_key():
|
||||
source = SourceConfig(
|
||||
id="s1",
|
||||
source_type=SourceType.PATENT,
|
||||
name="Patents",
|
||||
base_url=None,
|
||||
configuration_metadata={"fixture_key": "does_not_exist"},
|
||||
)
|
||||
result = await PatentSourceCollector().collect(source, COMPANY)
|
||||
assert result.status == SourceStatus.DISABLED
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_patent_collector_loads_fixture_when_configured():
|
||||
source = SourceConfig(
|
||||
id="s1",
|
||||
source_type=SourceType.PATENT,
|
||||
name="Patents",
|
||||
base_url=None,
|
||||
configuration_metadata={"fixture_key": "acme_mobility"},
|
||||
)
|
||||
result = await PatentSourceCollector().collect(source, COMPANY)
|
||||
assert result.status == SourceStatus.ACTIVE
|
||||
assert len(result.documents) == 1
|
||||
assert result.documents[0].metadata["is_fixture"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_review_collector_disabled_without_fixture_key():
|
||||
source = SourceConfig(id="s1", source_type=SourceType.REVIEW, name="Reviews", base_url=None)
|
||||
result = await ReviewSourceCollector().collect(source, COMPANY)
|
||||
assert result.status == SourceStatus.DISABLED
|
||||
assert result.documents == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_review_collector_loads_fixture_when_configured():
|
||||
source = SourceConfig(
|
||||
id="s1",
|
||||
source_type=SourceType.REVIEW,
|
||||
name="Reviews",
|
||||
base_url=None,
|
||||
configuration_metadata={"fixture_key": "acme_mobility"},
|
||||
)
|
||||
result = await ReviewSourceCollector().collect(source, COMPANY)
|
||||
assert result.status == SourceStatus.ACTIVE
|
||||
assert len(result.documents) == 2
|
||||
assert all(d.metadata["is_fixture"] for d in result.documents)
|
||||
@@ -0,0 +1,73 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from app.collectors.base import CompanyContext, SourceConfig
|
||||
from app.collectors.github import GithubCollector
|
||||
from app.models.enums import SourceStatus, SourceType
|
||||
|
||||
COMPANY = CompanyContext(id="c1", name="Acme Corp", official_website=None, monitoring_focus=None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_finds_org_login():
|
||||
with respx.mock:
|
||||
respx.get("https://api.github.com/search/users").mock(
|
||||
return_value=httpx.Response(200, text=json.dumps({"items": [{"login": "acme-corp"}]}))
|
||||
)
|
||||
discovered = await GithubCollector().discover(COMPANY)
|
||||
|
||||
assert discovered[0].configuration_metadata["org"] == "acme-corp"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_returns_repo_documents():
|
||||
source = SourceConfig(
|
||||
id="s1",
|
||||
source_type=SourceType.GITHUB,
|
||||
name="Acme GitHub",
|
||||
base_url="https://github.com/acme-corp",
|
||||
configuration_metadata={"org": "acme-corp"},
|
||||
)
|
||||
repos = [
|
||||
{
|
||||
"full_name": "acme-corp/battery-sdk",
|
||||
"description": "SDK for battery management systems",
|
||||
"language": "Python",
|
||||
"stargazers_count": 42,
|
||||
"pushed_at": "2026-05-01T12:00:00Z",
|
||||
"html_url": "https://github.com/acme-corp/battery-sdk",
|
||||
"archived": False,
|
||||
}
|
||||
]
|
||||
with respx.mock:
|
||||
respx.get("https://api.github.com/orgs/acme-corp/repos").mock(
|
||||
return_value=httpx.Response(200, text=json.dumps(repos))
|
||||
)
|
||||
result = await GithubCollector().collect(source, COMPANY)
|
||||
|
||||
assert result.status == SourceStatus.ACTIVE
|
||||
assert len(result.documents) == 1
|
||||
assert "battery management" in result.documents[0].content_text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_reports_rate_limited_on_403():
|
||||
source = SourceConfig(
|
||||
id="s1",
|
||||
source_type=SourceType.GITHUB,
|
||||
name="Acme GitHub",
|
||||
base_url=None,
|
||||
configuration_metadata={"org": "acme-corp"},
|
||||
)
|
||||
with respx.mock:
|
||||
respx.get("https://api.github.com/orgs/acme-corp/repos").mock(
|
||||
return_value=httpx.Response(403)
|
||||
)
|
||||
result = await GithubCollector().collect(source, COMPANY)
|
||||
|
||||
assert result.status == SourceStatus.RATE_LIMITED
|
||||
@@ -0,0 +1,94 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from app.collectors.base import CompanyContext, SourceConfig
|
||||
from app.collectors.gov_contracts import GovContractCollector
|
||||
from app.models.enums import SourceStatus, SourceType
|
||||
|
||||
COMPANY = CompanyContext(id="c1", name="Acme Corp", official_website=None, monitoring_focus=None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_always_returns_a_source_without_a_network_call():
|
||||
with respx.mock:
|
||||
discovered = await GovContractCollector().discover(COMPANY)
|
||||
|
||||
assert len(discovered) == 1
|
||||
assert discovered[0].source_type == SourceType.GOV_CONTRACT
|
||||
assert discovered[0].configuration_metadata["recipient_search_text"] == "Acme Corp"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_maps_awards_to_documents():
|
||||
source = SourceConfig(
|
||||
id="s1",
|
||||
source_type=SourceType.GOV_CONTRACT,
|
||||
name="Acme Federal Contracts",
|
||||
base_url=None,
|
||||
configuration_metadata={"recipient_search_text": "Acme Corp"},
|
||||
)
|
||||
payload = {
|
||||
"results": [
|
||||
{
|
||||
"Award ID": "W91CRB-26-C-0001",
|
||||
"Recipient Name": "Acme Corp",
|
||||
"Award Amount": 4_500_000,
|
||||
"Start Date": "2026-02-01",
|
||||
"Awarding Agency": "Department of Defense",
|
||||
"Description": "Supply of tactical equipment.",
|
||||
}
|
||||
]
|
||||
}
|
||||
with respx.mock:
|
||||
respx.post("https://api.usaspending.gov/api/v2/search/spending_by_award/").mock(
|
||||
return_value=httpx.Response(200, json=payload)
|
||||
)
|
||||
result = await GovContractCollector().collect(source, COMPANY)
|
||||
|
||||
assert result.status == SourceStatus.ACTIVE
|
||||
assert len(result.documents) == 1
|
||||
doc = result.documents[0]
|
||||
assert doc.metadata["award_id"] == "W91CRB-26-C-0001"
|
||||
assert "Department of Defense" in doc.title
|
||||
assert "$4,500,000" in doc.title
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_zero_awards_is_not_a_failure():
|
||||
source = SourceConfig(
|
||||
id="s1",
|
||||
source_type=SourceType.GOV_CONTRACT,
|
||||
name="Acme Federal Contracts",
|
||||
base_url=None,
|
||||
configuration_metadata={"recipient_search_text": "Acme Corp"},
|
||||
)
|
||||
with respx.mock:
|
||||
respx.post("https://api.usaspending.gov/api/v2/search/spending_by_award/").mock(
|
||||
return_value=httpx.Response(200, json={"results": []})
|
||||
)
|
||||
result = await GovContractCollector().collect(source, COMPANY)
|
||||
|
||||
assert result.status == SourceStatus.ACTIVE
|
||||
assert result.documents == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_fails_gracefully_on_http_error():
|
||||
source = SourceConfig(
|
||||
id="s1",
|
||||
source_type=SourceType.GOV_CONTRACT,
|
||||
name="Acme Federal Contracts",
|
||||
base_url=None,
|
||||
configuration_metadata={"recipient_search_text": "Acme Corp"},
|
||||
)
|
||||
with respx.mock:
|
||||
respx.post("https://api.usaspending.gov/api/v2/search/spending_by_award/").mock(
|
||||
return_value=httpx.Response(500, text="internal error")
|
||||
)
|
||||
result = await GovContractCollector().collect(source, COMPANY)
|
||||
|
||||
assert result.status == SourceStatus.FAILED
|
||||
assert "500" in result.error
|
||||
@@ -0,0 +1,73 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from app.collectors.base import CompanyContext, SourceConfig
|
||||
from app.collectors.jobs import JobPostingCollector
|
||||
from app.models.enums import SourceStatus, SourceType
|
||||
|
||||
CAREERS_HTML = """
|
||||
<html><body>
|
||||
<h1>Careers at Acme</h1>
|
||||
<ul>
|
||||
<li><a href="/careers/job/battery-engineer">Senior Battery Engineer</a></li>
|
||||
<li><a href="/careers/job/battery-technician">Battery Systems Technician</a></li>
|
||||
<li><a href="/about">About us</a></li>
|
||||
</ul>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
COMPANY = CompanyContext(
|
||||
id="c1", name="Acme Corp", official_website="https://example.com", monitoring_focus=None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_targets_careers_page():
|
||||
discovered = await JobPostingCollector().discover(COMPANY)
|
||||
assert discovered[0].base_url == "https://example.com/careers"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_extracts_job_listings_and_ignores_unrelated_links():
|
||||
source = SourceConfig(
|
||||
id="s1",
|
||||
source_type=SourceType.JOB_POSTING,
|
||||
name="Acme Careers",
|
||||
base_url="https://example.com/careers",
|
||||
)
|
||||
with respx.mock:
|
||||
respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404))
|
||||
respx.get("https://example.com/careers").mock(
|
||||
return_value=httpx.Response(200, html=CAREERS_HTML)
|
||||
)
|
||||
result = await JobPostingCollector().collect(source, COMPANY)
|
||||
|
||||
assert result.status == SourceStatus.ACTIVE
|
||||
titles = {d.title for d in result.documents}
|
||||
assert titles == {"Senior Battery Engineer", "Battery Systems Technician"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_falls_back_to_whole_page_when_no_job_links_found():
|
||||
source = SourceConfig(
|
||||
id="s1",
|
||||
source_type=SourceType.JOB_POSTING,
|
||||
name="Acme Careers",
|
||||
base_url="https://example.com/careers",
|
||||
)
|
||||
with respx.mock:
|
||||
respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404))
|
||||
respx.get("https://example.com/careers").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
html="<html><body><article><h1>Careers</h1><p>We use an external ATS widget.</p></article></body></html>",
|
||||
)
|
||||
)
|
||||
result = await JobPostingCollector().collect(source, COMPANY)
|
||||
|
||||
assert result.status == SourceStatus.ACTIVE
|
||||
assert len(result.documents) == 1
|
||||
assert result.documents[0].metadata["extraction"] == "fallback_whole_page"
|
||||
@@ -0,0 +1,160 @@
|
||||
"""PatentSourceCollector's live USPTO Open Data Portal branch - only
|
||||
active when `Settings.uspto_api_key` is configured. The no-key disabled/
|
||||
fixture path is covered separately in test_fixture_collectors.py and is
|
||||
asserted here to stay unchanged.
|
||||
|
||||
Live-verified (2026-08): USPTO's Patent Application Search has no
|
||||
queryable assignee/company field at all, so `collect()` searches by each
|
||||
of the company's known leadership names (from NinjaPear enrichment)
|
||||
instead - see the module docstring in app/collectors/patents.py.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from app.collectors.base import CompanyContext, SourceConfig
|
||||
from app.collectors.patents import PatentSourceCollector
|
||||
from app.core.config import Settings
|
||||
from app.models.enums import SourceStatus, SourceType
|
||||
|
||||
COMPANY = CompanyContext(
|
||||
id="c1", name="Acme Mobility", official_website=None, monitoring_focus=None
|
||||
)
|
||||
COMPANY_WITH_LEADERSHIP = CompanyContext(
|
||||
id="c1",
|
||||
name="Acme Mobility",
|
||||
official_website=None,
|
||||
monitoring_focus=None,
|
||||
leadership_names=["Jane Doe", "John Smith"],
|
||||
)
|
||||
|
||||
|
||||
def _with_key(monkeypatch, key: str = "test-uspto-key") -> None:
|
||||
monkeypatch.setattr("app.collectors.patents.get_settings", lambda: Settings(uspto_api_key=key))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_returns_empty_without_a_key():
|
||||
discovered = await PatentSourceCollector().discover(COMPANY)
|
||||
assert discovered == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_returns_a_source_when_a_key_is_configured(monkeypatch):
|
||||
_with_key(monkeypatch)
|
||||
discovered = await PatentSourceCollector().discover(COMPANY)
|
||||
|
||||
assert len(discovered) == 1
|
||||
assert discovered[0].source_type == SourceType.PATENT
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_with_no_leadership_names_is_an_empty_result_with_no_network_call(
|
||||
monkeypatch,
|
||||
):
|
||||
"""No names to search USPTO's inventor index with (enrichment never
|
||||
ran, still pending, or found no leadership) - an honest empty result,
|
||||
not a wasted/failed call."""
|
||||
_with_key(monkeypatch)
|
||||
source = SourceConfig(
|
||||
id="s1", source_type=SourceType.PATENT, name="Acme Patents", base_url=None
|
||||
)
|
||||
with respx.mock:
|
||||
route = respx.post("https://api.uspto.gov/api/v1/patent/applications/search")
|
||||
result = await PatentSourceCollector().collect(source, COMPANY)
|
||||
|
||||
assert route.call_count == 0
|
||||
assert result.status == SourceStatus.ACTIVE
|
||||
assert result.documents == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_searches_by_each_leadership_name_and_dedupes(monkeypatch):
|
||||
_with_key(monkeypatch)
|
||||
source = SourceConfig(
|
||||
id="s1", source_type=SourceType.PATENT, name="Acme Patents", base_url=None
|
||||
)
|
||||
# title/filingDate/abstractText live under applicationMetaData, not at
|
||||
# the entry's top level - matches the real API's confirmed live shape.
|
||||
shared_patent = {
|
||||
"applicationNumberText": "18/123456",
|
||||
"applicationMetaData": {
|
||||
"inventionTitle": "Battery Swapping System for Electric Vehicles",
|
||||
"filingDate": "2026-01-15",
|
||||
"abstractText": "A system for rapid battery exchange in electric scooters.",
|
||||
},
|
||||
}
|
||||
|
||||
with respx.mock:
|
||||
route = respx.post("https://api.uspto.gov/api/v1/patent/applications/search").mock(
|
||||
return_value=httpx.Response(200, json={"patentFileWrapperDataBag": [shared_patent]})
|
||||
)
|
||||
result = await PatentSourceCollector().collect(source, COMPANY_WITH_LEADERSHIP)
|
||||
|
||||
# Both names searched (2 requests)...
|
||||
assert route.call_count == 2
|
||||
assert route.calls[0].request.headers["x-api-key"] == "test-uspto-key"
|
||||
for call in route.calls:
|
||||
assert b"inventorNameText" in call.request.content
|
||||
# ...but the same application number returned by both searches is
|
||||
# only kept once.
|
||||
assert result.status == SourceStatus.ACTIVE
|
||||
assert len(result.documents) == 1
|
||||
doc = result.documents[0]
|
||||
assert doc.metadata["application_number"] == "18/123456"
|
||||
assert doc.metadata["match_type"] == "leadership_name_heuristic"
|
||||
assert doc.trust_score == 0.5
|
||||
assert "Battery Swapping" in doc.title
|
||||
assert "heuristic" in doc.content_text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_matching_records_for_a_name_is_not_a_failure(monkeypatch):
|
||||
"""USPTO returns 404 for "no matching records" rather than a 200 with
|
||||
an empty array - a real, expected outcome for most names, not a
|
||||
collection failure."""
|
||||
_with_key(monkeypatch)
|
||||
source = SourceConfig(
|
||||
id="s1", source_type=SourceType.PATENT, name="Acme Patents", base_url=None
|
||||
)
|
||||
with respx.mock:
|
||||
respx.post("https://api.uspto.gov/api/v1/patent/applications/search").mock(
|
||||
return_value=httpx.Response(404, json={"message": "No matching records found"})
|
||||
)
|
||||
result = await PatentSourceCollector().collect(source, COMPANY_WITH_LEADERSHIP)
|
||||
|
||||
assert result.status == SourceStatus.ACTIVE
|
||||
assert result.documents == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_surfaces_a_real_api_error_when_every_search_fails(monkeypatch):
|
||||
"""Once a key is configured, a real failure must be reported honestly -
|
||||
not silently masked as the "no provider configured" disabled state,
|
||||
which would look like the user's key was never even attempted."""
|
||||
_with_key(monkeypatch)
|
||||
source = SourceConfig(
|
||||
id="s1", source_type=SourceType.PATENT, name="Acme Patents", base_url=None
|
||||
)
|
||||
with respx.mock:
|
||||
respx.post("https://api.uspto.gov/api/v1/patent/applications/search").mock(
|
||||
return_value=httpx.Response(401, text="Invalid API key")
|
||||
)
|
||||
result = await PatentSourceCollector().collect(source, COMPANY_WITH_LEADERSHIP)
|
||||
|
||||
assert result.status == SourceStatus.FAILED
|
||||
assert "401" in result.error
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_without_a_key_is_unchanged_disabled_behavior():
|
||||
source = SourceConfig(
|
||||
id="s1", source_type=SourceType.PATENT, name="Acme Patents", base_url=None
|
||||
)
|
||||
result = await PatentSourceCollector().collect(source, COMPANY_WITH_LEADERSHIP)
|
||||
|
||||
assert result.status == SourceStatus.DISABLED
|
||||
assert "does not fabricate" in result.error
|
||||
@@ -0,0 +1,85 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from app.collectors.base import CompanyContext, SourceConfig
|
||||
from app.collectors.rss import RssCollector
|
||||
from app.models.enums import SourceStatus, SourceType
|
||||
|
||||
FEED_XML = """<?xml version="1.0"?>
|
||||
<rss version="2.0"><channel>
|
||||
<title>Acme News</title>
|
||||
<item>
|
||||
<title>Acme opens new facility</title>
|
||||
<link>https://example.com/news/1</link>
|
||||
<description>Acme Corp opened a new manufacturing facility this week.</description>
|
||||
<pubDate>Mon, 01 Jun 2026 10:00:00 GMT</pubDate>
|
||||
</item>
|
||||
<item>
|
||||
<title>Acme hires new VP</title>
|
||||
<link>https://example.com/news/2</link>
|
||||
<description>Acme Corp announced a new VP of Engineering.</description>
|
||||
</item>
|
||||
</channel></rss>
|
||||
"""
|
||||
|
||||
COMPANY = CompanyContext(id="c1", name="Acme Corp", official_website=None, monitoring_focus=None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_builds_a_google_news_search_url():
|
||||
discovered = await RssCollector().discover(COMPANY)
|
||||
|
||||
assert len(discovered) == 1
|
||||
assert discovered[0].source_type == SourceType.RSS
|
||||
assert discovered[0].base_url.startswith("https://news.google.com/rss/search?q=")
|
||||
assert "Acme%20Corp" in discovered[0].base_url
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_url_encodes_ampersands_in_the_name():
|
||||
company = CompanyContext(id="c1", name="AT&T", official_website=None, monitoring_focus=None)
|
||||
discovered = await RssCollector().discover(company)
|
||||
|
||||
# A raw "&" from the company name must not leak into the query string,
|
||||
# since it would be parsed as a new URL parameter separator (splitting
|
||||
# the query in two) rather than part of the search text.
|
||||
query_part = discovered[0].base_url.split("q=", 1)[1].split("&hl=", 1)[0]
|
||||
assert "&" not in query_part
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_parses_feed_items():
|
||||
source = SourceConfig(
|
||||
id="s1", source_type=SourceType.RSS, name="Acme RSS", base_url="https://example.com/feed"
|
||||
)
|
||||
with respx.mock:
|
||||
respx.get("https://example.com/feed").mock(return_value=httpx.Response(200, text=FEED_XML))
|
||||
result = await RssCollector().collect(source, COMPANY)
|
||||
|
||||
assert result.status == SourceStatus.ACTIVE
|
||||
assert len(result.documents) == 2
|
||||
assert result.documents[0].title == "Acme opens new facility"
|
||||
assert result.documents[0].publication_date is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_fails_gracefully_on_http_error():
|
||||
source = SourceConfig(
|
||||
id="s1", source_type=SourceType.RSS, name="Acme RSS", base_url="https://example.com/feed"
|
||||
)
|
||||
with respx.mock:
|
||||
respx.get("https://example.com/feed").mock(return_value=httpx.Response(500))
|
||||
result = await RssCollector().collect(source, COMPANY)
|
||||
|
||||
assert result.status == SourceStatus.FAILED
|
||||
assert result.documents == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_without_base_url_fails_without_network_call():
|
||||
source = SourceConfig(id="s1", source_type=SourceType.RSS, name="Acme RSS", base_url=None)
|
||||
result = await RssCollector().collect(source, COMPANY)
|
||||
assert result.status == SourceStatus.FAILED
|
||||
@@ -0,0 +1,91 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from app.collectors.base import CompanyContext, SourceConfig
|
||||
from app.collectors.sec_edgar import SecEdgarCollector
|
||||
from app.models.enums import SourceStatus, SourceType
|
||||
|
||||
ATOM_RESPONSE = """<?xml version="1.0"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
<entry>
|
||||
<title>ACME CORP</title>
|
||||
<content type="text">CIK=0000320193&ScoreKeyed=true</content>
|
||||
</entry>
|
||||
</feed>
|
||||
"""
|
||||
|
||||
COMPANY = CompanyContext(id="c1", name="Acme Corp", official_website=None, monitoring_focus=None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_finds_cik_from_atom_search():
|
||||
with respx.mock:
|
||||
respx.get("https://www.sec.gov/cgi-bin/browse-edgar").mock(
|
||||
return_value=httpx.Response(200, text=ATOM_RESPONSE)
|
||||
)
|
||||
discovered = await SecEdgarCollector().discover(COMPANY)
|
||||
|
||||
assert len(discovered) == 1
|
||||
assert discovered[0].configuration_metadata["cik"] == "0000320193"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_returns_empty_when_no_match():
|
||||
with respx.mock:
|
||||
respx.get("https://www.sec.gov/cgi-bin/browse-edgar").mock(
|
||||
return_value=httpx.Response(
|
||||
200, text='<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom"></feed>'
|
||||
)
|
||||
)
|
||||
discovered = await SecEdgarCollector().discover(COMPANY)
|
||||
|
||||
assert discovered == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_extracts_relevant_filings():
|
||||
source = SourceConfig(
|
||||
id="s1",
|
||||
source_type=SourceType.SEC_EDGAR,
|
||||
name="Acme EDGAR",
|
||||
base_url=None,
|
||||
configuration_metadata={"cik": "0000320193"},
|
||||
)
|
||||
submissions = {
|
||||
"name": "Acme Corp",
|
||||
"filings": {
|
||||
"recent": {
|
||||
"form": ["10-K", "S-1", "8-K"],
|
||||
"filingDate": ["2026-02-01", "2026-01-15", "2026-03-01"],
|
||||
"accessionNumber": [
|
||||
"0000320193-26-000001",
|
||||
"0000320193-26-000002",
|
||||
"0000320193-26-000003",
|
||||
],
|
||||
"primaryDocument": ["10k.htm", "s1.htm", "8k.htm"],
|
||||
}
|
||||
},
|
||||
}
|
||||
with respx.mock:
|
||||
respx.get("https://data.sec.gov/submissions/CIK0000320193.json").mock(
|
||||
return_value=httpx.Response(200, text=json.dumps(submissions))
|
||||
)
|
||||
result = await SecEdgarCollector().collect(source, COMPANY)
|
||||
|
||||
assert result.status == SourceStatus.ACTIVE
|
||||
forms = {d.metadata["form"] for d in result.documents}
|
||||
assert forms == {"10-K", "8-K"} # S-1 filtered out - not in _RELEVANT_FORMS
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_without_cik_fails_without_network_call():
|
||||
source = SourceConfig(
|
||||
id="s1", source_type=SourceType.SEC_EDGAR, name="Acme EDGAR", base_url=None
|
||||
)
|
||||
result = await SecEdgarCollector().collect(source, COMPANY)
|
||||
assert result.status == SourceStatus.FAILED
|
||||
@@ -0,0 +1,139 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from app.collectors.base import CompanyContext, SourceConfig
|
||||
from app.collectors.website import WebsiteCollector
|
||||
from app.models.enums import SourceStatus, SourceType
|
||||
|
||||
ARTICLE_HTML = """
|
||||
<html><body>
|
||||
<nav>Home | About</nav>
|
||||
<article>
|
||||
<h1>{title}</h1>
|
||||
<p>{body}</p>
|
||||
</article>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
COMPANY = CompanyContext(
|
||||
id="c1", name="Acme Corp", official_website="https://example.com", monitoring_focus=None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_uses_sitemap_when_available():
|
||||
with respx.mock:
|
||||
respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404))
|
||||
respx.get("https://example.com/sitemap.xml").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
content=(
|
||||
'<?xml version="1.0"?>'
|
||||
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
|
||||
"<url><loc>https://example.com/about</loc></url>"
|
||||
"<url><loc>https://example.com/products</loc></url>"
|
||||
"</urlset>"
|
||||
),
|
||||
)
|
||||
)
|
||||
collector = WebsiteCollector()
|
||||
discovered = await collector.discover(COMPANY)
|
||||
|
||||
assert len(discovered) == 1
|
||||
pages = discovered[0].configuration_metadata["pages"]
|
||||
assert "https://example.com/about" in pages
|
||||
assert "https://example.com/products" in pages
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_falls_back_to_heuristic_paths_without_sitemap():
|
||||
with respx.mock:
|
||||
respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404))
|
||||
respx.get("https://example.com/sitemap.xml").mock(return_value=httpx.Response(404))
|
||||
collector = WebsiteCollector()
|
||||
discovered = await collector.discover(COMPANY)
|
||||
|
||||
pages = discovered[0].configuration_metadata["pages"]
|
||||
assert any(p.endswith("/about") for p in pages)
|
||||
assert any(p.endswith("/careers") for p in pages)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_extracts_documents_from_configured_pages():
|
||||
source = SourceConfig(
|
||||
id="s1",
|
||||
source_type=SourceType.WEBSITE,
|
||||
name="Acme website",
|
||||
base_url="https://example.com",
|
||||
configuration_metadata={
|
||||
"pages": ["https://example.com/about", "https://example.com/products"]
|
||||
},
|
||||
)
|
||||
with respx.mock:
|
||||
respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404))
|
||||
respx.get("https://example.com/about").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
html=ARTICLE_HTML.format(
|
||||
title="About Acme", body="Acme Corp builds electric utility vehicles."
|
||||
),
|
||||
)
|
||||
)
|
||||
respx.get("https://example.com/products").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
html=ARTICLE_HTML.format(
|
||||
title="Products", body="Our product line includes trucks and vans."
|
||||
),
|
||||
)
|
||||
)
|
||||
collector = WebsiteCollector()
|
||||
result = await collector.collect(source, COMPANY)
|
||||
|
||||
assert result.status == SourceStatus.ACTIVE
|
||||
assert len(result.documents) == 2
|
||||
assert any("electric utility vehicles" in d.content_text for d in result.documents)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_skips_pages_disallowed_by_robots_txt():
|
||||
source = SourceConfig(
|
||||
id="s1",
|
||||
source_type=SourceType.WEBSITE,
|
||||
name="Acme website",
|
||||
base_url="https://example.com",
|
||||
configuration_metadata={"pages": ["https://example.com/private"]},
|
||||
)
|
||||
with respx.mock:
|
||||
respx.get("https://example.com/robots.txt").mock(
|
||||
return_value=httpx.Response(200, text="User-agent: *\nDisallow: /private\n")
|
||||
)
|
||||
collector = WebsiteCollector()
|
||||
result = await collector.collect(source, COMPANY)
|
||||
|
||||
assert result.documents == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_deduplicates_identical_content_across_pages():
|
||||
source = SourceConfig(
|
||||
id="s1",
|
||||
source_type=SourceType.WEBSITE,
|
||||
name="Acme website",
|
||||
base_url="https://example.com",
|
||||
configuration_metadata={"pages": ["https://example.com/a", "https://example.com/a-mirror"]},
|
||||
)
|
||||
same_html = ARTICLE_HTML.format(title="Same", body="Identical content on both URLs.")
|
||||
with respx.mock:
|
||||
respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404))
|
||||
respx.get("https://example.com/a").mock(return_value=httpx.Response(200, html=same_html))
|
||||
respx.get("https://example.com/a-mirror").mock(
|
||||
return_value=httpx.Response(200, html=same_html)
|
||||
)
|
||||
collector = WebsiteCollector()
|
||||
result = await collector.collect(source, COMPANY)
|
||||
|
||||
assert len(result.documents) == 1
|
||||
Reference in New Issue
Block a user