Initial commit: CI Agent competitive-intelligence monitoring app
FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification).
This commit is contained in:
@@ -0,0 +1,160 @@
|
||||
"""MockLLMProvider: every analysis task's response schema must come back
|
||||
valid and genuinely reflect the evidence passed in - never an empty
|
||||
placeholder unrelated to the input."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.analysis.llm.mock import MockLLMProvider
|
||||
from app.prompts.alert_summarization import AlertSummary
|
||||
from app.prompts.base import build_user_prompt
|
||||
from app.prompts.change_significance import ChangeSignificanceAssessment
|
||||
from app.prompts.extraction import ExtractionResult
|
||||
from app.prompts.relevance import RelevanceAssessment
|
||||
from app.prompts.report_generation import ReportContent
|
||||
from app.prompts.synthesis import SynthesisResult
|
||||
|
||||
provider = MockLLMProvider()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_relevance_matches_focus_keyword():
|
||||
prompt = build_user_prompt(
|
||||
"assess",
|
||||
{
|
||||
"monitoring_focus": "electric vehicle manufacturing expansion",
|
||||
"document_text": "The company announced a new manufacturing facility for electric vehicles.",
|
||||
},
|
||||
)
|
||||
result = await provider.generate_structured("system", prompt, RelevanceAssessment)
|
||||
assert result.is_relevant is True
|
||||
assert result.matches_focus is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extraction_pulls_first_sentence_as_signal():
|
||||
prompt = build_user_prompt(
|
||||
"extract", {"document_text": "Acme Corp opened a new facility. It will employ 200 people."}
|
||||
)
|
||||
result = await provider.generate_structured("system", prompt, ExtractionResult)
|
||||
assert len(result.signals) == 1
|
||||
assert "Acme Corp opened a new facility" in result.signals[0].description
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_synthesis_requires_multiple_signals():
|
||||
prompt_one = build_user_prompt("synthesize", {"signals": [{"description": "a"}]})
|
||||
result_one = await provider.generate_structured("system", prompt_one, SynthesisResult)
|
||||
assert result_one.conclusions == []
|
||||
|
||||
prompt_two = build_user_prompt(
|
||||
"synthesize", {"signals": [{"description": "a"}, {"description": "b"}]}
|
||||
)
|
||||
result_two = await provider.generate_structured("system", prompt_two, SynthesisResult)
|
||||
assert len(result_two.conclusions) == 1
|
||||
assert result_two.conclusions[0].source_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_report_reflects_evidence_counts():
|
||||
prompt = build_user_prompt(
|
||||
"report",
|
||||
{
|
||||
"company_profile": {"name": "Acme Corp"},
|
||||
"source_documents": [
|
||||
{
|
||||
"id": "d1",
|
||||
"title": "Job posting",
|
||||
"url": "https://x.com/1",
|
||||
"source_type": "job_posting",
|
||||
"retrieved_date": "2026-01-01",
|
||||
}
|
||||
],
|
||||
"detected_changes": [
|
||||
{
|
||||
"id": "c1",
|
||||
"summary": "New job posting detected",
|
||||
"change_type": "new_document",
|
||||
"severity": "medium",
|
||||
"confidence_score": 0.6,
|
||||
"created_at": "2026-01-01",
|
||||
}
|
||||
],
|
||||
"sources_that_failed_to_collect": ["Broken Source"],
|
||||
},
|
||||
)
|
||||
result = await provider.generate_structured("system", prompt, ReportContent)
|
||||
assert "Acme Corp" in result.executive_summary
|
||||
assert "1" in result.executive_summary # document/change counts mentioned
|
||||
assert len(result.recent_developments) == 1
|
||||
assert len(result.hiring_signals) == 1
|
||||
assert any("Broken Source" in u for u in result.unknowns_and_missing_data)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_report_grounds_overview_in_discovered_profile_even_with_no_documents():
|
||||
"""The bug this covers: a report generated before any monitoring run had
|
||||
collected evidence used to come back near-empty even though the company's
|
||||
discovered profile (from onboarding) had real data. That profile data
|
||||
must now ground company_overview/market_positioning."""
|
||||
prompt = build_user_prompt(
|
||||
"report",
|
||||
{
|
||||
"company_profile": {
|
||||
"name": "Stripe",
|
||||
"description": "Stripe builds economic infrastructure for the internet.",
|
||||
"industry": "Financial infrastructure",
|
||||
"headquarters": "South San Francisco, California",
|
||||
"competitors": ["PayPal"],
|
||||
"aliases": [],
|
||||
},
|
||||
"source_documents": [],
|
||||
"detected_changes": [],
|
||||
"sources_that_failed_to_collect": [],
|
||||
},
|
||||
)
|
||||
result = await provider.generate_structured("system", prompt, ReportContent)
|
||||
assert "economic infrastructure" in result.company_overview
|
||||
assert "South San Francisco" in result.company_overview
|
||||
assert "PayPal" in result.market_positioning
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_change_significance_reflects_deterministic_severity():
|
||||
prompt = build_user_prompt(
|
||||
"assess",
|
||||
{
|
||||
"change_type": "leadership_change",
|
||||
"deterministic_severity": "high",
|
||||
"deterministic_confidence": 0.8,
|
||||
},
|
||||
)
|
||||
result = await provider.generate_structured("system", prompt, ChangeSignificanceAssessment)
|
||||
assert result.is_meaningful is True
|
||||
assert result.should_notify is True
|
||||
assert "high" in result.why_it_matters
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alert_summary_title_under_100_chars():
|
||||
prompt = build_user_prompt(
|
||||
"summarize",
|
||||
{
|
||||
"company_name": "Acme Corp",
|
||||
"change_type": "price_change",
|
||||
"severity": "medium",
|
||||
"confidence": 0.6,
|
||||
},
|
||||
)
|
||||
result = await provider.generate_structured("system", prompt, AlertSummary)
|
||||
assert len(result.title) <= 100
|
||||
assert "Acme Corp" in result.title
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_text_does_not_crash():
|
||||
prompt = build_user_prompt("do something", {"a": 1})
|
||||
text = await provider.generate_text("system", prompt)
|
||||
assert isinstance(text, str)
|
||||
assert len(text) > 0
|
||||
Reference in New Issue
Block a user