FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification).
189 lines
6.3 KiB
Python
189 lines
6.3 KiB
Python
"""Generates and persists a company's CI report from accumulated evidence.
|
|
|
|
Evidence gathering happens here, not inside the LLM prompt module - the
|
|
model only ever sees data this pipeline actually collected (recent
|
|
SourceDocuments + DetectedChanges), so it cannot introduce facts we never
|
|
stored. See app/prompts/report_generation.py for the schema/prompt itself.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import uuid
|
|
from datetime import UTC, datetime
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.analysis.llm.base import LLMProvider
|
|
from app.core.config import Settings
|
|
from app.core.errors import NotFoundError
|
|
from app.models.company import Company
|
|
from app.models.detected_change import DetectedChange
|
|
from app.models.enums import EnrichmentStatus, ReportType, SourceStatus
|
|
from app.models.report import Report
|
|
from app.models.source import Source
|
|
from app.models.source_document import SourceDocument
|
|
from app.prompts.report_generation import generate_report
|
|
from app.repositories.company_enrichment_repository import CompanyEnrichmentRepository
|
|
from app.repositories.report_repository import ReportRepository
|
|
from app.services import company_service
|
|
from app.services.report_markdown import render_report_markdown
|
|
|
|
_MAX_DOCUMENTS = 40
|
|
_MAX_CHANGES = 20
|
|
|
|
|
|
async def _gather_documents(db: AsyncSession, company_id: uuid.UUID) -> list[dict]:
|
|
result = await db.execute(
|
|
select(SourceDocument, Source.source_type)
|
|
.join(Source, Source.id == SourceDocument.source_id)
|
|
.where(SourceDocument.company_id == company_id)
|
|
.order_by(SourceDocument.retrieved_date.desc())
|
|
.limit(_MAX_DOCUMENTS)
|
|
)
|
|
return [
|
|
{
|
|
"id": str(doc.id),
|
|
"title": doc.title,
|
|
"url": doc.url,
|
|
"excerpt": doc.content_text[:500],
|
|
"source_type": source_type.value,
|
|
"retrieved_date": doc.retrieved_date.isoformat(),
|
|
}
|
|
for doc, source_type in result.all()
|
|
]
|
|
|
|
|
|
async def _gather_changes(db: AsyncSession, company_id: uuid.UUID) -> list[dict]:
|
|
result = await db.execute(
|
|
select(DetectedChange)
|
|
.where(DetectedChange.company_id == company_id)
|
|
.order_by(DetectedChange.created_at.desc())
|
|
.limit(_MAX_CHANGES)
|
|
)
|
|
return [
|
|
{
|
|
"id": str(c.id),
|
|
"summary": c.summary,
|
|
"change_type": c.change_type.value,
|
|
"severity": c.severity.value,
|
|
"confidence_score": c.confidence_score,
|
|
"created_at": c.created_at.isoformat(),
|
|
}
|
|
for c in result.scalars().all()
|
|
]
|
|
|
|
|
|
async def _gather_failed_source_names(db: AsyncSession, company_id: uuid.UUID) -> list[str]:
|
|
result = await db.execute(
|
|
select(Source.name).where(
|
|
Source.company_id == company_id, Source.status != SourceStatus.ACTIVE
|
|
)
|
|
)
|
|
return list(result.scalars().all())
|
|
|
|
|
|
async def _gather_enrichment(db: AsyncSession, company_id: uuid.UUID) -> dict | None:
|
|
enrichment = await CompanyEnrichmentRepository(db).get_for_company(company_id)
|
|
if enrichment is None or enrichment.status == EnrichmentStatus.FAILED:
|
|
return None
|
|
return enrichment.data
|
|
|
|
|
|
def _model_name(settings: Settings, llm: LLMProvider) -> str:
|
|
if llm.provider_name == "anthropic":
|
|
return settings.anthropic_model
|
|
if llm.provider_name == "ollama":
|
|
return settings.ollama_model
|
|
return "mock"
|
|
|
|
|
|
async def generate_and_persist_report(
|
|
db: AsyncSession,
|
|
settings: Settings,
|
|
llm: LLMProvider,
|
|
company: Company,
|
|
*,
|
|
report_type: ReportType,
|
|
monitoring_run_id: uuid.UUID | None = None,
|
|
) -> Report:
|
|
documents = await _gather_documents(db, company.id)
|
|
changes = await _gather_changes(db, company.id)
|
|
failed_sources = await _gather_failed_source_names(db, company.id)
|
|
enrichment = await _gather_enrichment(db, company.id)
|
|
|
|
content = await generate_report(
|
|
llm,
|
|
company_name=company.name,
|
|
company_aliases=[a.alias for a in company.aliases],
|
|
competitors=[c.name for c in company.competitors],
|
|
monitoring_focus=company.monitoring_focus,
|
|
industry=company.industry,
|
|
documents=documents,
|
|
detected_changes=changes,
|
|
sources_failed=failed_sources,
|
|
description=company.description,
|
|
official_website=company.official_website,
|
|
headquarters=company.headquarters,
|
|
country=company.country,
|
|
region=company.region,
|
|
public_identifiers=company.public_identifiers,
|
|
enrichment=enrichment,
|
|
)
|
|
|
|
generated_at = datetime.now(UTC)
|
|
model_name = _model_name(settings, llm)
|
|
markdown = render_report_markdown(
|
|
content,
|
|
company_name=company.name,
|
|
generated_at=generated_at.isoformat(),
|
|
model_provider=llm.provider_name,
|
|
model_name=model_name,
|
|
sources=[
|
|
{"title": d["title"], "url": d["url"], "retrieved_date": d["retrieved_date"]}
|
|
for d in documents
|
|
],
|
|
)
|
|
|
|
report = Report(
|
|
company_id=company.id,
|
|
monitoring_run_id=monitoring_run_id,
|
|
report_type=report_type,
|
|
title=f"{company.name} — Competitive Intelligence Report",
|
|
executive_summary=content.executive_summary,
|
|
structured_report=content.model_dump(mode="json"),
|
|
markdown_content=markdown,
|
|
model_provider=llm.provider_name,
|
|
model_name=model_name,
|
|
)
|
|
db.add(report)
|
|
await db.commit()
|
|
await db.refresh(report)
|
|
return report
|
|
|
|
|
|
async def list_reports(db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID) -> list[Report]:
|
|
await company_service.get_company(db, user_id, company_id)
|
|
return await ReportRepository(db).list_for_company(company_id)
|
|
|
|
|
|
async def get_report(db: AsyncSession, user_id: uuid.UUID, report_id: uuid.UUID) -> Report:
|
|
report = await ReportRepository(db).get(report_id)
|
|
if report is None:
|
|
raise NotFoundError("Report not found")
|
|
await company_service.get_company(db, user_id, report.company_id) # ownership check
|
|
return report
|
|
|
|
|
|
async def generate_report_now(
|
|
db: AsyncSession,
|
|
settings: Settings,
|
|
llm: LLMProvider,
|
|
user_id: uuid.UUID,
|
|
company_id: uuid.UUID,
|
|
) -> Report:
|
|
company = await company_service.get_company(db, user_id, company_id)
|
|
return await generate_and_persist_report(
|
|
db, settings, llm, company, report_type=ReportType.MANUAL
|
|
)
|