Backfill sparse report sections, add enrichment section refresh, and bootstrap first-admin

Reports: the LLM reliably used company_enrichment for prose fields but
inconsistently populated the parallel Finding-list/string-list fields from
the same evidence, even with progressively more explicit prompting. Add a
code-level backfill (products, recent developments, financial signals,
strategic initiatives, regulatory signals, risks/opportunities mirrored
from SWOT, unknowns, monitoring recommendations) that only ever fills in
what the model left empty, never overwrites what it produced.

Enrichment tab: reorder sections (Products/Recent updates before
Customers/Competitors) and add a per-section "Refresh" button that
re-fetches just one of NinjaPear's six independent per-company endpoints
when it came back empty - confirmed live that a data-coverage gap (e.g.
Amazon returning no products) is real provider behavior, not a bug.

Auth: the first account registered on a deployment with zero existing
admins is now auto-promoted to admin, closing the chicken-and-egg gap
where the only path to admin access was direct DB access. Self-heals if
the last admin ever deletes their account.

Also bumps nginx's proxy_read_timeout for api.ciagent.org to cover the
enrichment refresh's synchronous funding-endpoint call (up to 5 minutes
per NinjaPear's docs).

Co-Authored-By: Claude Sonnet 5 <[email protected]>
This commit is contained in:
2026-08-06 17:41:02 -04:00
co-authored by Claude Sonnet 5
parent 1be3e53584
commit 18305b545c
18 changed files with 1087 additions and 53 deletions
+32 -1
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
import uuid
from typing import Literal
from fastapi import APIRouter, Depends, Request, status
from sqlalchemy.ext.asyncio import AsyncSession
@@ -13,6 +14,7 @@ from app.core.config import Settings, get_settings
from app.core.errors import NotFoundError
from app.core.rate_limit import limiter
from app.db.session import get_db
from app.enrichment.factory import get_enrichment_provider
from app.models.user import User
from app.schemas.company import (
CompanyCreate,
@@ -23,7 +25,14 @@ from app.schemas.company import (
)
from app.schemas.discovery import DiscoverCompanyRequest, DiscoveredCompanyProfile
from app.search.factory import get_search_provider
from app.services import company_service, discovery_service, user_api_key_service
from app.services import (
company_service,
discovery_service,
enrichment_service,
user_api_key_service,
)
EnrichmentSection = Literal["details", "funding", "updates", "competitors", "products", "customers"]
router = APIRouter(prefix="/companies", tags=["companies"])
@@ -129,6 +138,28 @@ async def resume_company(
return CompanyResponse.from_company(company)
@router.post("/{company_id}/enrichment/sections/{section}/refresh", response_model=CompanyResponse)
@limiter.limit("10/minute")
async def refresh_enrichment_section(
request: Request,
company_id: uuid.UUID,
section: EnrichmentSection,
user: User = Depends(get_current_user),
settings: Settings = Depends(get_settings),
db: AsyncSession = Depends(get_db),
) -> CompanyResponse:
"""Re-fetches one enrichment section (e.g. `products`) that came back
empty or failed, without touching the other sections - see
enrichment_service.refresh_section. Rate-limited since each call spends
real NinjaPear credits."""
company = await company_service.get_company(db, user.id, company_id)
settings = await user_api_key_service.get_effective_settings(db, user.id, settings)
provider = get_enrichment_provider(settings)
await enrichment_service.refresh_section(db, settings, provider, company, section)
refreshed = await company_service.get_company(db, user.id, company_id)
return CompanyResponse.from_company(refreshed)
@router.get("/{company_id}/monitor", response_model=MonitorConfigurationResponse)
async def get_monitor_configuration(
company_id: uuid.UUID,
+59 -10
View File
@@ -21,17 +21,66 @@ SYSTEM_PROMPT = (
"third-party data provider at onboarding, also real evidence, not your own knowledge), "
"stored source documents, and previously detected changes. Never introduce "
"facts from outside knowledge, even if you recognize the company - if it isn't in the "
"evidence block, it doesn't go in the report. The company_profile and company_enrichment "
"fields ARE real evidence and should ground company_overview/market_positioning/"
"financial_signals/leadership_changes/products_and_services/competitor_comparison/"
"customer_sentiment/etc even when source_documents and detected_changes are sparse or "
"evidence block, it doesn't go in the report.\n\n"
"company_profile and company_enrichment are real evidence and must ground every "
"applicable section even when source_documents and detected_changes are sparse or "
"empty - do not say 'insufficient evidence' for a field company_profile or "
"company_enrichment already answers. Every finding must be "
"traceable to the evidence given and must carry an honest confidence label: confirmed "
"(the source states it directly), strongly_indicated, likely, possible, unconfirmed, or "
"insufficient_evidence. When evidence is thin or missing for a section, say so explicitly "
"in that section rather than inventing content. Distinguish clearly between what a source "
"states and what you are inferring."
"company_enrichment already answers. In particular, the list-of-Finding sections are "
"NOT limited to newly detected changes - they must also represent the company's "
"current, confirmed state whenever company_enrichment/company_profile answers them "
"directly, the same way you would use that evidence for a prose field:\n"
"- products_and_services: if company_enrichment.products is present, emit ONE Finding "
"per product (headline = product name, summary = its description/category, "
"confidence = confirmed). A Finding's evidence array may be empty ([]) - "
"company_enrichment is itself the evidence, and an empty evidence array is valid and "
"expected here since there is no source_document or url to cite for it. For example, "
"given company_enrichment.products containing {\"name\": \"Acme Pay\", \"category\": "
"\"Payments\", \"description\": \"Lets merchants accept cards online.\"}, emit: "
"{\"headline\": \"Acme Pay\", \"summary\": \"Lets merchants accept cards online "
"(Payments).\", \"confidence\": \"confirmed\", \"evidence\": []}. Do not leave "
"products_and_services empty just because no source_document specifically announces a "
"product, and do not skip a product merely because you have nothing to put in its "
"evidence array.\n"
"- recent_developments: if company_enrichment.recent_updates is present, emit ONE "
"Finding per genuinely distinct update (headline paraphrasing its text, date = its "
"date, confidence = confirmed since it is the company's own published content, "
"evidence = [{\"url\": its url, \"description\": \"one sentence\"}]). You may skip "
"near-duplicate or routine items, but do not leave this empty when recent_updates has "
"substantive entries.\n"
"- strategic_initiatives and key_inferred_projects: these are your own synthesis "
"across company_enrichment.recent_updates, products, and description - identify "
"recurring or notable strategic themes (a new market entered, a technology bet, a "
"business-model shift) that no single item states outright. Use a lower confidence "
"label here (likely/possible) since this is inference, not a directly-stated fact - "
"an empty evidence array is fine here too. Do not leave these empty just because "
"nothing states 'this is a strategic initiative' in so many words - if you can name a "
"theme in executive_summary or market_positioning, that same theme belongs here as a "
"Finding/InferredProject too, not only as prose. If market_positioning or "
"company_overview names ANY theme, direction, or bet the company is making, restate it "
"here as at least one Finding/InferredProject before considering this section done.\n"
"- regulatory_and_legal_signals: derive from anything in company_enrichment or "
"source_documents touching licensing, compliance, jurisdictions of operation, or legal "
"structure (e.g. a payments company operating in many countries implies licensing/"
"compliance obligations even if no single document states them). If genuinely nothing "
"in the evidence touches this even indirectly, emit one Finding with confidence "
"insufficient_evidence explaining that rather than an empty list.\n"
"- unknowns_and_missing_data and monitoring_recommendations: these two are NOT findings "
"about the company - they are your own meta-analysis of this report and evidence set, so "
"they almost never have 'insufficient evidence' as a valid reason to be empty. You "
"always have something to say: unknowns_and_missing_data should name specific "
"categories of information this evidence set does NOT cover (e.g. 'no financial "
"statements or revenue figures were available', 'no employee reviews or Glassdoor "
"sentiment data', 'pricing details were not provided'); monitoring_recommendations "
"should name specific things worth watching for on the next run (e.g. 'watch for "
"updates to the products list', 'monitor for new funding rounds', 'track leadership "
"page for executive changes'). Leave these empty ONLY if you truly cannot think of a "
"single gap or follow-up, which should be rare.\n\n"
"Every finding must still be traceable to the evidence given and must carry an honest "
"confidence label: confirmed (the source states it directly), strongly_indicated, "
"likely, possible, unconfirmed, or insufficient_evidence. When evidence is genuinely "
"thin or missing for a section, say so explicitly in that section rather than "
"inventing content. Distinguish clearly between what a source states and what you are "
"inferring."
)
+7 -1
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
import uuid
from sqlalchemy import select
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.user import User
@@ -15,6 +15,12 @@ class UserRepository:
async def get_by_id(self, user_id: uuid.UUID) -> User | None:
return await self.db.get(User, user_id)
async def count_admins(self) -> int:
result = await self.db.execute(
select(func.count()).select_from(User).where(User.is_admin.is_(True))
)
return result.scalar_one()
async def get_by_email(self, email: str) -> User | None:
result = await self.db.execute(select(User).where(User.email == email.lower()))
return result.scalar_one_or_none()
+14
View File
@@ -125,11 +125,25 @@ async def register(
if existing is not None:
raise ConflictError("An account with this email already exists")
# Bootstraps admin access on a fresh deployment - otherwise the only way
# to ever get an admin account is direct DB access, which is a real
# chicken-and-egg problem for anyone self-hosting from a clean clone.
# Checked by admin *count*, not total user count, so this also
# self-heals if the last admin ever deletes their own account (Settings
# -> Delete account) - the next registration becomes admin again rather
# than leaving the deployment permanently admin-less. A benign race is
# possible if two people register in the same instant on a brand-new,
# zero-admin deployment (both could become admin) - acceptable for a
# bootstrapping check that only ever matters once, before any real
# traffic exists.
is_first_admin = await repo.count_admins() == 0
user = await repo.create(
email=payload.email,
password_hash=hash_password(payload.password),
display_name=payload.display_name,
timezone=payload.timezone,
is_admin=is_first_admin,
# Test suite has no inbox to read a real code from - same
# app_env == "test" precedent already used to disable rate limiting
# (app/core/rate_limit.py). The code-generation/sending/throttle
+102
View File
@@ -20,6 +20,7 @@ from datetime import UTC, datetime
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.config import Settings
from app.core.errors import NotFoundError
from app.core.logging import get_logger
from app.enrichment.base import EnrichmentProvider
from app.models.company import Company
@@ -174,3 +175,104 @@ async def enrich_company(
failed_sections=list(errors.keys()),
)
return enrichment
# Every entry here maps 1:1 onto one of NinjaPear's independent
# per-company endpoints (see app/enrichment/ninjapear.py) - refreshing one
# section never re-fetches or touches any of the others.
REFRESHABLE_SECTIONS = ("details", "funding", "updates", "competitors", "products", "customers")
async def refresh_section(
db: AsyncSession,
settings: Settings,
provider: EnrichmentProvider,
company: Company,
section: str,
) -> CompanyEnrichment:
"""Re-runs a single NinjaPear endpoint for a company that already has
an enrichment record but came back empty or failed for just this one
section (e.g. `products` returning `[]` while everything else
succeeded - a real, honest gap in NinjaPear's own coverage for that
company, not a bug in this app). Only ever replaces this section's own
slice of `data`/`errors` - every other section's stored data is left
exactly as it was."""
if section not in REFRESHABLE_SECTIONS:
raise ValueError(f"Unknown enrichment section: {section}")
repo = CompanyEnrichmentRepository(db)
existing = await repo.get_for_company(company.id)
if existing is None:
raise NotFoundError("Company has no enrichment record to refresh")
website = company.official_website
data = dict(existing.data)
errors = dict(existing.errors)
credits_spent = existing.credits_spent or 0
try:
if section == "details":
details = await provider.get_company_details(company.name, website)
data["employee_count"] = details.employee_count_range
data["description"] = details.description
data["industry"] = details.industry
data["founded_year"] = details.founded_year
data["specialties"] = details.specialties
new_leadership = [m.model_dump() for m in details.leadership_team]
# A details refresh re-fetches the leadership roster itself,
# but not the separate per-leader work_email/person_profile
# lookups - preserve those by matching on name so a refresh
# never regresses contact info that was already found.
old_by_name = {
m.get("name"): m for m in (data.get("leadership_team") or []) if m.get("name")
}
for member in new_leadership:
old = old_by_name.get(member.get("name"))
if old:
member["work_email"] = member.get("work_email") or old.get("work_email")
member["profile_url"] = member.get("profile_url") or old.get("profile_url")
member["bio"] = member.get("bio") or old.get("bio")
data["leadership_team"] = new_leadership
elif section == "funding":
funding = await provider.get_funding(company.name, website)
data["funding"] = funding.model_dump()
elif section == "updates":
updates = await provider.get_updates(company.name, website)
data["recent_updates"] = [u.model_dump() for u in updates]
elif section == "competitors":
competitors = await provider.get_competitors(company.name, website)
data["competitors"] = [c.model_dump() for c in competitors]
elif section == "products":
products = await provider.get_products(company.name, website)
data["products"] = [p.model_dump() for p in products]
elif section == "customers":
customers = await provider.get_customers(company.name, website)
data["customers"] = [c.model_dump() for c in customers]
errors.pop(section, None)
credits_spent += _CREDIT_COSTS.get(section, 0)
except Exception as exc: # noqa: BLE001 - surfaced via `errors`, same as the initial run
errors[section] = str(exc)
logger.warning(
"enrichment_section_refresh_failed",
section=section,
company_id=str(company.id),
error=str(exc),
)
new_status = EnrichmentStatus.PARTIAL if errors else EnrichmentStatus.COMPLETE
enrichment = await repo.upsert(
company.id,
status=new_status,
data=data,
errors=errors,
credits_spent=credits_spent,
fetched_at=datetime.now(UTC),
)
await db.commit()
logger.info(
"company_enrichment_section_refreshed",
company_id=str(company.id),
section=section,
succeeded=section not in errors,
)
return enrichment
+215 -1
View File
@@ -23,7 +23,8 @@ from app.models.enums import EnrichmentStatus, ReportType, SourceStatus
from app.models.report import Report
from app.models.source import Source
from app.models.source_document import SourceDocument
from app.prompts.report_generation import generate_report
from app.prompts.report_generation import ReportContent, generate_report
from app.prompts.schemas import ConfidenceLabel, EvidenceRef, Finding
from app.repositories.company_enrichment_repository import CompanyEnrichmentRepository
from app.repositories.report_repository import ReportRepository
from app.services import company_service
@@ -31,6 +32,210 @@ from app.services.report_markdown import render_report_markdown
_MAX_DOCUMENTS = 40
_MAX_CHANGES = 20
_MAX_BACKFILLED_PRODUCTS = 30
_MAX_BACKFILLED_UPDATES = 15
_MAX_BACKFILLED_FUNDING_ROUNDS = 15
def _format_amount(amount: object) -> str | None:
if amount is None:
return None
text = str(amount)
return f"${int(text):,}" if text.isdigit() else text
def _backfill_from_enrichment(content: ReportContent, enrichment: dict | None) -> None:
"""company_enrichment.products/recent_updates/funding map onto
products_and_services/recent_developments/financial_signals with no
inference required - a direct transcription, not a judgment call.
Confirmed live that the LLM is nonetheless inconsistent about
populating these Finding-list fields from enrichment alone (it
reliably uses the same data for prose fields like company_overview,
but repeated real API calls with increasingly explicit instructions
still came back with these lists empty). Rather than keep fighting
prompt compliance for a purely mechanical transform, backfill directly
from the source data whenever the model leaves a field empty despite
the data being available - this only ever *adds* real, non-fabricated
content the model chose not to surface, never overwrites what the
model did produce."""
if not enrichment:
return
if not content.products_and_services:
content.products_and_services = [
Finding(
headline=product["name"],
summary=product.get("description") or product.get("category") or product["name"],
category=product.get("category"),
confidence=ConfidenceLabel.CONFIRMED,
)
for product in enrichment.get("products", [])[:_MAX_BACKFILLED_PRODUCTS]
if product.get("name")
]
if not content.recent_developments:
content.recent_developments = [
Finding(
headline=update["text"],
summary=f"{update.get('type', 'update').capitalize()} published by the company.",
date=update.get("date"),
confidence=ConfidenceLabel.CONFIRMED,
evidence=(
[EvidenceRef(url=update["url"], description="Company-published update.")]
if update.get("url")
else []
),
)
for update in enrichment.get("recent_updates", [])[:_MAX_BACKFILLED_UPDATES]
if update.get("text")
]
if not content.financial_signals:
funding = enrichment.get("funding") or {}
findings = []
total_raised = _format_amount(funding.get("total_raised"))
if total_raised:
findings.append(
Finding(
headline=f"Total funding raised: {total_raised}",
summary="Cumulative funding raised across all rounds, per company_enrichment.",
confidence=ConfidenceLabel.CONFIRMED,
)
)
for round_ in funding.get("rounds", [])[:_MAX_BACKFILLED_FUNDING_ROUNDS]:
name = round_.get("round_name")
if not name:
continue
amount = _format_amount(round_.get("amount"))
investors = round_.get("investors") or []
headline = name.replace("_", " ").title() + (f" - {amount}" if amount else "")
findings.append(
Finding(
headline=headline,
summary=(
f"Investors: {', '.join(investors)}."
if investors
else "No investor detail provided."
),
date=round_.get("date"),
confidence=ConfidenceLabel.CONFIRMED,
)
)
content.financial_signals = findings
def _mirror_swot_into_flat_lists(content: ReportContent) -> None:
"""risks/opportunities are meant to be the same analysis as
swot.threats/swot.opportunities, just in a flat top-level list rather
than nested under swot - not a second, independent judgment call.
Confirmed live: the model reliably populates the SWOT fields but is
inconsistent about also populating these parallel flat fields with the
same content, even though nothing about them requires different
evidence. Mirror rather than re-derive, since the model already did
the real analytical work once."""
if not content.risks and content.swot.threats:
content.risks = list(content.swot.threats)
if not content.opportunities and content.swot.opportunities:
content.opportunities = list(content.swot.opportunities)
def _backfill_reflective_sections(
content: ReportContent,
*,
enrichment: dict | None,
documents: list[dict],
changes: list[dict],
company_name: str,
) -> None:
"""strategic_initiatives, regulatory_and_legal_signals,
unknowns_and_missing_data, and monitoring_recommendations are the
fields the LLM was most persistently reluctant to populate even after
two rounds of explicit prompt strengthening (confirmed live - see
SYSTEM_PROMPT in report_generation.py). Unlike products/recent_updates,
these don't have a single obvious mechanical source, but each still
has SOMETHING honest and non-fabricated to fall back on:
- strategic_initiatives: company_enrichment.specialties names the
company's own stated focus areas - a real, low-confidence signal,
not invented.
- regulatory_and_legal_signals: when truly nothing applies, a single
insufficient_evidence Finding is what the prompt already asks the
model to emit in this situation instead of leaving the list empty -
the model just isn't doing it reliably, so this fills the same gap.
- unknowns_and_missing_data / monitoring_recommendations: these are
meta-analysis of the evidence set itself, not claims about the
company, so they can be derived honestly from what evidence this
pipeline actually did or didn't collect for this company."""
enrichment = enrichment or {}
if not content.strategic_initiatives:
specialties = enrichment.get("specialties") or []
content.strategic_initiatives = [
Finding(
headline=f"Focus area: {specialty}",
summary=(
f"{company_name} lists \"{specialty}\" among its specialties, "
"indicating an area of strategic focus."
),
confidence=ConfidenceLabel.POSSIBLE,
)
for specialty in specialties
if specialty
]
if not content.regulatory_and_legal_signals:
content.regulatory_and_legal_signals = [
Finding(
headline="No regulatory or legal signals identified",
summary=(
"No licensing, compliance, jurisdictional, or legal-structure "
"information was present in the available evidence for "
f"{company_name}."
),
confidence=ConfidenceLabel.INSUFFICIENT_EVIDENCE,
)
]
if not content.unknowns_and_missing_data:
unknowns = []
if not enrichment:
unknowns.append(
"No third-party company enrichment data was available for this company."
)
else:
if not enrichment.get("funding", {}).get("total_raised") and not enrichment.get(
"funding", {}
).get("rounds"):
unknowns.append(
"No detailed financial statements, revenue, or funding figures were found."
)
if not enrichment.get("leadership_team"):
unknowns.append("No leadership or executive team data was found.")
if not enrichment.get("customers"):
unknowns.append("No named customers or case studies were found.")
if not documents:
unknowns.append(
"No source documents have been collected yet from ongoing monitoring, so "
"hiring, technology, patent, and manufacturing signals are not yet available."
)
if not changes:
unknowns.append(
"No changes have been detected yet between monitoring runs for this company."
)
content.unknowns_and_missing_data = unknowns
if not content.monitoring_recommendations:
recommendations = [
f"Monitor company_enrichment for {company_name} on its next refresh for changes "
"to products, leadership, or funding.",
"Track newly published company updates for announcements of new initiatives "
"or partnerships.",
]
if documents:
recommendations.append(
"Continue reviewing newly collected source documents for signals not yet "
"reflected in company_enrichment."
)
content.monitoring_recommendations = recommendations
async def _gather_documents(db: AsyncSession, company_id: uuid.UUID) -> list[dict]:
@@ -130,6 +335,15 @@ async def generate_and_persist_report(
public_identifiers=company.public_identifiers,
enrichment=enrichment,
)
_backfill_from_enrichment(content, enrichment)
_mirror_swot_into_flat_lists(content)
_backfill_reflective_sections(
content,
enrichment=enrichment,
documents=documents,
changes=changes,
company_name=company.name,
)
generated_at = datetime.now(UTC)
model_name = _model_name(settings, llm)