Initial commit: CI Agent competitive-intelligence monitoring app
FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification).
This commit is contained in:
@@ -0,0 +1,202 @@
|
||||
"""Turns a DetectedChange that crosses the company's alert threshold into an
|
||||
Alert, generates its summary via Task F (LLM), and dispatches it to every
|
||||
enabled notification destination that also meets its own severity threshold
|
||||
- recording one NotificationDelivery per attempt. Two independent
|
||||
thresholds by design: MonitorConfiguration.severity_threshold gates whether
|
||||
an Alert is created at all; NotificationDestination.minimum_severity then
|
||||
gates whether *this* destination gets notified about it (spec section 6I/22).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.analysis.llm.base import LLMProvider
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import NotFoundError
|
||||
from app.models.alert import Alert
|
||||
from app.models.company import Company
|
||||
from app.models.detected_change import DetectedChange
|
||||
from app.models.enums import (
|
||||
SEVERITY_ORDER,
|
||||
NotificationDeliveryStatus,
|
||||
NotificationType,
|
||||
SeverityLevel,
|
||||
)
|
||||
from app.models.notification_delivery import NotificationDelivery
|
||||
from app.notifications.base import DeliveryResult, NotificationMessage
|
||||
from app.notifications.factory import get_notification_provider
|
||||
from app.notifications.message_builder import build_alert_message
|
||||
from app.prompts.alert_summarization import summarize_alert
|
||||
from app.repositories.alert_repository import AlertRepository
|
||||
from app.repositories.notification_delivery_repository import NotificationDeliveryRepository
|
||||
from app.repositories.notification_destination_repository import NotificationDestinationRepository
|
||||
|
||||
|
||||
def _meets_threshold(severity: SeverityLevel, threshold: SeverityLevel) -> bool:
|
||||
"""SEVERITY_ORDER[0] is most severe, so meeting a threshold means being
|
||||
at least as severe - i.e. an index <= the threshold's index."""
|
||||
return SEVERITY_ORDER.index(severity) <= SEVERITY_ORDER.index(threshold)
|
||||
|
||||
|
||||
async def create_alert_for_change(
|
||||
db: AsyncSession, settings: Settings, llm: LLMProvider, change: DetectedChange, company: Company
|
||||
) -> Alert | None:
|
||||
config = company.monitor_configuration
|
||||
if config is None or not _meets_threshold(change.severity, config.severity_threshold):
|
||||
return None
|
||||
|
||||
evidence_snippets = [
|
||||
*change.raw_diff.get("text_added_lines", [])[:5],
|
||||
*change.raw_diff.get("structured_added", [])[:5],
|
||||
]
|
||||
summary = await summarize_alert(
|
||||
llm,
|
||||
company_name=company.name,
|
||||
change_type=change.change_type.value,
|
||||
change_summary=change.summary,
|
||||
severity=change.severity.value,
|
||||
confidence=change.confidence_score,
|
||||
evidence_snippets=evidence_snippets,
|
||||
)
|
||||
|
||||
alert = Alert(
|
||||
company_id=company.id,
|
||||
detected_change_id=change.id,
|
||||
user_id=company.user_id,
|
||||
title=summary.title,
|
||||
summary=summary.summary,
|
||||
why_it_matters=summary.why_it_matters,
|
||||
severity=change.severity,
|
||||
confidence=change.confidence_score,
|
||||
)
|
||||
await AlertRepository(db).create(alert)
|
||||
|
||||
destinations = await NotificationDestinationRepository(db).list_for_company(company.id)
|
||||
delivery_repo = NotificationDeliveryRepository(db)
|
||||
for destination in destinations:
|
||||
if not destination.enabled:
|
||||
continue
|
||||
if not _meets_threshold(change.severity, destination.minimum_severity):
|
||||
continue
|
||||
if destination.type == NotificationType.SMS and not settings.notification_sms_enabled:
|
||||
continue
|
||||
|
||||
provider = get_notification_provider(destination.type, settings)
|
||||
message = build_alert_message(
|
||||
destination.type, destination.destination_value, company, alert, settings
|
||||
)
|
||||
delivery = NotificationDelivery(
|
||||
alert_id=alert.id, destination_id=destination.id, provider=provider.provider_name
|
||||
)
|
||||
await delivery_repo.create(delivery)
|
||||
|
||||
result = await provider.send(message)
|
||||
delivery.attempt_count = 1
|
||||
delivery.last_attempt = datetime.now(UTC)
|
||||
delivery.status = (
|
||||
NotificationDeliveryStatus.SENT if result.success else NotificationDeliveryStatus.FAILED
|
||||
)
|
||||
delivery.external_message_id = result.external_message_id
|
||||
delivery.error_message = result.error
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(alert)
|
||||
return alert
|
||||
|
||||
|
||||
async def list_alerts(
|
||||
db: AsyncSession,
|
||||
user_id: uuid.UUID,
|
||||
*,
|
||||
company_id: uuid.UUID | None = None,
|
||||
severity: SeverityLevel | None = None,
|
||||
read: bool | None = None,
|
||||
resolved: bool | None = None,
|
||||
) -> list[Alert]:
|
||||
return await AlertRepository(db).list_for_user(
|
||||
user_id, company_id=company_id, severity=severity, read=read, resolved=resolved
|
||||
)
|
||||
|
||||
|
||||
async def get_alert(db: AsyncSession, user_id: uuid.UUID, alert_id: uuid.UUID) -> Alert:
|
||||
alert = await AlertRepository(db).get_for_user(alert_id, user_id)
|
||||
if alert is None:
|
||||
raise NotFoundError("Alert not found")
|
||||
return alert
|
||||
|
||||
|
||||
async def mark_read(db: AsyncSession, user_id: uuid.UUID, alert_id: uuid.UUID) -> Alert:
|
||||
alert = await get_alert(db, user_id, alert_id)
|
||||
alert.read = True
|
||||
await db.commit()
|
||||
await db.refresh(alert)
|
||||
return alert
|
||||
|
||||
|
||||
async def mark_resolved(db: AsyncSession, user_id: uuid.UUID, alert_id: uuid.UUID) -> Alert:
|
||||
alert = await get_alert(db, user_id, alert_id)
|
||||
alert.resolved = True
|
||||
await db.commit()
|
||||
await db.refresh(alert)
|
||||
return alert
|
||||
|
||||
|
||||
async def update_alert(
|
||||
db: AsyncSession,
|
||||
user_id: uuid.UUID,
|
||||
alert_id: uuid.UUID,
|
||||
*,
|
||||
read: bool | None,
|
||||
resolved: bool | None,
|
||||
) -> Alert:
|
||||
alert = await get_alert(db, user_id, alert_id)
|
||||
if read is not None:
|
||||
alert.read = read
|
||||
if resolved is not None:
|
||||
alert.resolved = resolved
|
||||
await db.commit()
|
||||
await db.refresh(alert)
|
||||
return alert
|
||||
|
||||
|
||||
async def get_alert_with_deliveries(
|
||||
db: AsyncSession, user_id: uuid.UUID, alert_id: uuid.UUID
|
||||
) -> tuple[Alert, list[NotificationDelivery]]:
|
||||
alert = await get_alert(db, user_id, alert_id)
|
||||
deliveries = await NotificationDeliveryRepository(db).list_for_alert(alert.id)
|
||||
return alert, deliveries
|
||||
|
||||
|
||||
async def send_test_notification(
|
||||
db: AsyncSession, settings: Settings, user_id: uuid.UUID, destination_id: uuid.UUID
|
||||
):
|
||||
destination = await NotificationDestinationRepository(db).get_for_user(destination_id, user_id)
|
||||
if destination is None:
|
||||
raise NotFoundError("Notification destination not found")
|
||||
|
||||
if destination.type == NotificationType.SMS and not settings.notification_sms_enabled:
|
||||
return DeliveryResult(
|
||||
success=False,
|
||||
error="SMS delivery is currently disabled (NOTIFICATION_SMS_ENABLED=false) - no "
|
||||
"message was sent.",
|
||||
)
|
||||
|
||||
provider = get_notification_provider(destination.type, settings)
|
||||
if destination.type == NotificationType.SMS:
|
||||
message = NotificationMessage(
|
||||
destination_value=destination.destination_value,
|
||||
subject="CI Agent test",
|
||||
body_text="CI Agent test SMS: your notification destination is configured correctly.",
|
||||
)
|
||||
else:
|
||||
message = NotificationMessage(
|
||||
destination_value=destination.destination_value,
|
||||
subject="CI Agent test notification",
|
||||
body_text="This is a test notification from CI Agent. Your destination is configured correctly.",
|
||||
body_html="<p>This is a test notification from <strong>CI Agent</strong>. Your destination is configured correctly.</p>",
|
||||
)
|
||||
return await provider.send(message)
|
||||
@@ -0,0 +1,105 @@
|
||||
"""Dashboard analytics: aggregate counts across every company a user owns,
|
||||
scoped by joining through Company.user_id (none of the source tables carry
|
||||
user_id directly except Alert). Every bucketed dict is pre-seeded with every
|
||||
enum member at 0 so the frontend never has to guess which keys might be
|
||||
missing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.alert import Alert
|
||||
from app.models.company import Company
|
||||
from app.models.detected_change import DetectedChange
|
||||
from app.models.enums import ChangeType, MonitoringRunStatus, SeverityLevel, SourceStatus
|
||||
from app.models.monitoring_run import MonitoringRun
|
||||
from app.models.source import Source
|
||||
from app.schemas.dashboard import DashboardAnalytics, RecentSignal, RunsByDayPoint
|
||||
|
||||
|
||||
async def get_dashboard_analytics(
|
||||
db: AsyncSession, user_id: uuid.UUID, *, days: int = 30, recent_limit: int = 10
|
||||
) -> DashboardAnalytics:
|
||||
since = datetime.now(UTC) - timedelta(days=days)
|
||||
|
||||
changes_by_type = {ct.value: 0 for ct in ChangeType}
|
||||
changes_result = await db.execute(
|
||||
select(DetectedChange.change_type, func.count())
|
||||
.join(Company, DetectedChange.company_id == Company.id)
|
||||
.where(Company.user_id == user_id, DetectedChange.created_at >= since)
|
||||
.group_by(DetectedChange.change_type)
|
||||
)
|
||||
for change_type, count in changes_result.all():
|
||||
changes_by_type[change_type.value] = count
|
||||
|
||||
alerts_by_severity = {s.value: 0 for s in SeverityLevel}
|
||||
alerts_result = await db.execute(
|
||||
select(Alert.severity, func.count())
|
||||
.where(Alert.user_id == user_id, Alert.created_at >= since)
|
||||
.group_by(Alert.severity)
|
||||
)
|
||||
for severity, count in alerts_result.all():
|
||||
alerts_by_severity[severity.value] = count
|
||||
|
||||
sources_by_status = {s.value: 0 for s in SourceStatus}
|
||||
sources_result = await db.execute(
|
||||
select(Source.status, func.count())
|
||||
.join(Company, Source.company_id == Company.id)
|
||||
.where(Company.user_id == user_id)
|
||||
.group_by(Source.status)
|
||||
)
|
||||
for source_status, count in sources_result.all():
|
||||
sources_by_status[source_status.value] = count
|
||||
|
||||
runs_result = await db.execute(
|
||||
select(MonitoringRun.created_at, MonitoringRun.status)
|
||||
.join(Company, MonitoringRun.company_id == Company.id)
|
||||
.where(Company.user_id == user_id, MonitoringRun.created_at >= since)
|
||||
)
|
||||
day_buckets: dict[str, dict[str, int]] = {}
|
||||
for created_at, run_status in runs_result.all():
|
||||
day = created_at.date().isoformat()
|
||||
bucket = day_buckets.setdefault(day, {"successful": 0, "failed": 0, "other": 0})
|
||||
if run_status == MonitoringRunStatus.SUCCESSFUL:
|
||||
bucket["successful"] += 1
|
||||
elif run_status == MonitoringRunStatus.FAILED:
|
||||
bucket["failed"] += 1
|
||||
else:
|
||||
bucket["other"] += 1
|
||||
runs_by_day = [
|
||||
RunsByDayPoint(date=day, **counts) for day, counts in sorted(day_buckets.items())
|
||||
]
|
||||
|
||||
signals_result = await db.execute(
|
||||
select(DetectedChange, Company.name)
|
||||
.join(Company, DetectedChange.company_id == Company.id)
|
||||
.where(Company.user_id == user_id)
|
||||
.order_by(DetectedChange.created_at.desc())
|
||||
.limit(recent_limit)
|
||||
)
|
||||
recent_signals = [
|
||||
RecentSignal(
|
||||
id=change.id,
|
||||
company_id=change.company_id,
|
||||
company_name=company_name,
|
||||
change_type=change.change_type.value,
|
||||
severity=change.severity.value,
|
||||
confidence_score=change.confidence_score,
|
||||
summary=change.summary,
|
||||
created_at=change.created_at,
|
||||
)
|
||||
for change, company_name in signals_result.all()
|
||||
]
|
||||
|
||||
return DashboardAnalytics(
|
||||
changes_by_type=changes_by_type,
|
||||
alerts_by_severity=alerts_by_severity,
|
||||
sources_by_status=sources_by_status,
|
||||
runs_by_day=runs_by_day,
|
||||
recent_signals=recent_signals,
|
||||
)
|
||||
@@ -0,0 +1,428 @@
|
||||
"""Registration/login/refresh/logout/verify-email/password-reset business
|
||||
logic.
|
||||
|
||||
Kept independent of FastAPI so it's reachable from tests and (later) from
|
||||
Celery tasks or an admin script without pulling in the HTTP layer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import AuthenticationError, ConflictError, ThrottledError, ValidationAppError
|
||||
from app.core.security import (
|
||||
InvalidTokenError,
|
||||
TokenType,
|
||||
create_access_token,
|
||||
create_refresh_token,
|
||||
decode_token,
|
||||
generate_email_code,
|
||||
hash_email_code,
|
||||
hash_password,
|
||||
hash_token_identifier,
|
||||
verify_password,
|
||||
)
|
||||
from app.db.base import ensure_aware_utc
|
||||
from app.models.enums import EmailCodePurpose, SecurityEventType, ThrottleAction
|
||||
from app.models.user import LOCAL_DEV_USER_EMAIL, LOCAL_DEV_USER_ID, User
|
||||
from app.models.user_security_event import UserSecurityEvent
|
||||
from app.repositories.email_code_repository import EmailCodeRepository
|
||||
from app.repositories.password_history_repository import PasswordHistoryRepository
|
||||
from app.repositories.refresh_token_repository import RefreshTokenRepository
|
||||
from app.repositories.user_known_ip_repository import UserKnownIpRepository
|
||||
from app.repositories.user_repository import UserRepository
|
||||
from app.repositories.user_security_event_repository import UserSecurityEventRepository
|
||||
from app.schemas.auth import (
|
||||
ConfirmPasswordResetRequest,
|
||||
LoginRequest,
|
||||
RegisterRequest,
|
||||
RequestPasswordResetRequest,
|
||||
ResendVerificationRequest,
|
||||
TokenResponse,
|
||||
VerifyEmailRequest,
|
||||
)
|
||||
from app.services import ip_throttle_service, security_email_service
|
||||
|
||||
EMAIL_CODE_VALID_HOURS = 36
|
||||
|
||||
# The local-dev bypass (AUTH_MODE=local + loopback, see
|
||||
# app/auth/dependencies.py) has no real login step - get_or_create_local_user
|
||||
# runs on every authenticated request. Logging a login_success event on every
|
||||
# single request would flood Account activity, so a fresh one is only
|
||||
# recorded once per this cooldown window, treated as a proxy for "a new
|
||||
# session" rather than every request within one.
|
||||
LOCAL_DEV_LOGIN_LOG_COOLDOWN_MINUTES = 30
|
||||
|
||||
|
||||
async def get_or_create_local_user(db: AsyncSession, client_ip: str) -> User:
|
||||
repo = UserRepository(db)
|
||||
user = await repo.get_by_id(LOCAL_DEV_USER_ID)
|
||||
if user is None:
|
||||
user = await repo.create(
|
||||
email=LOCAL_DEV_USER_EMAIL,
|
||||
password_hash=None,
|
||||
display_name="Local Developer",
|
||||
timezone="America/New_York",
|
||||
user_id=LOCAL_DEV_USER_ID,
|
||||
is_admin=True,
|
||||
email_verified=True,
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
event_repo = UserSecurityEventRepository(db)
|
||||
last_login = await event_repo.most_recent_of_type(user.id, SecurityEventType.LOGIN_SUCCESS)
|
||||
cutoff = datetime.now(UTC) - timedelta(minutes=LOCAL_DEV_LOGIN_LOG_COOLDOWN_MINUTES)
|
||||
if last_login is None or ensure_aware_utc(last_login.created_at) < cutoff:
|
||||
await event_repo.create(
|
||||
user_id=user.id, event_type=SecurityEventType.LOGIN_SUCCESS, ip_address=client_ip
|
||||
)
|
||||
await UserKnownIpRepository(db).record_login(user.id, client_ip, datetime.now(UTC))
|
||||
await db.commit()
|
||||
return user
|
||||
|
||||
|
||||
async def _issue_and_send_email_code(
|
||||
db: AsyncSession, settings: Settings, user: User, purpose: EmailCodePurpose
|
||||
) -> None:
|
||||
code = generate_email_code()
|
||||
code_repo = EmailCodeRepository(db)
|
||||
# A fresh code must fully supersede every prior unused one for this
|
||||
# purpose - otherwise an old code (e.g. still sitting in an earlier
|
||||
# email) stays valid until it naturally expires, even after the user
|
||||
# explicitly asked for a new one.
|
||||
await code_repo.invalidate_unused(user.id, purpose)
|
||||
await code_repo.create(
|
||||
user_id=user.id,
|
||||
purpose=purpose,
|
||||
code_hash=hash_email_code(code),
|
||||
expires_at=datetime.now(UTC) + timedelta(hours=EMAIL_CODE_VALID_HOURS),
|
||||
)
|
||||
if purpose == EmailCodePurpose.VERIFY_EMAIL:
|
||||
await security_email_service.send_verification_code_email(settings, user.email, code)
|
||||
else:
|
||||
await security_email_service.send_password_reset_email(settings, user.email, code)
|
||||
|
||||
|
||||
async def register(
|
||||
db: AsyncSession, settings: Settings, client_ip: str, payload: RegisterRequest
|
||||
) -> User:
|
||||
if await ip_throttle_service.is_banned(db, client_ip):
|
||||
raise ThrottledError("This IP address has been temporarily blocked.")
|
||||
|
||||
repo = UserRepository(db)
|
||||
existing = await repo.get_by_email(payload.email)
|
||||
if existing is not None:
|
||||
raise ConflictError("An account with this email already exists")
|
||||
|
||||
user = await repo.create(
|
||||
email=payload.email,
|
||||
password_hash=hash_password(payload.password),
|
||||
display_name=payload.display_name,
|
||||
timezone=payload.timezone,
|
||||
# Test suite has no inbox to read a real code from - same
|
||||
# app_env == "test" precedent already used to disable rate limiting
|
||||
# (app/core/rate_limit.py). The code-generation/sending/throttle
|
||||
# path below still runs unconditionally either way, so it's still
|
||||
# exercised by every test that registers a user, not skipped.
|
||||
email_verified=settings.app_env == "test",
|
||||
)
|
||||
await _issue_and_send_email_code(db, settings, user, EmailCodePurpose.VERIFY_EMAIL)
|
||||
# Deliberately does NOT call ip_throttle_service.record_attempt here -
|
||||
# this is per-IP, not per-account (see ip_throttle_service's docstring),
|
||||
# so unconditionally charging every new registration against it would
|
||||
# let unrelated people sharing an IP (an office, a NAT'd network) drive
|
||||
# each other toward a shared ban purely through legitimate signups. The
|
||||
# escalation ladder starts from the first *manual* resend click instead
|
||||
# (resend_verification endpoint) - "first resend allowed in 30s" is a
|
||||
# frontend-only initial cooldown after registration, not a backend-
|
||||
# enforced one; the backend ladder governs resend #2 onward.
|
||||
await UserSecurityEventRepository(db).create(
|
||||
user_id=user.id, event_type=SecurityEventType.EMAIL_VERIFICATION_SENT, ip_address=client_ip
|
||||
)
|
||||
await db.commit()
|
||||
return user
|
||||
|
||||
|
||||
async def verify_email(db: AsyncSession, client_ip: str, payload: VerifyEmailRequest) -> None:
|
||||
"""Generic failure for both 'no such account' and 'wrong/expired code' -
|
||||
never distinguishes the two to the caller. Guessing the code itself is
|
||||
IP-throttled the same way a login password guess is - a 6-digit code
|
||||
has only 1M possible values, so without this an attacker could
|
||||
brute-force it well within its 36-hour validity window."""
|
||||
throttle = await ip_throttle_service.peek_throttle(
|
||||
db, client_ip, ThrottleAction.VERIFY_EMAIL_CODE
|
||||
)
|
||||
if not throttle.allowed:
|
||||
_raise_throttled(throttle)
|
||||
|
||||
user_repo = UserRepository(db)
|
||||
code_repo = EmailCodeRepository(db)
|
||||
user = await user_repo.get_by_email(payload.email)
|
||||
record = (
|
||||
await code_repo.get_latest_valid(
|
||||
user.id, EmailCodePurpose.VERIFY_EMAIL, hash_email_code(payload.code)
|
||||
)
|
||||
if user is not None
|
||||
else None
|
||||
)
|
||||
if record is None:
|
||||
await ip_throttle_service.record_attempt(
|
||||
db,
|
||||
client_ip,
|
||||
ThrottleAction.VERIFY_EMAIL_CODE,
|
||||
ip_throttle_service.LOGIN_BACKOFF_SECONDS,
|
||||
)
|
||||
await db.commit()
|
||||
raise AuthenticationError("Invalid or expired verification code")
|
||||
|
||||
await code_repo.mark_used(record)
|
||||
user.email_verified = True
|
||||
await ip_throttle_service.reset_on_success(db, client_ip, ThrottleAction.VERIFY_EMAIL_CODE)
|
||||
await UserSecurityEventRepository(db).create(
|
||||
user_id=user.id, event_type=SecurityEventType.EMAIL_VERIFIED, ip_address=client_ip
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def resend_verification(
|
||||
db: AsyncSession, settings: Settings, client_ip: str, payload: ResendVerificationRequest
|
||||
) -> None:
|
||||
throttle = await ip_throttle_service.peek_throttle(
|
||||
db, client_ip, ThrottleAction.RESEND_VERIFICATION
|
||||
)
|
||||
if not throttle.allowed:
|
||||
_raise_throttled(throttle)
|
||||
|
||||
user_repo = UserRepository(db)
|
||||
user = await user_repo.get_by_email(payload.email)
|
||||
# Always record the attempt, whether or not the account exists, so the
|
||||
# throttle behavior itself can never leak account existence.
|
||||
await ip_throttle_service.record_attempt(
|
||||
db,
|
||||
client_ip,
|
||||
ThrottleAction.RESEND_VERIFICATION,
|
||||
ip_throttle_service.RESEND_BACKOFF_SECONDS,
|
||||
)
|
||||
if user is not None and not user.email_verified:
|
||||
await _issue_and_send_email_code(db, settings, user, EmailCodePurpose.VERIFY_EMAIL)
|
||||
await UserSecurityEventRepository(db).create(
|
||||
user_id=user.id,
|
||||
event_type=SecurityEventType.EMAIL_VERIFICATION_SENT,
|
||||
ip_address=client_ip,
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def request_password_reset(
|
||||
db: AsyncSession, settings: Settings, client_ip: str, payload: RequestPasswordResetRequest
|
||||
) -> None:
|
||||
throttle = await ip_throttle_service.peek_throttle(db, client_ip, ThrottleAction.RESEND_RESET)
|
||||
if not throttle.allowed:
|
||||
_raise_throttled(throttle)
|
||||
|
||||
user_repo = UserRepository(db)
|
||||
user = await user_repo.get_by_email(payload.email)
|
||||
await ip_throttle_service.record_attempt(
|
||||
db, client_ip, ThrottleAction.RESEND_RESET, ip_throttle_service.RESEND_BACKOFF_SECONDS
|
||||
)
|
||||
if user is not None:
|
||||
await _issue_and_send_email_code(db, settings, user, EmailCodePurpose.PASSWORD_RESET)
|
||||
await UserSecurityEventRepository(db).create(
|
||||
user_id=user.id,
|
||||
event_type=SecurityEventType.PASSWORD_RESET_REQUESTED,
|
||||
ip_address=client_ip,
|
||||
)
|
||||
# Always a generic success, regardless of whether the account exists -
|
||||
# the classic enumeration-safe pattern.
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def confirm_password_reset(
|
||||
db: AsyncSession, client_ip: str, payload: ConfirmPasswordResetRequest
|
||||
) -> None:
|
||||
"""Same code-guess throttling rationale as verify_email - a reset code
|
||||
is just as brute-forceable if left unthrottled."""
|
||||
throttle = await ip_throttle_service.peek_throttle(
|
||||
db, client_ip, ThrottleAction.CONFIRM_RESET_CODE
|
||||
)
|
||||
if not throttle.allowed:
|
||||
_raise_throttled(throttle)
|
||||
|
||||
user_repo = UserRepository(db)
|
||||
code_repo = EmailCodeRepository(db)
|
||||
user = await user_repo.get_by_email(payload.email)
|
||||
record = (
|
||||
await code_repo.get_latest_valid(
|
||||
user.id, EmailCodePurpose.PASSWORD_RESET, hash_email_code(payload.code)
|
||||
)
|
||||
if user is not None
|
||||
else None
|
||||
)
|
||||
if record is None:
|
||||
await ip_throttle_service.record_attempt(
|
||||
db,
|
||||
client_ip,
|
||||
ThrottleAction.CONFIRM_RESET_CODE,
|
||||
ip_throttle_service.LOGIN_BACKOFF_SECONDS,
|
||||
)
|
||||
await db.commit()
|
||||
raise AuthenticationError("Invalid or expired reset code")
|
||||
|
||||
# Checked before consuming the code, and before recording any throttle
|
||||
# attempt, so a rejected-for-reuse password never burns the (correctly
|
||||
# entered) code - the user can immediately retry with a different one.
|
||||
history_repo = PasswordHistoryRepository(db)
|
||||
previous_hashes = await history_repo.list_hashes_for_user(user.id)
|
||||
if user.password_hash is not None:
|
||||
previous_hashes.append(user.password_hash)
|
||||
if any(verify_password(payload.new_password, h) for h in previous_hashes):
|
||||
raise ValidationAppError("You've used this password before. Choose a different one.")
|
||||
|
||||
await code_repo.mark_used(record)
|
||||
if user.password_hash is not None:
|
||||
await history_repo.add(user_id=user.id, password_hash=user.password_hash)
|
||||
user.password_hash = hash_password(payload.new_password)
|
||||
# This is the documented unlock mechanism - a successful reset always
|
||||
# clears any prior lockout, regardless of how it got there.
|
||||
user.locked_at = None
|
||||
user.failed_login_count = 0
|
||||
|
||||
await ip_throttle_service.reset_on_success(db, client_ip, ThrottleAction.CONFIRM_RESET_CODE)
|
||||
await RefreshTokenRepository(db).revoke_all_for_user(user.id)
|
||||
await UserSecurityEventRepository(db).create(
|
||||
user_id=user.id, event_type=SecurityEventType.PASSWORD_RESET_COMPLETED, ip_address=client_ip
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
|
||||
def _raise_throttled(throttle: ip_throttle_service.ThrottleResult) -> None:
|
||||
if throttle.banned:
|
||||
raise ThrottledError("This IP address has been temporarily blocked.")
|
||||
raise ThrottledError(
|
||||
f"Too many attempts. Try again in {throttle.retry_after_seconds} seconds.",
|
||||
retry_after_seconds=throttle.retry_after_seconds,
|
||||
)
|
||||
|
||||
|
||||
async def _issue_token_pair(db: AsyncSession, settings: Settings, user: User) -> TokenResponse:
|
||||
access_token = create_access_token(user.id, settings)
|
||||
refresh_jwt, jti, expires_at = create_refresh_token(user.id, settings)
|
||||
|
||||
token_repo = RefreshTokenRepository(db)
|
||||
await token_repo.create(
|
||||
user_id=user.id, token_hash=hash_token_identifier(jti), expires_at=expires_at
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
return TokenResponse(
|
||||
access_token=access_token,
|
||||
refresh_token=refresh_jwt,
|
||||
expires_in_minutes=settings.jwt_access_token_minutes,
|
||||
)
|
||||
|
||||
|
||||
# 5 free instant retries, then 5s/15s/30s/60s/2min/5min/15min - the 12th
|
||||
# failure (index past the end of LOGIN_BACKOFF_SECONDS) is also where the
|
||||
# account itself locks, in lockstep with the IP throttle's own escalation.
|
||||
LOGIN_LOCKOUT_THRESHOLD = len(ip_throttle_service.LOGIN_BACKOFF_SECONDS) + 1
|
||||
|
||||
|
||||
async def login(
|
||||
db: AsyncSession, settings: Settings, client_ip: str, payload: LoginRequest
|
||||
) -> TokenResponse:
|
||||
# Enforced *before* password verification - a correct password during a
|
||||
# penalty window must still be rejected, or the delay is meaningless.
|
||||
throttle = await ip_throttle_service.peek_throttle(db, client_ip, ThrottleAction.FAILED_LOGIN)
|
||||
if not throttle.allowed:
|
||||
_raise_throttled(throttle)
|
||||
|
||||
repo = UserRepository(db)
|
||||
user = await repo.get_by_email(payload.email)
|
||||
|
||||
if user is not None and user.locked_at is not None:
|
||||
# Generic message either way - never confirms the account exists.
|
||||
raise AuthenticationError("Account locked. Reset your password to unlock it.")
|
||||
|
||||
password_ok = (
|
||||
user is not None
|
||||
and user.password_hash is not None
|
||||
and verify_password(payload.password, user.password_hash)
|
||||
)
|
||||
|
||||
if not password_ok:
|
||||
await ip_throttle_service.record_attempt(
|
||||
db, client_ip, ThrottleAction.FAILED_LOGIN, ip_throttle_service.LOGIN_BACKOFF_SECONDS
|
||||
)
|
||||
if user is not None:
|
||||
user.failed_login_count += 1
|
||||
await UserSecurityEventRepository(db).create(
|
||||
user_id=user.id, event_type=SecurityEventType.LOGIN_FAILED, ip_address=client_ip
|
||||
)
|
||||
if user.failed_login_count >= LOGIN_LOCKOUT_THRESHOLD:
|
||||
user.locked_at = datetime.now(UTC)
|
||||
await UserSecurityEventRepository(db).create(
|
||||
user_id=user.id,
|
||||
event_type=SecurityEventType.ACCOUNT_LOCKED,
|
||||
ip_address=client_ip,
|
||||
)
|
||||
await security_email_service.send_account_locked_email(settings, user.email)
|
||||
await db.commit()
|
||||
raise AuthenticationError("Invalid email or password")
|
||||
|
||||
assert user is not None # password_ok implies this
|
||||
if not user.is_active:
|
||||
raise AuthenticationError("This account has been deactivated")
|
||||
if not user.email_verified:
|
||||
raise AuthenticationError("Verify your email before signing in.")
|
||||
|
||||
user.failed_login_count = 0
|
||||
await ip_throttle_service.reset_on_success(db, client_ip, ThrottleAction.FAILED_LOGIN)
|
||||
await UserSecurityEventRepository(db).create(
|
||||
user_id=user.id, event_type=SecurityEventType.LOGIN_SUCCESS, ip_address=client_ip
|
||||
)
|
||||
await UserKnownIpRepository(db).record_login(user.id, client_ip, datetime.now(UTC))
|
||||
return await _issue_token_pair(db, settings, user)
|
||||
|
||||
|
||||
async def refresh(db: AsyncSession, settings: Settings, raw_refresh_token: str) -> TokenResponse:
|
||||
try:
|
||||
decoded = decode_token(raw_refresh_token, settings, TokenType.REFRESH)
|
||||
except InvalidTokenError as exc:
|
||||
raise AuthenticationError("Invalid or expired refresh token") from exc
|
||||
|
||||
token_repo = RefreshTokenRepository(db)
|
||||
stored = await token_repo.get_valid_by_hash(hash_token_identifier(decoded.jti))
|
||||
if stored is None:
|
||||
raise AuthenticationError("Invalid or expired refresh token")
|
||||
|
||||
user_repo = UserRepository(db)
|
||||
user = await user_repo.get_by_id(decoded.user_id)
|
||||
if user is None or not user.is_active:
|
||||
raise AuthenticationError("Invalid or expired refresh token")
|
||||
|
||||
# Rotate: revoke the token just used before issuing a new pair.
|
||||
await token_repo.revoke(stored)
|
||||
return await _issue_token_pair(db, settings, user)
|
||||
|
||||
|
||||
async def logout(db: AsyncSession, settings: Settings, raw_refresh_token: str) -> None:
|
||||
try:
|
||||
decoded = decode_token(raw_refresh_token, settings, TokenType.REFRESH)
|
||||
except InvalidTokenError:
|
||||
return # Already unusable; logout is idempotent.
|
||||
|
||||
token_repo = RefreshTokenRepository(db)
|
||||
stored = await token_repo.get_valid_by_hash(hash_token_identifier(decoded.jti))
|
||||
if stored is not None:
|
||||
await token_repo.revoke(stored)
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def list_security_events(db: AsyncSession, user_id: uuid.UUID) -> list[UserSecurityEvent]:
|
||||
"""The calling user's own security activity - the user-facing
|
||||
counterpart to the admin-only app-wide log feed (core/logging.py)."""
|
||||
return await UserSecurityEventRepository(db).list_for_user(user_id)
|
||||
@@ -0,0 +1,172 @@
|
||||
"""Orchestrates the change-detection layers (hash -> structured diff -> text
|
||||
diff -> scoring) into a single `DetectedChange` row, or nothing if the
|
||||
change isn't real/meaningful/novel enough to record.
|
||||
|
||||
Cross-source corroboration (Task C in the spec's LLM analysis section) is
|
||||
Phase 7 scope - `independent_source_count` is always 1 here. See
|
||||
KNOWN_LIMITATIONS.md.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.change_detection.extractors import extract_prices, mentions_leadership_title
|
||||
from app.change_detection.scoring import classify_severity, compute_confidence, compute_significance
|
||||
from app.change_detection.structured_diff import StructuredDiff, diff_item_sets
|
||||
from app.change_detection.text_diff import TextDiffResult, bounded_text_diff
|
||||
from app.models.company import Company
|
||||
from app.models.detected_change import DetectedChange
|
||||
from app.models.enums import ChangeType, SourceType
|
||||
from app.models.snapshot import Snapshot
|
||||
from app.models.source import Source
|
||||
from app.repositories.detected_change_repository import DetectedChangeRepository
|
||||
from app.repositories.source_repository import SnapshotRepository
|
||||
|
||||
_COOLDOWN = timedelta(hours=24)
|
||||
_MIN_CONTENT_DIFF_RATIO = 0.05
|
||||
|
||||
_EXTRACTION_CONFIDENCE: dict[ChangeType, float] = {
|
||||
ChangeType.LEADERSHIP_CHANGE: 0.6,
|
||||
ChangeType.FILING_NEW: 0.95,
|
||||
ChangeType.PRICE_CHANGE: 0.6,
|
||||
ChangeType.NEW_DOCUMENT: 0.85,
|
||||
ChangeType.REMOVED_DOCUMENT: 0.85,
|
||||
ChangeType.CONTENT_MODIFIED: 0.7,
|
||||
}
|
||||
|
||||
|
||||
async def detect_change_for_source(
|
||||
db: AsyncSession,
|
||||
source: Source,
|
||||
company: Company,
|
||||
current_snapshot: Snapshot,
|
||||
monitoring_run_id: uuid.UUID,
|
||||
) -> DetectedChange | None:
|
||||
snapshot_repo = SnapshotRepository(db)
|
||||
previous = await snapshot_repo.get_previous(source.id, current_snapshot)
|
||||
if previous is None:
|
||||
return None # Baseline snapshot - nothing to compare against yet.
|
||||
|
||||
# Layer 1: exact hash comparison.
|
||||
if previous.hash == current_snapshot.hash:
|
||||
return None
|
||||
|
||||
structured_diff = diff_item_sets(
|
||||
previous.structured_summary.get("urls", []),
|
||||
current_snapshot.structured_summary.get("urls", []),
|
||||
)
|
||||
text_diff = bounded_text_diff(previous.text_summary or "", current_snapshot.text_summary or "")
|
||||
|
||||
change_type = _classify_change(source, structured_diff, text_diff)
|
||||
if change_type is None:
|
||||
return None # The hash differed, but only noise (Layer 3 already strips it).
|
||||
|
||||
# Always drawn from the actual new text (not the URL/title evidence used
|
||||
# for display) - that's what's meaningful to compare against the user's
|
||||
# stated focus, regardless of which change_type it triggered.
|
||||
focus_evidence_text = "\n".join(text_diff.added_lines)
|
||||
|
||||
raw_diff = {
|
||||
"structured_added": structured_diff.added,
|
||||
"structured_removed": structured_diff.removed,
|
||||
"text_diff_ratio": text_diff.diff_ratio,
|
||||
"text_added_lines": text_diff.added_lines,
|
||||
"text_removed_lines": text_diff.removed_lines,
|
||||
}
|
||||
|
||||
change_repo = DetectedChangeRepository(db)
|
||||
since = datetime.now(UTC) - _COOLDOWN
|
||||
recent = await change_repo.get_recent_for_source_and_type(source.id, change_type, since)
|
||||
if recent is not None and recent.raw_diff == raw_diff:
|
||||
return None # Exact repeat within the cooldown window - nothing new to report.
|
||||
is_repeat = recent is not None
|
||||
|
||||
significance = compute_significance(
|
||||
change_type=change_type,
|
||||
source_trust_score=source.trust_score,
|
||||
independent_source_count=1,
|
||||
focus_match=_matches_focus(company.monitoring_focus, focus_evidence_text),
|
||||
is_repeat=is_repeat,
|
||||
diff_ratio=text_diff.diff_ratio if change_type is ChangeType.CONTENT_MODIFIED else None,
|
||||
)
|
||||
confidence = compute_confidence(
|
||||
extraction_confidence=_EXTRACTION_CONFIDENCE[change_type],
|
||||
source_trust_score=source.trust_score,
|
||||
independent_source_count=1,
|
||||
)
|
||||
severity = classify_severity(significance, confidence)
|
||||
|
||||
change = await change_repo.create(
|
||||
company_id=company.id,
|
||||
source_id=source.id,
|
||||
monitoring_run_id=monitoring_run_id,
|
||||
previous_snapshot_id=previous.id,
|
||||
current_snapshot_id=current_snapshot.id,
|
||||
change_type=change_type,
|
||||
raw_diff=raw_diff,
|
||||
significance_score=significance,
|
||||
confidence_score=confidence,
|
||||
severity=severity,
|
||||
summary=_build_summary(change_type, structured_diff, text_diff),
|
||||
)
|
||||
await db.commit()
|
||||
return change
|
||||
|
||||
|
||||
def _classify_change(
|
||||
source: Source, structured_diff: StructuredDiff, text_diff: TextDiffResult
|
||||
) -> ChangeType | None:
|
||||
added_text = "\n".join(text_diff.added_lines)
|
||||
|
||||
if mentions_leadership_title(added_text):
|
||||
return ChangeType.LEADERSHIP_CHANGE
|
||||
|
||||
if source.source_type is SourceType.SEC_EDGAR and structured_diff.added:
|
||||
return ChangeType.FILING_NEW
|
||||
|
||||
price_delta = extract_prices(added_text) | extract_prices("\n".join(text_diff.removed_lines))
|
||||
if price_delta:
|
||||
return ChangeType.PRICE_CHANGE
|
||||
|
||||
if structured_diff.added:
|
||||
return ChangeType.NEW_DOCUMENT
|
||||
|
||||
if structured_diff.removed:
|
||||
return ChangeType.REMOVED_DOCUMENT
|
||||
|
||||
if text_diff.diff_ratio >= _MIN_CONTENT_DIFF_RATIO:
|
||||
return ChangeType.CONTENT_MODIFIED
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _matches_focus(monitoring_focus: str | None, evidence_text: str) -> bool:
|
||||
if not monitoring_focus:
|
||||
return False
|
||||
focus_words = {w.lower() for w in re.findall(r"[a-zA-Z]{5,}", monitoring_focus)}
|
||||
evidence_words = {w.lower() for w in re.findall(r"[a-zA-Z]{5,}", evidence_text)}
|
||||
return bool(focus_words & evidence_words)
|
||||
|
||||
|
||||
def _build_summary(
|
||||
change_type: ChangeType, structured_diff: StructuredDiff, text_diff: TextDiffResult
|
||||
) -> str:
|
||||
if change_type is ChangeType.NEW_DOCUMENT:
|
||||
n = len(structured_diff.added)
|
||||
return f"{n} new item{'s' if n != 1 else ''} detected"
|
||||
if change_type is ChangeType.REMOVED_DOCUMENT:
|
||||
n = len(structured_diff.removed)
|
||||
return f"{n} item{'s' if n != 1 else ''} removed"
|
||||
if change_type is ChangeType.PRICE_CHANGE:
|
||||
return "Pricing information changed"
|
||||
if change_type is ChangeType.LEADERSHIP_CHANGE:
|
||||
return "Possible leadership change mentioned"
|
||||
if change_type is ChangeType.FILING_NEW:
|
||||
n = len(structured_diff.added)
|
||||
return f"{n} new regulatory filing{'s' if n != 1 else ''}"
|
||||
return f"Content changed ({text_diff.diff_ratio:.0%} different)"
|
||||
@@ -0,0 +1,198 @@
|
||||
"""Orchestrates collectors against the database: turns `DiscoveredSource`s
|
||||
into `Source` rows, and a collector's `CollectionResult` into persisted
|
||||
`SourceDocument` + `Snapshot` rows. Collectors themselves stay
|
||||
database-free (see collectors/base.py) so they're trivially unit-testable;
|
||||
this module is the seam where that plain-dataclass world meets the ORM.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.collectors.base import CollectionResult, CompanyContext, SourceConfig
|
||||
from app.collectors.registry import get_collector
|
||||
from app.core.config import Settings
|
||||
from app.core.logging import get_logger
|
||||
from app.models.company import Company
|
||||
from app.models.enums import EnrichmentStatus, SourceStatus, SourceType
|
||||
from app.models.source import Source
|
||||
from app.repositories.source_repository import (
|
||||
SnapshotRepository,
|
||||
SourceDocumentRepository,
|
||||
SourceRepository,
|
||||
)
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# Collector types with real (non-fixture) auto-discovery.
|
||||
_DISCOVERABLE_TYPES = (
|
||||
SourceType.WEBSITE,
|
||||
SourceType.GITHUB,
|
||||
SourceType.SEC_EDGAR,
|
||||
SourceType.JOB_POSTING,
|
||||
SourceType.RSS,
|
||||
SourceType.GOV_CONTRACT,
|
||||
SourceType.PATENT,
|
||||
)
|
||||
|
||||
|
||||
def _leadership_names(company: Company) -> list[str]:
|
||||
enrichment = company.enrichment
|
||||
if enrichment is None or enrichment.status == EnrichmentStatus.FAILED:
|
||||
return []
|
||||
return [
|
||||
member["name"]
|
||||
for member in enrichment.data.get("leadership_team", []) or []
|
||||
if member.get("name")
|
||||
]
|
||||
|
||||
|
||||
def to_company_context(company: Company, settings: Settings | None = None) -> CompanyContext:
|
||||
return CompanyContext(
|
||||
id=str(company.id),
|
||||
name=company.name,
|
||||
official_website=company.official_website,
|
||||
monitoring_focus=company.monitoring_focus,
|
||||
aliases=[a.alias for a in company.aliases],
|
||||
competitors=[c.name for c in company.competitors],
|
||||
leadership_names=_leadership_names(company),
|
||||
uspto_api_key=settings.uspto_api_key if settings is not None else None,
|
||||
)
|
||||
|
||||
|
||||
def _to_source_config(source: Source) -> SourceConfig:
|
||||
return SourceConfig(
|
||||
id=str(source.id),
|
||||
source_type=source.source_type,
|
||||
name=source.name,
|
||||
base_url=source.base_url,
|
||||
configuration_metadata=source.configuration_metadata or {},
|
||||
)
|
||||
|
||||
|
||||
async def discover_sources_for_company(
|
||||
db: AsyncSession, company: Company, settings: Settings | None = None
|
||||
) -> list[Source]:
|
||||
"""Runs discovery for every collector type capable of it and creates a
|
||||
`Source` row per suggestion, skipping ones that already exist for this
|
||||
company (same type + base_url)."""
|
||||
repo = SourceRepository(db)
|
||||
existing = await repo.list_for_company(company.id)
|
||||
existing_keys = {(s.source_type, s.base_url) for s in existing}
|
||||
|
||||
context = to_company_context(company, settings)
|
||||
created: list[Source] = []
|
||||
|
||||
for source_type in _DISCOVERABLE_TYPES:
|
||||
collector = get_collector(source_type)
|
||||
try:
|
||||
discovered = await collector.discover(context)
|
||||
except Exception as exc: # pragma: no cover - defensive, discovery is best-effort
|
||||
logger.warning("source_discovery_failed", source_type=source_type, error=str(exc))
|
||||
continue
|
||||
|
||||
for candidate in discovered:
|
||||
key = (candidate.source_type, candidate.base_url)
|
||||
if key in existing_keys:
|
||||
continue
|
||||
source = await repo.create(
|
||||
company_id=company.id,
|
||||
source_type=candidate.source_type,
|
||||
name=candidate.name,
|
||||
base_url=candidate.base_url,
|
||||
configuration_metadata=candidate.configuration_metadata,
|
||||
)
|
||||
existing_keys.add(key)
|
||||
created.append(source)
|
||||
|
||||
return created
|
||||
|
||||
|
||||
def _summarize_documents(documents) -> dict:
|
||||
return {
|
||||
"document_count": len(documents),
|
||||
"titles": [d.title for d in documents if d.title][:50],
|
||||
"urls": [d.url for d in documents][:50],
|
||||
"content_hashes": [d.content_hash for d in documents][:50],
|
||||
}
|
||||
|
||||
|
||||
def _build_text_summary(documents) -> str:
|
||||
"""Concatenated per-document excerpts used for bounded text diffing
|
||||
(change_detection's text-diff layer) - a title-only summary is too thin
|
||||
to catch wording-level changes within a page."""
|
||||
parts = []
|
||||
for doc in documents[:8]:
|
||||
title = doc.title or doc.url
|
||||
excerpt = doc.content_text[:600]
|
||||
parts.append(f"### {title}\n{excerpt}")
|
||||
return "\n\n".join(parts)
|
||||
|
||||
|
||||
async def collect_source(
|
||||
db: AsyncSession,
|
||||
settings: Settings,
|
||||
source: Source,
|
||||
company: Company,
|
||||
*,
|
||||
monitoring_run_id: uuid.UUID | None = None,
|
||||
) -> CollectionResult:
|
||||
"""Runs one source's collector, persists new documents (deduped by
|
||||
content hash within the source), writes a Snapshot summarizing the
|
||||
batch, and updates the Source's health/status fields."""
|
||||
collector = get_collector(source.source_type)
|
||||
context = to_company_context(company, settings)
|
||||
config = _to_source_config(source)
|
||||
|
||||
result = await collector.collect(config, context)
|
||||
|
||||
doc_repo = SourceDocumentRepository(db)
|
||||
persisted_hashes: list[str] = []
|
||||
for doc in result.documents:
|
||||
if await doc_repo.exists_with_hash(source.id, doc.content_hash):
|
||||
continue
|
||||
await doc_repo.create(
|
||||
source_id=source.id,
|
||||
company_id=company.id,
|
||||
url=doc.url,
|
||||
canonical_url=doc.canonical_url,
|
||||
title=doc.title,
|
||||
author=doc.author,
|
||||
publication_date=doc.publication_date,
|
||||
retrieved_date=doc.retrieved_date,
|
||||
content_text=doc.content_text,
|
||||
content_hash=doc.content_hash,
|
||||
metadata_json=doc.metadata,
|
||||
language=doc.language,
|
||||
http_status=doc.http_status,
|
||||
extraction_method=doc.extraction_method,
|
||||
trust_score=doc.trust_score,
|
||||
)
|
||||
persisted_hashes.append(doc.content_hash)
|
||||
|
||||
now = datetime.now(UTC)
|
||||
if result.documents:
|
||||
batch_hash = hashlib.sha256(
|
||||
"|".join(sorted(d.content_hash for d in result.documents)).encode("utf-8")
|
||||
).hexdigest()
|
||||
snapshot_repo = SnapshotRepository(db)
|
||||
await snapshot_repo.create(
|
||||
company_id=company.id,
|
||||
source_id=source.id,
|
||||
snapshot_type=source.source_type.value,
|
||||
hash=batch_hash,
|
||||
structured_summary=_summarize_documents(result.documents),
|
||||
text_summary=_build_text_summary(result.documents),
|
||||
monitoring_run_id=monitoring_run_id,
|
||||
)
|
||||
|
||||
source_repo = SourceRepository(db)
|
||||
success = result.status in (SourceStatus.ACTIVE,)
|
||||
await source_repo.mark_checked(source, status=result.status, checked_at=now, success=success)
|
||||
|
||||
await db.commit()
|
||||
return result
|
||||
@@ -0,0 +1,243 @@
|
||||
"""Company + monitor configuration business logic.
|
||||
|
||||
Every lookup here is scoped to `user_id` at the query level (see
|
||||
CompanyRepository), so a company that exists but belongs to another user
|
||||
raises NotFoundError exactly like one that doesn't exist - this avoids
|
||||
leaking existence via a 403-vs-404 timing/response difference.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import ConflictError, NotFoundError
|
||||
from app.core.text import slugify
|
||||
from app.models.company import Company
|
||||
from app.models.enums import CompanyStatus, EnrichmentStatus
|
||||
from app.models.monitor_configuration import MonitorConfiguration
|
||||
from app.repositories.company_enrichment_repository import CompanyEnrichmentRepository
|
||||
from app.repositories.company_repository import CompanyRepository, MonitorConfigurationRepository
|
||||
from app.repositories.notification_destination_repository import (
|
||||
NotificationDestinationRepository,
|
||||
)
|
||||
from app.schemas.company import CompanyCreate, CompanyUpdate, MonitorConfigurationUpdate
|
||||
from app.services.scheduling import validate_and_compute_next_run
|
||||
|
||||
|
||||
async def _unique_slug(repo: CompanyRepository, user_id: uuid.UUID, name: str) -> str:
|
||||
base = slugify(name)
|
||||
slug = base
|
||||
suffix = 1
|
||||
while await repo.slug_exists_for_user(user_id, slug):
|
||||
suffix += 1
|
||||
slug = f"{base}-{suffix}"
|
||||
return slug
|
||||
|
||||
|
||||
async def _unique_display_name(repo: CompanyRepository, user_id: uuid.UUID, name: str) -> str:
|
||||
"""Guarantees the created row's name is unique for this user, the same
|
||||
way a filesystem silently renames a colliding file - "Stripe" stays
|
||||
"Stripe" unless the user already has one, in which case this becomes
|
||||
"Stripe (2)", "Stripe (3)", etc. The wizard warns about likely
|
||||
duplicates *before* this ever runs (see the frontend's own near-duplicate
|
||||
check) so this is a last-resort guarantee, not the primary UX."""
|
||||
if not await repo.name_exists_for_user(user_id, name):
|
||||
return name
|
||||
suffix = 2
|
||||
while await repo.name_exists_for_user(user_id, f"{name} ({suffix})"):
|
||||
suffix += 1
|
||||
return f"{name} ({suffix})"
|
||||
|
||||
|
||||
async def list_companies(db: AsyncSession, user_id: uuid.UUID) -> list[Company]:
|
||||
repo = CompanyRepository(db)
|
||||
return await repo.list_for_user(user_id)
|
||||
|
||||
|
||||
async def get_company(db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID) -> Company:
|
||||
repo = CompanyRepository(db)
|
||||
company = await repo.get_for_user(company_id, user_id)
|
||||
if company is None:
|
||||
raise NotFoundError("Company not found")
|
||||
return company
|
||||
|
||||
|
||||
async def create_company(
|
||||
db: AsyncSession, settings: Settings, user_id: uuid.UUID, payload: CompanyCreate
|
||||
) -> Company:
|
||||
company_repo = CompanyRepository(db)
|
||||
monitor_repo = MonitorConfigurationRepository(db)
|
||||
|
||||
existing_count = await company_repo.count_for_user(user_id)
|
||||
if existing_count >= settings.max_companies_per_user:
|
||||
raise ConflictError(
|
||||
f"You've reached the maximum of {settings.max_companies_per_user} monitored companies"
|
||||
)
|
||||
|
||||
name = await _unique_display_name(company_repo, user_id, payload.name)
|
||||
slug = await _unique_slug(company_repo, user_id, payload.name)
|
||||
|
||||
next_run = validate_and_compute_next_run(
|
||||
frequency_type=payload.frequency_type,
|
||||
interval_minutes=payload.interval_minutes,
|
||||
cron_expression=payload.cron_expression,
|
||||
tz_name=payload.timezone,
|
||||
minimum_interval_minutes=settings.minimum_monitoring_interval_minutes,
|
||||
)
|
||||
|
||||
company = await company_repo.create(
|
||||
user_id=user_id,
|
||||
name=name,
|
||||
slug=slug,
|
||||
official_website=payload.official_website,
|
||||
description=payload.description,
|
||||
monitoring_focus=payload.monitoring_focus,
|
||||
industry=payload.industry,
|
||||
country=payload.country,
|
||||
region=payload.region,
|
||||
headquarters=payload.headquarters,
|
||||
public_identifiers=payload.public_identifiers,
|
||||
alias_names=payload.alias_names,
|
||||
competitor_names=payload.competitor_names,
|
||||
)
|
||||
await monitor_repo.create_default(
|
||||
company_id=company.id,
|
||||
frequency_type=payload.frequency_type,
|
||||
interval_minutes=payload.interval_minutes,
|
||||
cron_expression=payload.cron_expression,
|
||||
timezone=payload.timezone,
|
||||
severity_threshold=payload.severity_threshold,
|
||||
next_run=next_run,
|
||||
)
|
||||
if settings.ninjapear_api_key:
|
||||
# A PENDING row is created up front (not just enqueued) so the
|
||||
# frontend has something real to poll on - otherwise "not yet
|
||||
# enriched" and "never configured" would look identical (both
|
||||
# `enrichment: null`). enrichment_service.enrich_company updates
|
||||
# this same row in place once the task finishes.
|
||||
await CompanyEnrichmentRepository(db).upsert(
|
||||
company.id,
|
||||
status=EnrichmentStatus.PENDING,
|
||||
data={},
|
||||
errors={},
|
||||
credits_spent=None,
|
||||
fetched_at=None,
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
if settings.ninjapear_api_key:
|
||||
# Fire-and-forget, onboarding-only enrichment - gated on the key
|
||||
# itself (not just falling through to a mock provider) so a user
|
||||
# who never configured NinjaPear gets zero extra background-task
|
||||
# volume. See app/services/enrichment_service.py.
|
||||
from app.tasks.enrichment import (
|
||||
enrich_company, # local import: keeps Celery out of API startup path
|
||||
)
|
||||
|
||||
enrich_company.delay(str(company.id))
|
||||
|
||||
refreshed = await company_repo.get_for_user(company.id, user_id)
|
||||
assert refreshed is not None
|
||||
return refreshed
|
||||
|
||||
|
||||
async def update_company(
|
||||
db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID, payload: CompanyUpdate
|
||||
) -> Company:
|
||||
company_repo = CompanyRepository(db)
|
||||
company = await get_company(db, user_id, company_id)
|
||||
|
||||
updates = payload.model_dump(exclude_unset=True, exclude={"alias_names", "competitor_names"})
|
||||
for field, value in updates.items():
|
||||
setattr(company, field, value)
|
||||
|
||||
if payload.alias_names is not None:
|
||||
await company_repo.replace_aliases(company, payload.alias_names)
|
||||
if payload.competitor_names is not None:
|
||||
await company_repo.replace_competitors(company, payload.competitor_names)
|
||||
|
||||
await db.commit()
|
||||
refreshed = await company_repo.get_for_user(company_id, user_id)
|
||||
assert refreshed is not None
|
||||
return refreshed
|
||||
|
||||
|
||||
async def delete_company(db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID) -> None:
|
||||
company_repo = CompanyRepository(db)
|
||||
company = await get_company(db, user_id, company_id)
|
||||
await company_repo.delete(company)
|
||||
await db.commit()
|
||||
|
||||
# Garbage-collect any notification destination that was only ever
|
||||
# linked to this now-deleted company - a destination with zero company
|
||||
# links left behind is dead weight, not a valid "applies to nothing"
|
||||
# state (see NotificationDestinationRepository.delete_orphaned_for_user).
|
||||
await NotificationDestinationRepository(db).delete_orphaned_for_user(user_id)
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def _set_company_status(
|
||||
db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID, status: CompanyStatus
|
||||
) -> Company:
|
||||
company = await get_company(db, user_id, company_id)
|
||||
company.status = status
|
||||
if company.monitor_configuration is not None:
|
||||
company.monitor_configuration.enabled = status == CompanyStatus.ACTIVE
|
||||
await db.commit()
|
||||
company_repo = CompanyRepository(db)
|
||||
refreshed = await company_repo.get_for_user(company_id, user_id)
|
||||
assert refreshed is not None
|
||||
return refreshed
|
||||
|
||||
|
||||
async def pause_company(db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID) -> Company:
|
||||
return await _set_company_status(db, user_id, company_id, CompanyStatus.PAUSED)
|
||||
|
||||
|
||||
async def resume_company(db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID) -> Company:
|
||||
return await _set_company_status(db, user_id, company_id, CompanyStatus.ACTIVE)
|
||||
|
||||
|
||||
async def update_monitor_configuration(
|
||||
db: AsyncSession,
|
||||
settings: Settings,
|
||||
user_id: uuid.UUID,
|
||||
company_id: uuid.UUID,
|
||||
payload: MonitorConfigurationUpdate,
|
||||
) -> MonitorConfiguration:
|
||||
company = await get_company(db, user_id, company_id)
|
||||
config = company.monitor_configuration
|
||||
if config is None:
|
||||
raise NotFoundError("Monitor configuration not found")
|
||||
|
||||
updates = payload.model_dump(exclude_unset=True)
|
||||
frequency_type = updates.get("frequency_type", config.frequency_type)
|
||||
interval_minutes = updates.get("interval_minutes", config.interval_minutes)
|
||||
cron_expression = updates.get("cron_expression", config.cron_expression)
|
||||
tz_name = updates.get("timezone", config.timezone)
|
||||
|
||||
schedule_changed = any(
|
||||
key in updates
|
||||
for key in ("frequency_type", "interval_minutes", "cron_expression", "timezone")
|
||||
)
|
||||
if schedule_changed:
|
||||
config.next_run = validate_and_compute_next_run(
|
||||
frequency_type=frequency_type,
|
||||
interval_minutes=interval_minutes,
|
||||
cron_expression=cron_expression,
|
||||
tz_name=tz_name,
|
||||
minimum_interval_minutes=settings.minimum_monitoring_interval_minutes,
|
||||
)
|
||||
|
||||
for field, value in updates.items():
|
||||
setattr(config, field, value)
|
||||
|
||||
if "enabled" in updates:
|
||||
company.status = CompanyStatus.ACTIVE if updates["enabled"] else CompanyStatus.PAUSED
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(config)
|
||||
return config
|
||||
@@ -0,0 +1,190 @@
|
||||
"""Company-metadata discovery: given just a name (plus optional user
|
||||
hints), proposes official_website/industry/country/region/headquarters/
|
||||
aliases/competitors/public_identifiers and a preview of sources the
|
||||
pipeline would start monitoring - all before anything is persisted. This
|
||||
is the "System performs company discovery" step between "user enters a
|
||||
name" and "user confirms/edits" in the onboarding flow.
|
||||
|
||||
Two independent, deliberately-separated concerns per the architecture
|
||||
direction:
|
||||
- SearchProvider answers "where should we look" (this module's job).
|
||||
- The LLM only ever analyzes evidence this module already gathered - see
|
||||
app/prompts/company_profile.py's docstring. It is never asked to recall
|
||||
facts about the company from its own training data.
|
||||
|
||||
Runs exactly once, at onboarding time, driven by an explicit user action
|
||||
(the wizard's "Discover" step) - never re-triggered by scheduled monitoring
|
||||
runs. Source *persistence* still happens exactly as it already did before
|
||||
this module existed: lazily, on the company's first monitoring run (see
|
||||
tasks/collection.py). This module only ever previews what that step would
|
||||
find, via the same collector.discover() calls, without writing anything.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.analysis.llm.base import LLMProvider
|
||||
from app.collectors.base import CompanyContext
|
||||
from app.collectors.extraction import extract_readable_text
|
||||
from app.collectors.registry import get_collector
|
||||
from app.collectors.robots import is_allowed
|
||||
from app.core.config import Settings
|
||||
from app.core.http import FetchError, SsrfBlockedError, fetch_with_retries
|
||||
from app.core.logging import get_logger
|
||||
from app.models.enums import SourceType
|
||||
from app.prompts.company_profile import extract_company_profile
|
||||
from app.schemas.discovery import DiscoveredCompanyProfile, PotentialSource
|
||||
from app.search.base import SearchProvider
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# A "{name} official website" search frequently ranks a reference/social
|
||||
# page above the company's own domain for well-known companies (observed
|
||||
# live: Brave's top result for "Stripe official website" was Stripe's
|
||||
# Wikipedia article, not stripe.com). Picking that as `official_website`
|
||||
# then feeds a wrong base domain into every downstream source-preview
|
||||
# collector. Skip these hosts when a better-ranked alternative exists in
|
||||
# the same result set, rather than blindly taking the top hit.
|
||||
_NON_CORPORATE_HOSTS = (
|
||||
"wikipedia.org",
|
||||
"linkedin.com",
|
||||
"crunchbase.com",
|
||||
"bloomberg.com",
|
||||
"facebook.com",
|
||||
"twitter.com",
|
||||
"x.com",
|
||||
"youtube.com",
|
||||
"reddit.com",
|
||||
"glassdoor.com",
|
||||
)
|
||||
|
||||
|
||||
def _is_non_corporate_host(url: str) -> bool:
|
||||
host = url.split("//", 1)[-1].split("/", 1)[0].lower()
|
||||
return any(host == d or host.endswith(f".{d}") for d in _NON_CORPORATE_HOSTS)
|
||||
|
||||
|
||||
# Same set collection_service.py's discover_sources_for_company already
|
||||
# discovers from for a real company - kept in sync deliberately, not
|
||||
# imported, since this module previews without a persisted Company/Source
|
||||
# and the coupling would only make both harder to read.
|
||||
_PREVIEWABLE_TYPES = (
|
||||
SourceType.WEBSITE,
|
||||
SourceType.GITHUB,
|
||||
SourceType.SEC_EDGAR,
|
||||
SourceType.JOB_POSTING,
|
||||
SourceType.RSS,
|
||||
SourceType.GOV_CONTRACT,
|
||||
SourceType.PATENT,
|
||||
)
|
||||
|
||||
|
||||
async def _resolve_official_website(
|
||||
search: SearchProvider, name: str, hint: str | None
|
||||
) -> tuple[str | None, list[str]]:
|
||||
if hint:
|
||||
return hint, []
|
||||
results = await search.search(f"{name} official website", count=3)
|
||||
if not results:
|
||||
return None, []
|
||||
best = next((r for r in results if not _is_non_corporate_host(r.url)), results[0])
|
||||
return best.url, [best.url]
|
||||
|
||||
|
||||
async def _fetch_homepage_text(
|
||||
settings: Settings, official_website: str | None
|
||||
) -> tuple[str | None, list[str]]:
|
||||
if not official_website:
|
||||
return None, []
|
||||
try:
|
||||
if not await is_allowed(official_website, settings=settings):
|
||||
return None, []
|
||||
result = await fetch_with_retries(official_website, settings=settings, max_attempts=1)
|
||||
except (FetchError, SsrfBlockedError) as exc:
|
||||
logger.info("discovery_homepage_fetch_failed", url=official_website, error=str(exc))
|
||||
return None, []
|
||||
|
||||
if result.status_code >= 400:
|
||||
return None, []
|
||||
|
||||
text, _method = extract_readable_text(result.text, official_website)
|
||||
return (text or None), [official_website]
|
||||
|
||||
|
||||
async def _preview_sources(
|
||||
context: CompanyContext,
|
||||
) -> list[PotentialSource]:
|
||||
previews: list[PotentialSource] = []
|
||||
for source_type in _PREVIEWABLE_TYPES:
|
||||
collector = get_collector(source_type)
|
||||
try:
|
||||
discovered = await collector.discover(context)
|
||||
except Exception as exc: # pragma: no cover - defensive, preview is best-effort
|
||||
logger.warning(
|
||||
"discovery_source_preview_failed", source_type=source_type, error=str(exc)
|
||||
)
|
||||
continue
|
||||
previews.extend(
|
||||
PotentialSource(source_type=d.source_type, name=d.name, base_url=d.base_url)
|
||||
for d in discovered
|
||||
)
|
||||
return previews
|
||||
|
||||
|
||||
async def discover_company_profile(
|
||||
search: SearchProvider,
|
||||
llm: LLMProvider,
|
||||
settings: Settings,
|
||||
*,
|
||||
name: str,
|
||||
official_website: str | None,
|
||||
monitoring_focus: str | None,
|
||||
competitor_names: list[str],
|
||||
alias_names: list[str],
|
||||
) -> DiscoveredCompanyProfile:
|
||||
resolved_website, consulted_website = await _resolve_official_website(
|
||||
search, name, official_website
|
||||
)
|
||||
homepage_text, consulted_homepage = await _fetch_homepage_text(settings, resolved_website)
|
||||
|
||||
search_results = []
|
||||
consulted_queries: list[str] = []
|
||||
for query in (f"{name} headquarters", f"{name} competitors", f"{name} formerly known as"):
|
||||
results = await search.search(query, count=3)
|
||||
search_results.extend({"query": query, **r.model_dump()} for r in results)
|
||||
consulted_queries.append(query)
|
||||
|
||||
extraction = await extract_company_profile(
|
||||
llm,
|
||||
company_name=name,
|
||||
homepage_url=resolved_website,
|
||||
homepage_text=homepage_text,
|
||||
search_results=search_results,
|
||||
)
|
||||
|
||||
context = CompanyContext(
|
||||
id="pending",
|
||||
name=name,
|
||||
official_website=resolved_website,
|
||||
monitoring_focus=monitoring_focus,
|
||||
uspto_api_key=settings.uspto_api_key,
|
||||
)
|
||||
# Individual collectors already handle a missing official_website
|
||||
# gracefully (e.g. JobPostingCollector.discover returns [] rather than
|
||||
# raising) - GitHub/SEC EDGAR search by name and don't need one at all.
|
||||
potential_sources = await _preview_sources(context)
|
||||
|
||||
return DiscoveredCompanyProfile(
|
||||
name=name,
|
||||
official_website=resolved_website,
|
||||
description=extraction.description,
|
||||
monitoring_focus=monitoring_focus,
|
||||
industry=extraction.industry,
|
||||
country=extraction.country,
|
||||
region=extraction.region,
|
||||
headquarters=extraction.headquarters,
|
||||
aliases=alias_names or extraction.aliases,
|
||||
competitors=competitor_names or extraction.competitors,
|
||||
public_identifiers={pi.key: pi.value for pi in extraction.public_identifiers},
|
||||
potential_sources=potential_sources,
|
||||
sources_consulted=[*consulted_website, *consulted_homepage, *consulted_queries],
|
||||
)
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Onboarding-time company enrichment via a paid third-party provider
|
||||
(NinjaPear/nubela.co) - fires exactly once per company, never on a
|
||||
recurring schedule (see app/tasks/enrichment.py and
|
||||
company_service.create_company, which gates the enqueue itself on
|
||||
NINJAPEAR_API_KEY being set).
|
||||
|
||||
Orchestrates several independent, per-endpoint provider calls; one bad
|
||||
call must never sink the others (same principle as tasks/collection.py's
|
||||
per-source loop) - every failure is recorded in `errors` rather than
|
||||
silently dropped or allowed to fail the whole enrichment. Person-level
|
||||
lookups (work email, profile) are capped at
|
||||
`settings.ninjapear_max_leadership_lookups` to bound the fan-out from a
|
||||
large leadership team.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.logging import get_logger
|
||||
from app.enrichment.base import EnrichmentProvider
|
||||
from app.models.company import Company
|
||||
from app.models.company_enrichment import CompanyEnrichment
|
||||
from app.models.enums import EnrichmentStatus
|
||||
from app.repositories.company_enrichment_repository import CompanyEnrichmentRepository
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# Approximate credit cost per call per NinjaPear's published pricing -
|
||||
# tracked for user-facing cost transparency (Settings page) only, not
|
||||
# billed or enforced by this app.
|
||||
_CREDIT_COSTS = {
|
||||
"details": 5, # 3 base + 2 for the employee-count add-on
|
||||
"funding": 3, # 2 base + a rough per-investor estimate
|
||||
"updates": 2,
|
||||
"competitors": 5, # NinjaPear's documented minimum per request
|
||||
"products": 3,
|
||||
"customers": 3, # 1 base + a rough per-company estimate
|
||||
"work_email": 2,
|
||||
"person_profile": 3,
|
||||
}
|
||||
_PER_COMPANY_SECTIONS = ("details", "funding", "updates", "competitors", "products", "customers")
|
||||
_PER_LEADER_SECTIONS = ("work_email", "person_profile")
|
||||
|
||||
|
||||
def estimate_max_credits_per_company(max_leadership_lookups: int) -> int:
|
||||
"""Worst-case credit ceiling for one company's enrichment - every
|
||||
company-level section succeeding plus every leadership-lookup slot
|
||||
used. Shown to the user before they commit to creating a company (see
|
||||
the add-company wizard's Confirm step) so the real cost is never a
|
||||
surprise."""
|
||||
base = sum(_CREDIT_COSTS[section] for section in _PER_COMPANY_SECTIONS)
|
||||
per_leader = sum(_CREDIT_COSTS[section] for section in _PER_LEADER_SECTIONS)
|
||||
return base + per_leader * max_leadership_lookups
|
||||
|
||||
|
||||
async def enrich_company(
|
||||
db: AsyncSession, settings: Settings, provider: EnrichmentProvider, company: Company
|
||||
) -> CompanyEnrichment:
|
||||
website = company.official_website
|
||||
if not website:
|
||||
# NinjaPear identifies a company by website only - every call would
|
||||
# fail the same way, so skip straight to FAILED instead of burning
|
||||
# credits on N doomed requests.
|
||||
repo = CompanyEnrichmentRepository(db)
|
||||
enrichment = await repo.upsert(
|
||||
company.id,
|
||||
status=EnrichmentStatus.FAILED,
|
||||
data={},
|
||||
errors={"details": "No official_website on file - NinjaPear requires one"},
|
||||
credits_spent=0,
|
||||
fetched_at=datetime.now(UTC),
|
||||
)
|
||||
await db.commit()
|
||||
return enrichment
|
||||
|
||||
data: dict = {}
|
||||
errors: dict[str, str] = {}
|
||||
credits_spent = 0
|
||||
attempted = 0
|
||||
succeeded = 0
|
||||
|
||||
async def _run(section: str, coro):
|
||||
nonlocal credits_spent, attempted, succeeded
|
||||
attempted += 1
|
||||
try:
|
||||
result = await coro
|
||||
except Exception as exc: # noqa: BLE001 - one bad call must never sink the rest
|
||||
errors[section] = str(exc)
|
||||
logger.warning(
|
||||
"enrichment_section_failed",
|
||||
section=section,
|
||||
company_id=str(company.id),
|
||||
error=str(exc),
|
||||
)
|
||||
return None
|
||||
credits_spent += _CREDIT_COSTS.get(section, 0)
|
||||
succeeded += 1
|
||||
return result
|
||||
|
||||
details = await _run("details", provider.get_company_details(company.name, website))
|
||||
leadership_team: list[dict] = []
|
||||
if details is not None:
|
||||
data["employee_count"] = details.employee_count_range
|
||||
data["description"] = details.description
|
||||
data["industry"] = details.industry
|
||||
data["founded_year"] = details.founded_year
|
||||
data["specialties"] = details.specialties
|
||||
leadership_team = [m.model_dump() for m in details.leadership_team]
|
||||
|
||||
funding = await _run("funding", provider.get_funding(company.name, website))
|
||||
if funding is not None:
|
||||
data["funding"] = funding.model_dump()
|
||||
|
||||
updates = await _run("updates", provider.get_updates(company.name, website))
|
||||
if updates is not None:
|
||||
data["recent_updates"] = [u.model_dump() for u in updates]
|
||||
|
||||
competitors = await _run("competitors", provider.get_competitors(company.name, website))
|
||||
if competitors is not None:
|
||||
data["competitors"] = [c.model_dump() for c in competitors]
|
||||
|
||||
products = await _run("products", provider.get_products(company.name, website))
|
||||
if products is not None:
|
||||
data["products"] = [p.model_dump() for p in products]
|
||||
|
||||
customers = await _run("customers", provider.get_customers(company.name, website))
|
||||
if customers is not None:
|
||||
data["customers"] = [c.model_dump() for c in customers]
|
||||
|
||||
if leadership_team and website:
|
||||
cap = settings.ninjapear_max_leadership_lookups
|
||||
for member in leadership_team[:cap]:
|
||||
person_name = member.get("name")
|
||||
if not person_name:
|
||||
continue
|
||||
email = await _run("work_email", provider.get_work_email(person_name, website))
|
||||
if email:
|
||||
member["work_email"] = email
|
||||
profile = await _run(
|
||||
"person_profile", provider.get_person_profile(person_name, website)
|
||||
)
|
||||
if profile is not None:
|
||||
profile_url, bio = profile
|
||||
member["profile_url"] = member.get("profile_url") or profile_url
|
||||
member["bio"] = member.get("bio") or bio
|
||||
data["leadership_team"] = leadership_team
|
||||
|
||||
if attempted == 0 or succeeded == 0:
|
||||
status = EnrichmentStatus.FAILED
|
||||
elif succeeded == attempted:
|
||||
status = EnrichmentStatus.COMPLETE
|
||||
else:
|
||||
status = EnrichmentStatus.PARTIAL
|
||||
|
||||
repo = CompanyEnrichmentRepository(db)
|
||||
enrichment = await repo.upsert(
|
||||
company.id,
|
||||
status=status,
|
||||
data=data,
|
||||
errors=errors,
|
||||
credits_spent=credits_spent,
|
||||
fetched_at=datetime.now(UTC),
|
||||
)
|
||||
await db.commit()
|
||||
logger.info(
|
||||
"company_enrichment_finished",
|
||||
company_id=str(company.id),
|
||||
status=status.value,
|
||||
credits_spent=credits_spent,
|
||||
failed_sections=list(errors.keys()),
|
||||
)
|
||||
return enrichment
|
||||
@@ -0,0 +1,151 @@
|
||||
"""Generic, per-IP escalation engine shared by resend-verification,
|
||||
resend-password-reset, and failed-login throttling.
|
||||
|
||||
Two functions, not one, so login can enforce the gate *before* verifying a
|
||||
password (a correct password during a penalty window must still be
|
||||
rejected, or the delay is meaningless) without a check-then-act race:
|
||||
|
||||
- `peek_throttle` - read-only. Is this IP allowed to attempt `action` right
|
||||
now? Also lazily resets a completed timeout cycle (memory of the offense
|
||||
survives via `offense_count`; only the attempt stage resets - see
|
||||
`IpThrottleState`).
|
||||
- `record_attempt` - call only after the gated action actually happens (a
|
||||
failed login, or a resend that's being sent). Advances the stage/backoff,
|
||||
and escalates into a timeout (and eventually a permanent ban) once the
|
||||
stage array is exhausted.
|
||||
|
||||
All time comparisons go through `_now()` so tests can monkeypatch it
|
||||
directly to walk the whole escalation ladder in milliseconds of real time -
|
||||
no waiting, no manual brute-forcing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.base import ensure_aware_utc
|
||||
from app.models.enums import ThrottleAction
|
||||
from app.repositories.ip_throttle_repository import IpThrottleRepository
|
||||
|
||||
# 30s, 1min, 2min, 5min, 5min - shared identically by resend_verification and
|
||||
# resend_reset. The *first* code send (at registration / at a reset request)
|
||||
# also advances this, so "first resend allowed in 30s" is measured from that
|
||||
# original send, not from a first manual resend click.
|
||||
RESEND_BACKOFF_SECONDS: list[int] = [30, 60, 120, 300, 300]
|
||||
|
||||
# A stage's value is the wait *before the next* attempt (recording attempt K
|
||||
# sets the delay gating attempt K+1) - so 5 truly free attempts (1-5, no
|
||||
# wait before any of them) needs only 4 leading zeros (gating attempts
|
||||
# 2-5), then the 7 real delays gate attempts 6-12 (5s/15s/30s/60s/2min/5min/
|
||||
# 15min). Recording attempt 12 itself lands on stage_index=11, past the end
|
||||
# of this 11-entry array - exhausted, which is exactly the intended "12th
|
||||
# failure locks the account and starts the IP's timeout ladder" behavior.
|
||||
LOGIN_BACKOFF_SECONDS: list[int] = [0, 0, 0, 0, 5, 15, 30, 60, 120, 300, 900]
|
||||
|
||||
# 30min, 1h, 2h, 3h, 4h, 5h - indexed by offense_count. Exceeding this length
|
||||
# is a permanent ban.
|
||||
TIMEOUT_LADDER_SECONDS: list[int] = [1800, 3600, 7200, 10800, 14400, 18000]
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ThrottleResult:
|
||||
allowed: bool
|
||||
retry_after_seconds: int | None = None
|
||||
banned: bool = False
|
||||
|
||||
|
||||
async def is_banned(db: AsyncSession, ip_address: str) -> bool:
|
||||
"""Ban-only check, decoupled from any specific action's stage/timeout
|
||||
state - for call sites like register() that have no throttle action of
|
||||
their own but still must never let a banned IP through."""
|
||||
return await IpThrottleRepository(db).get_ban(ip_address) is not None
|
||||
|
||||
|
||||
async def peek_throttle(
|
||||
db: AsyncSession, ip_address: str, action: ThrottleAction
|
||||
) -> ThrottleResult:
|
||||
repo = IpThrottleRepository(db)
|
||||
|
||||
ban = await repo.get_ban(ip_address)
|
||||
if ban is not None:
|
||||
return ThrottleResult(allowed=False, banned=True)
|
||||
|
||||
state = await repo.get_or_create_state(ip_address, action)
|
||||
now = _now()
|
||||
|
||||
if state.timeout_until is not None:
|
||||
timeout_until = ensure_aware_utc(state.timeout_until)
|
||||
if timeout_until > now:
|
||||
return ThrottleResult(
|
||||
allowed=False, retry_after_seconds=int((timeout_until - now).total_seconds())
|
||||
)
|
||||
# Timeout has elapsed - cycle reset. offense_count (the memory of
|
||||
# this IP's history) is deliberately left untouched.
|
||||
state.timeout_until = None
|
||||
state.attempt_count = 0
|
||||
state.next_allowed_at = None
|
||||
await db.flush()
|
||||
|
||||
if state.next_allowed_at is not None:
|
||||
next_allowed_at = ensure_aware_utc(state.next_allowed_at)
|
||||
if next_allowed_at > now:
|
||||
return ThrottleResult(
|
||||
allowed=False, retry_after_seconds=int((next_allowed_at - now).total_seconds())
|
||||
)
|
||||
|
||||
return ThrottleResult(allowed=True)
|
||||
|
||||
|
||||
async def record_attempt(
|
||||
db: AsyncSession, ip_address: str, action: ThrottleAction, backoff_stages: list[int]
|
||||
) -> None:
|
||||
repo = IpThrottleRepository(db)
|
||||
state = await repo.get_or_create_state(ip_address, action)
|
||||
now = _now()
|
||||
|
||||
stage_index = state.attempt_count
|
||||
state.attempt_count += 1
|
||||
|
||||
if stage_index < len(backoff_stages):
|
||||
delay = backoff_stages[stage_index]
|
||||
state.next_allowed_at = now + timedelta(seconds=delay)
|
||||
await db.flush()
|
||||
return
|
||||
|
||||
# Backoff stages exhausted - enter a timeout, escalating in length with
|
||||
# each repeat offense.
|
||||
if state.offense_count < len(TIMEOUT_LADDER_SECONDS):
|
||||
duration = TIMEOUT_LADDER_SECONDS[state.offense_count]
|
||||
state.timeout_until = now + timedelta(seconds=duration)
|
||||
state.next_allowed_at = None
|
||||
state.offense_count += 1
|
||||
await db.flush()
|
||||
return
|
||||
|
||||
# Offended again after exhausting the entire timeout ladder - permanent.
|
||||
state.offense_count += 1
|
||||
state.timeout_until = None
|
||||
state.next_allowed_at = None
|
||||
await repo.create_ban(ip_address, reason=action.value, banned_at=now)
|
||||
await db.flush()
|
||||
|
||||
|
||||
async def reset_on_success(db: AsyncSession, ip_address: str, action: ThrottleAction) -> None:
|
||||
"""Login-only convenience: a correct login ends that specific attack
|
||||
scenario for this IP, so the stage/cooldown resets - but `offense_count`
|
||||
(this IP's history) is never cleared by a success, only by an admin
|
||||
unban."""
|
||||
repo = IpThrottleRepository(db)
|
||||
state = await repo.get_state(ip_address, action)
|
||||
if state is None:
|
||||
return
|
||||
state.attempt_count = 0
|
||||
state.next_allowed_at = None
|
||||
await db.flush()
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Run-now / run-history service. Enqueuing goes through Celery
|
||||
(`run_monitoring.delay`); this module only ever touches the `MonitoringRun`
|
||||
row and ownership checks - the actual collection work happens in the task
|
||||
(app/tasks/collection.py) and collection_service.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import NotFoundError, RateLimitedError
|
||||
from app.models.enums import MonitoringRunTrigger
|
||||
from app.models.monitoring_run import MonitoringRun
|
||||
from app.repositories.monitoring_run_repository import MonitoringRunRepository
|
||||
from app.services import company_service
|
||||
|
||||
|
||||
async def enqueue_run_now(
|
||||
db: AsyncSession, settings: Settings, user_id: uuid.UUID, company_id: uuid.UUID
|
||||
) -> MonitoringRun:
|
||||
company = await company_service.get_company(db, user_id, company_id)
|
||||
run_repo = MonitoringRunRepository(db)
|
||||
|
||||
# Idempotent: a company already mid-run returns that run rather than
|
||||
# queuing a duplicate (spec: "unique job keys to prevent duplicate
|
||||
# concurrent runs").
|
||||
active = await run_repo.get_active_for_company(company.id)
|
||||
if active is not None:
|
||||
return active
|
||||
|
||||
since = datetime.now(UTC) - timedelta(days=1)
|
||||
manual_count = await run_repo.count_manual_since(company.id, since)
|
||||
if manual_count >= settings.max_manual_runs_per_day:
|
||||
raise RateLimitedError(
|
||||
f"This company has reached the maximum of {settings.max_manual_runs_per_day} "
|
||||
"manual runs per day. Scheduled runs are unaffected."
|
||||
)
|
||||
|
||||
run = await run_repo.create(company_id=company.id, trigger_type=MonitoringRunTrigger.MANUAL)
|
||||
await db.commit()
|
||||
|
||||
from app.tasks.collection import (
|
||||
run_monitoring,
|
||||
) # local import: keeps Celery out of API startup path
|
||||
|
||||
run_monitoring.delay(str(run.id))
|
||||
return run
|
||||
|
||||
|
||||
async def list_runs(
|
||||
db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID
|
||||
) -> list[MonitoringRun]:
|
||||
await company_service.get_company(db, user_id, company_id)
|
||||
return await MonitoringRunRepository(db).list_for_company(company_id)
|
||||
|
||||
|
||||
async def get_run(db: AsyncSession, user_id: uuid.UUID, run_id: uuid.UUID) -> MonitoringRun:
|
||||
run = await MonitoringRunRepository(db).get(run_id)
|
||||
if run is None:
|
||||
raise NotFoundError("Monitoring run not found")
|
||||
await company_service.get_company(db, user_id, run.company_id) # ownership check
|
||||
return run
|
||||
@@ -0,0 +1,109 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.errors import NotFoundError, ValidationAppError
|
||||
from app.models.notification_destination import NotificationDestination
|
||||
from app.repositories.company_repository import CompanyRepository
|
||||
from app.repositories.notification_destination_repository import (
|
||||
NotificationDestinationRepository,
|
||||
)
|
||||
from app.schemas.notification_destination import (
|
||||
NotificationDestinationCreate,
|
||||
NotificationDestinationUpdate,
|
||||
)
|
||||
|
||||
|
||||
async def list_destinations(db: AsyncSession, user_id: uuid.UUID) -> list[NotificationDestination]:
|
||||
repo = NotificationDestinationRepository(db)
|
||||
return await repo.list_for_user(user_id)
|
||||
|
||||
|
||||
async def get_destination(
|
||||
db: AsyncSession, user_id: uuid.UUID, destination_id: uuid.UUID
|
||||
) -> NotificationDestination:
|
||||
repo = NotificationDestinationRepository(db)
|
||||
destination = await repo.get_for_user(destination_id, user_id)
|
||||
if destination is None:
|
||||
raise NotFoundError("Notification destination not found")
|
||||
return destination
|
||||
|
||||
|
||||
async def create_destination(
|
||||
db: AsyncSession, user_id: uuid.UUID, payload: NotificationDestinationCreate
|
||||
) -> NotificationDestination:
|
||||
"""Reuses an existing destination for the same (user, type, value)
|
||||
rather than creating a duplicate row - this is the fix for the wizard
|
||||
previously creating a fresh row per company even when the email/phone
|
||||
was already registered. Either way, the result ends up linked to every
|
||||
company in payload.company_ids."""
|
||||
company_repo = CompanyRepository(db)
|
||||
for company_id in payload.company_ids:
|
||||
if await company_repo.get_for_user(company_id, user_id) is None:
|
||||
raise ValidationAppError(f"Company {company_id} not found")
|
||||
|
||||
repo = NotificationDestinationRepository(db)
|
||||
destination = await repo.find_by_value(user_id, payload.type, payload.destination_value)
|
||||
if destination is None:
|
||||
destination = await repo.create(
|
||||
user_id=user_id,
|
||||
type=payload.type,
|
||||
destination_value=payload.destination_value,
|
||||
minimum_severity=payload.minimum_severity,
|
||||
enabled=payload.enabled,
|
||||
)
|
||||
|
||||
for company_id in payload.company_ids:
|
||||
await repo.link_company(destination.id, company_id)
|
||||
|
||||
await db.commit()
|
||||
refreshed = await repo.get_for_user(destination.id, user_id)
|
||||
assert refreshed is not None
|
||||
return refreshed
|
||||
|
||||
|
||||
async def update_destination(
|
||||
db: AsyncSession,
|
||||
user_id: uuid.UUID,
|
||||
destination_id: uuid.UUID,
|
||||
payload: NotificationDestinationUpdate,
|
||||
) -> NotificationDestination:
|
||||
destination = await get_destination(db, user_id, destination_id)
|
||||
updates = payload.model_dump(exclude_unset=True)
|
||||
for field, value in updates.items():
|
||||
setattr(destination, field, value)
|
||||
if field == "destination_value":
|
||||
# Changing the destination value invalidates any prior verification.
|
||||
destination.verified = False
|
||||
await db.commit()
|
||||
# Re-fetch (rather than db.refresh) so company_links stays eager-loaded -
|
||||
# refresh() would expire it, and a bare lazy-load isn't safe under
|
||||
# SQLAlchemy's async ORM without an active await context.
|
||||
refreshed = await get_destination(db, user_id, destination_id)
|
||||
return refreshed
|
||||
|
||||
|
||||
async def delete_destination(
|
||||
db: AsyncSession, user_id: uuid.UUID, destination_id: uuid.UUID
|
||||
) -> None:
|
||||
repo = NotificationDestinationRepository(db)
|
||||
destination = await get_destination(db, user_id, destination_id)
|
||||
await repo.delete(destination)
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def unlink_company(
|
||||
db: AsyncSession, user_id: uuid.UUID, destination_id: uuid.UUID, company_id: uuid.UUID
|
||||
) -> None:
|
||||
"""Removes this one company's link, not the destination itself - a
|
||||
destination can be shared across companies (see create_destination's
|
||||
dedup-by-value). If this was its last remaining link, it's an orphan
|
||||
now and delete_orphaned_for_user removes it outright, same as already
|
||||
happens when a company itself is deleted."""
|
||||
await get_destination(db, user_id, destination_id) # ownership check
|
||||
repo = NotificationDestinationRepository(db)
|
||||
await repo.unlink_company(destination_id, company_id)
|
||||
await repo.delete_orphaned_for_user(user_id)
|
||||
await db.commit()
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Renders a ReportContent (structured, from the LLM) into the Markdown
|
||||
document the API/UI serve alongside the JSON - see spec section 6G for the
|
||||
16-section layout this follows.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.prompts.report_generation import Finding, InferredProject, ReportContent
|
||||
|
||||
|
||||
def _render_findings(findings: list[Finding]) -> str:
|
||||
if not findings:
|
||||
return "_No findings for this section from the current evidence._\n"
|
||||
lines = []
|
||||
for f in findings:
|
||||
lines.append(f"- **{f.headline}** _(confidence: {f.confidence.value.replace('_', ' ')})_")
|
||||
lines.append(f" {f.summary}")
|
||||
if f.date:
|
||||
lines.append(f" _Date: {f.date}_")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def _render_projects(projects: list[InferredProject]) -> str:
|
||||
if not projects:
|
||||
return "_No inferred strategic projects from the current evidence._\n"
|
||||
lines = []
|
||||
for p in projects:
|
||||
lines.append(
|
||||
f"- **{p.project_name}** _({p.status.value.replace('_', ' ')}, confidence {p.confidence:.0%})_"
|
||||
)
|
||||
lines.append(f" {p.summary}")
|
||||
if p.alternative_explanations:
|
||||
lines.append(f" Alternative explanations: {'; '.join(p.alternative_explanations)}")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def _render_list(items: list[str]) -> str:
|
||||
if not items:
|
||||
return "_None noted._\n"
|
||||
return "\n".join(f"- {item}" for item in items) + "\n"
|
||||
|
||||
|
||||
def render_report_markdown(
|
||||
content: ReportContent,
|
||||
*,
|
||||
company_name: str,
|
||||
generated_at: str,
|
||||
model_provider: str,
|
||||
model_name: str,
|
||||
sources: list[dict],
|
||||
) -> str:
|
||||
parts = [
|
||||
f"# Competitive Intelligence Report: {company_name}",
|
||||
f"_Generated {generated_at} · {model_provider}/{model_name}_",
|
||||
"",
|
||||
"## 1. Executive Summary",
|
||||
content.executive_summary,
|
||||
"",
|
||||
"## 2. Company Overview",
|
||||
content.company_overview,
|
||||
"",
|
||||
"## 3. Products and Service Landscape",
|
||||
_render_findings(content.products_and_services),
|
||||
"## 4. Recent Developments",
|
||||
_render_findings(content.recent_developments),
|
||||
"## 5. Strategic Initiatives",
|
||||
_render_findings(content.strategic_initiatives),
|
||||
"## 6. Key Project Signals",
|
||||
_render_projects(content.key_inferred_projects),
|
||||
"## 7. Competitive Positioning",
|
||||
content.market_positioning,
|
||||
"",
|
||||
content.competitor_comparison,
|
||||
"",
|
||||
"## 8. SWOT Analysis",
|
||||
"**Strengths**",
|
||||
_render_list(content.swot.strengths),
|
||||
"**Weaknesses**",
|
||||
_render_list(content.swot.weaknesses),
|
||||
"**Opportunities**",
|
||||
_render_list(content.swot.opportunities),
|
||||
"**Threats**",
|
||||
_render_list(content.swot.threats),
|
||||
"## 9. Hiring Signals",
|
||||
_render_findings(content.hiring_signals),
|
||||
"## 10. Product and Technology Signals",
|
||||
_render_findings(content.technology_signals + content.patent_signals),
|
||||
"## 11. Customer Sentiment",
|
||||
content.customer_sentiment,
|
||||
"",
|
||||
"## 12. Financial and Regulatory Signals",
|
||||
_render_findings(content.financial_signals + content.regulatory_and_legal_signals),
|
||||
"## 13. Risks and Opportunities",
|
||||
"**Risks**",
|
||||
_render_list(content.risks),
|
||||
"**Opportunities**",
|
||||
_render_list(content.opportunities),
|
||||
"## 14. Important Unknowns",
|
||||
_render_list(content.unknowns_and_missing_data),
|
||||
"## 15. Sources",
|
||||
_render_sources(sources),
|
||||
"## 16. Methodology and Limitations",
|
||||
content.methodology,
|
||||
"",
|
||||
content.limitations,
|
||||
]
|
||||
return "\n".join(str(p) for p in parts)
|
||||
|
||||
|
||||
def _render_sources(sources: list[dict]) -> str:
|
||||
if not sources:
|
||||
return "_No sources recorded for this report._\n"
|
||||
lines = [
|
||||
f"- [{s.get('title') or s.get('url')}]({s.get('url')}) — retrieved {s.get('retrieved_date')}"
|
||||
for s in sources
|
||||
]
|
||||
return "\n".join(lines) + "\n"
|
||||
@@ -0,0 +1,188 @@
|
||||
"""Generates and persists a company's CI report from accumulated evidence.
|
||||
|
||||
Evidence gathering happens here, not inside the LLM prompt module - the
|
||||
model only ever sees data this pipeline actually collected (recent
|
||||
SourceDocuments + DetectedChanges), so it cannot introduce facts we never
|
||||
stored. See app/prompts/report_generation.py for the schema/prompt itself.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.analysis.llm.base import LLMProvider
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import NotFoundError
|
||||
from app.models.company import Company
|
||||
from app.models.detected_change import DetectedChange
|
||||
from app.models.enums import EnrichmentStatus, ReportType, SourceStatus
|
||||
from app.models.report import Report
|
||||
from app.models.source import Source
|
||||
from app.models.source_document import SourceDocument
|
||||
from app.prompts.report_generation import generate_report
|
||||
from app.repositories.company_enrichment_repository import CompanyEnrichmentRepository
|
||||
from app.repositories.report_repository import ReportRepository
|
||||
from app.services import company_service
|
||||
from app.services.report_markdown import render_report_markdown
|
||||
|
||||
_MAX_DOCUMENTS = 40
|
||||
_MAX_CHANGES = 20
|
||||
|
||||
|
||||
async def _gather_documents(db: AsyncSession, company_id: uuid.UUID) -> list[dict]:
|
||||
result = await db.execute(
|
||||
select(SourceDocument, Source.source_type)
|
||||
.join(Source, Source.id == SourceDocument.source_id)
|
||||
.where(SourceDocument.company_id == company_id)
|
||||
.order_by(SourceDocument.retrieved_date.desc())
|
||||
.limit(_MAX_DOCUMENTS)
|
||||
)
|
||||
return [
|
||||
{
|
||||
"id": str(doc.id),
|
||||
"title": doc.title,
|
||||
"url": doc.url,
|
||||
"excerpt": doc.content_text[:500],
|
||||
"source_type": source_type.value,
|
||||
"retrieved_date": doc.retrieved_date.isoformat(),
|
||||
}
|
||||
for doc, source_type in result.all()
|
||||
]
|
||||
|
||||
|
||||
async def _gather_changes(db: AsyncSession, company_id: uuid.UUID) -> list[dict]:
|
||||
result = await db.execute(
|
||||
select(DetectedChange)
|
||||
.where(DetectedChange.company_id == company_id)
|
||||
.order_by(DetectedChange.created_at.desc())
|
||||
.limit(_MAX_CHANGES)
|
||||
)
|
||||
return [
|
||||
{
|
||||
"id": str(c.id),
|
||||
"summary": c.summary,
|
||||
"change_type": c.change_type.value,
|
||||
"severity": c.severity.value,
|
||||
"confidence_score": c.confidence_score,
|
||||
"created_at": c.created_at.isoformat(),
|
||||
}
|
||||
for c in result.scalars().all()
|
||||
]
|
||||
|
||||
|
||||
async def _gather_failed_source_names(db: AsyncSession, company_id: uuid.UUID) -> list[str]:
|
||||
result = await db.execute(
|
||||
select(Source.name).where(
|
||||
Source.company_id == company_id, Source.status != SourceStatus.ACTIVE
|
||||
)
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def _gather_enrichment(db: AsyncSession, company_id: uuid.UUID) -> dict | None:
|
||||
enrichment = await CompanyEnrichmentRepository(db).get_for_company(company_id)
|
||||
if enrichment is None or enrichment.status == EnrichmentStatus.FAILED:
|
||||
return None
|
||||
return enrichment.data
|
||||
|
||||
|
||||
def _model_name(settings: Settings, llm: LLMProvider) -> str:
|
||||
if llm.provider_name == "anthropic":
|
||||
return settings.anthropic_model
|
||||
if llm.provider_name == "ollama":
|
||||
return settings.ollama_model
|
||||
return "mock"
|
||||
|
||||
|
||||
async def generate_and_persist_report(
|
||||
db: AsyncSession,
|
||||
settings: Settings,
|
||||
llm: LLMProvider,
|
||||
company: Company,
|
||||
*,
|
||||
report_type: ReportType,
|
||||
monitoring_run_id: uuid.UUID | None = None,
|
||||
) -> Report:
|
||||
documents = await _gather_documents(db, company.id)
|
||||
changes = await _gather_changes(db, company.id)
|
||||
failed_sources = await _gather_failed_source_names(db, company.id)
|
||||
enrichment = await _gather_enrichment(db, company.id)
|
||||
|
||||
content = await generate_report(
|
||||
llm,
|
||||
company_name=company.name,
|
||||
company_aliases=[a.alias for a in company.aliases],
|
||||
competitors=[c.name for c in company.competitors],
|
||||
monitoring_focus=company.monitoring_focus,
|
||||
industry=company.industry,
|
||||
documents=documents,
|
||||
detected_changes=changes,
|
||||
sources_failed=failed_sources,
|
||||
description=company.description,
|
||||
official_website=company.official_website,
|
||||
headquarters=company.headquarters,
|
||||
country=company.country,
|
||||
region=company.region,
|
||||
public_identifiers=company.public_identifiers,
|
||||
enrichment=enrichment,
|
||||
)
|
||||
|
||||
generated_at = datetime.now(UTC)
|
||||
model_name = _model_name(settings, llm)
|
||||
markdown = render_report_markdown(
|
||||
content,
|
||||
company_name=company.name,
|
||||
generated_at=generated_at.isoformat(),
|
||||
model_provider=llm.provider_name,
|
||||
model_name=model_name,
|
||||
sources=[
|
||||
{"title": d["title"], "url": d["url"], "retrieved_date": d["retrieved_date"]}
|
||||
for d in documents
|
||||
],
|
||||
)
|
||||
|
||||
report = Report(
|
||||
company_id=company.id,
|
||||
monitoring_run_id=monitoring_run_id,
|
||||
report_type=report_type,
|
||||
title=f"{company.name} — Competitive Intelligence Report",
|
||||
executive_summary=content.executive_summary,
|
||||
structured_report=content.model_dump(mode="json"),
|
||||
markdown_content=markdown,
|
||||
model_provider=llm.provider_name,
|
||||
model_name=model_name,
|
||||
)
|
||||
db.add(report)
|
||||
await db.commit()
|
||||
await db.refresh(report)
|
||||
return report
|
||||
|
||||
|
||||
async def list_reports(db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID) -> list[Report]:
|
||||
await company_service.get_company(db, user_id, company_id)
|
||||
return await ReportRepository(db).list_for_company(company_id)
|
||||
|
||||
|
||||
async def get_report(db: AsyncSession, user_id: uuid.UUID, report_id: uuid.UUID) -> Report:
|
||||
report = await ReportRepository(db).get(report_id)
|
||||
if report is None:
|
||||
raise NotFoundError("Report not found")
|
||||
await company_service.get_company(db, user_id, report.company_id) # ownership check
|
||||
return report
|
||||
|
||||
|
||||
async def generate_report_now(
|
||||
db: AsyncSession,
|
||||
settings: Settings,
|
||||
llm: LLMProvider,
|
||||
user_id: uuid.UUID,
|
||||
company_id: uuid.UUID,
|
||||
) -> Report:
|
||||
company = await company_service.get_company(db, user_id, company_id)
|
||||
return await generate_and_persist_report(
|
||||
db, settings, llm, company, report_type=ReportType.MANUAL
|
||||
)
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Schedule validation + next-run computation.
|
||||
|
||||
Shared between company creation/monitor-config updates (this phase) and the
|
||||
Celery Beat dynamic schedule sync (Phase 5) so both paths agree on what a
|
||||
valid schedule is and when it next fires.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
||||
|
||||
from croniter import CroniterBadCronError, croniter
|
||||
|
||||
from app.core.errors import ValidationAppError
|
||||
from app.models.enums import FREQUENCY_MINUTES, MonitoringFrequency
|
||||
|
||||
|
||||
def validate_and_compute_next_run(
|
||||
*,
|
||||
frequency_type: MonitoringFrequency,
|
||||
interval_minutes: int | None,
|
||||
cron_expression: str | None,
|
||||
tz_name: str,
|
||||
minimum_interval_minutes: int,
|
||||
from_time: datetime | None = None,
|
||||
) -> datetime:
|
||||
now = from_time or datetime.now(UTC)
|
||||
|
||||
try:
|
||||
tzinfo = ZoneInfo(tz_name)
|
||||
except ZoneInfoNotFoundError as exc:
|
||||
raise ValidationAppError(f"Unknown timezone: {tz_name}") from exc
|
||||
|
||||
if frequency_type == MonitoringFrequency.CUSTOM:
|
||||
return _compute_custom_next_run(
|
||||
interval_minutes=interval_minutes,
|
||||
cron_expression=cron_expression,
|
||||
tzinfo=tzinfo,
|
||||
minimum_interval_minutes=minimum_interval_minutes,
|
||||
now=now,
|
||||
)
|
||||
|
||||
frequency_minutes = FREQUENCY_MINUTES[frequency_type]
|
||||
if frequency_minutes < minimum_interval_minutes:
|
||||
raise ValidationAppError(
|
||||
f"{frequency_type.value} monitoring is more frequent than the minimum allowed "
|
||||
f"interval of {minimum_interval_minutes} minutes"
|
||||
)
|
||||
return now + timedelta(minutes=frequency_minutes)
|
||||
|
||||
|
||||
def _compute_custom_next_run(
|
||||
*,
|
||||
interval_minutes: int | None,
|
||||
cron_expression: str | None,
|
||||
tzinfo: ZoneInfo,
|
||||
minimum_interval_minutes: int,
|
||||
now: datetime,
|
||||
) -> datetime:
|
||||
if interval_minutes is not None:
|
||||
if interval_minutes < minimum_interval_minutes:
|
||||
raise ValidationAppError(
|
||||
f"Custom interval must be at least {minimum_interval_minutes} minutes"
|
||||
)
|
||||
return now + timedelta(minutes=interval_minutes)
|
||||
|
||||
if cron_expression:
|
||||
try:
|
||||
base = now.astimezone(tzinfo)
|
||||
next_local = croniter(cron_expression, base).get_next(datetime)
|
||||
except (CroniterBadCronError, ValueError) as exc:
|
||||
raise ValidationAppError(f"Invalid cron expression: {cron_expression}") from exc
|
||||
|
||||
next_utc = next_local.astimezone(UTC)
|
||||
if (next_utc - now) < timedelta(minutes=minimum_interval_minutes):
|
||||
raise ValidationAppError(
|
||||
"Custom schedule resolves to less than the minimum allowed interval of "
|
||||
f"{minimum_interval_minutes} minutes"
|
||||
)
|
||||
return next_utc
|
||||
|
||||
raise ValidationAppError("Custom frequency requires either interval_minutes or cron_expression")
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Transactional account-security email (verification codes, password
|
||||
reset codes, lockout notices) - deliberately separate from the alert-
|
||||
notification path in app/services/alert_service.py, which dispatches to
|
||||
user-configured destinations via app/notifications/factory.py. This is
|
||||
always sent to the account's own registered email, from a distinct sender
|
||||
identity (settings.resend_security_from_email, e.g. [email protected]
|
||||
vs. [email protected]).
|
||||
|
||||
Picks the Resend HTTP API provider when RESEND_API_KEY is configured, else
|
||||
falls back to the existing SmtpEmailProvider. `resolve_provider` is also
|
||||
reused by app.services.unban_service for the admin unban-request
|
||||
notification, which has the same "Resend if configured, else SMTP" needs.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.notifications.base import DeliveryResult, NotificationMessage
|
||||
from app.notifications.resend_email import ResendEmailProvider
|
||||
from app.notifications.smtp_email import SmtpEmailProvider
|
||||
|
||||
|
||||
def resolve_provider(settings: Settings):
|
||||
if settings.resend_api_key:
|
||||
return ResendEmailProvider(settings)
|
||||
return SmtpEmailProvider(settings)
|
||||
|
||||
|
||||
async def send_verification_code_email(settings: Settings, to: str, code: str) -> DeliveryResult:
|
||||
message = NotificationMessage(
|
||||
destination_value=to,
|
||||
subject="Verify your CI Agent account",
|
||||
body_text=(
|
||||
f"Your verification code is {code}.\n\n"
|
||||
"This code expires in 36 hours. If you didn't request this, you can ignore this email."
|
||||
),
|
||||
body_html=(
|
||||
f"<p>Your verification code is <strong style='font-size:20px'>{code}</strong>.</p>"
|
||||
"<p>This code expires in 36 hours. If you didn't request this, you can ignore this email.</p>"
|
||||
),
|
||||
)
|
||||
return await resolve_provider(settings).send(message)
|
||||
|
||||
|
||||
async def send_password_reset_email(settings: Settings, to: str, code: str) -> DeliveryResult:
|
||||
message = NotificationMessage(
|
||||
destination_value=to,
|
||||
subject="Reset your CI Agent password",
|
||||
body_text=(
|
||||
f"Your password reset code is {code}.\n\n"
|
||||
"This code expires in 36 hours. If you didn't request this, you can ignore this email "
|
||||
"and your password will stay unchanged."
|
||||
),
|
||||
body_html=(
|
||||
f"<p>Your password reset code is <strong style='font-size:20px'>{code}</strong>.</p>"
|
||||
"<p>This code expires in 36 hours. If you didn't request this, you can ignore this "
|
||||
"email and your password will stay unchanged.</p>"
|
||||
),
|
||||
)
|
||||
return await resolve_provider(settings).send(message)
|
||||
|
||||
|
||||
async def send_account_locked_email(settings: Settings, to: str) -> DeliveryResult:
|
||||
reset_link = f"{settings.frontend_url}/forgot-password"
|
||||
message = NotificationMessage(
|
||||
destination_value=to,
|
||||
subject="Your CI Agent account was locked",
|
||||
body_text=(
|
||||
"Your account was locked after repeated failed login attempts.\n\n"
|
||||
f"To unlock it, reset your password: {reset_link}"
|
||||
),
|
||||
body_html=(
|
||||
"<p>Your account was locked after repeated failed login attempts.</p>"
|
||||
f'<p>To unlock it, <a href="{reset_link}">reset your password</a>.</p>'
|
||||
),
|
||||
)
|
||||
return await resolve_provider(settings).send(message)
|
||||
@@ -0,0 +1,20 @@
|
||||
"""Read-only snapshot history for a company. Snapshots themselves are only
|
||||
ever written by collection_service.py during a monitoring run - this module
|
||||
just lists what's already there, ownership-checked."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.snapshot import Snapshot
|
||||
from app.repositories.source_repository import SnapshotRepository
|
||||
from app.services import company_service
|
||||
|
||||
|
||||
async def list_snapshots(
|
||||
db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID
|
||||
) -> list[Snapshot]:
|
||||
await company_service.get_company(db, user_id, company_id)
|
||||
return await SnapshotRepository(db).list_for_company(company_id)
|
||||
@@ -0,0 +1,115 @@
|
||||
"""Source CRUD + the ad-hoc "test this source" action. Collection
|
||||
orchestration itself (persisting documents/snapshots) lives in
|
||||
collection_service.py; this module is the ownership-checked API-facing
|
||||
layer on top of it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import NotFoundError
|
||||
from app.models.company import Company
|
||||
from app.models.source import Source
|
||||
from app.repositories.company_repository import CompanyRepository
|
||||
from app.repositories.source_repository import SourceRepository
|
||||
from app.schemas.source import SourceCreate, SourceUpdate
|
||||
from app.services.collection_service import CollectionResult, collect_source
|
||||
from app.services.scheduling import validate_and_compute_next_run
|
||||
|
||||
|
||||
async def _get_owned_company(
|
||||
db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID
|
||||
) -> Company:
|
||||
company = await CompanyRepository(db).get_for_user(company_id, user_id)
|
||||
if company is None:
|
||||
raise NotFoundError("Company not found")
|
||||
return company
|
||||
|
||||
|
||||
async def list_sources(db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID) -> list[Source]:
|
||||
await _get_owned_company(db, user_id, company_id)
|
||||
return await SourceRepository(db).list_for_company(company_id)
|
||||
|
||||
|
||||
async def create_source(
|
||||
db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID, payload: SourceCreate
|
||||
) -> Source:
|
||||
await _get_owned_company(db, user_id, company_id)
|
||||
source = await SourceRepository(db).create(
|
||||
company_id=company_id,
|
||||
source_type=payload.source_type,
|
||||
name=payload.name,
|
||||
base_url=payload.base_url,
|
||||
)
|
||||
await db.commit()
|
||||
return source
|
||||
|
||||
|
||||
async def _get_owned_source(db: AsyncSession, user_id: uuid.UUID, source_id: uuid.UUID) -> Source:
|
||||
source = await SourceRepository(db).get_for_user(source_id, user_id)
|
||||
if source is None:
|
||||
raise NotFoundError("Source not found")
|
||||
return source
|
||||
|
||||
|
||||
async def update_source(
|
||||
db: AsyncSession,
|
||||
settings: Settings,
|
||||
user_id: uuid.UUID,
|
||||
source_id: uuid.UUID,
|
||||
payload: SourceUpdate,
|
||||
) -> Source:
|
||||
source = await _get_owned_source(db, user_id, source_id)
|
||||
updates = payload.model_dump(exclude_unset=True)
|
||||
|
||||
schedule_changed = any(
|
||||
key in updates for key in ("frequency_type", "interval_minutes", "cron_expression")
|
||||
)
|
||||
if schedule_changed:
|
||||
frequency_type = updates.get("frequency_type", source.frequency_type)
|
||||
if frequency_type is not None:
|
||||
company = await CompanyRepository(db).get_for_user(source.company_id, user_id)
|
||||
tz_name = (
|
||||
company.monitor_configuration.timezone
|
||||
if company is not None and company.monitor_configuration is not None
|
||||
else "UTC"
|
||||
)
|
||||
# Validate only - the actual next_check is computed for real the
|
||||
# next time this source is collected (tasks/collection.py), same
|
||||
# as a brand-new source. Resetting it to None here means a
|
||||
# changed override takes effect on the very next scheduler tick
|
||||
# rather than waiting out whatever cadence was previously set.
|
||||
validate_and_compute_next_run(
|
||||
frequency_type=frequency_type,
|
||||
interval_minutes=updates.get("interval_minutes", source.interval_minutes),
|
||||
cron_expression=updates.get("cron_expression", source.cron_expression),
|
||||
tz_name=tz_name,
|
||||
minimum_interval_minutes=settings.minimum_monitoring_interval_minutes,
|
||||
)
|
||||
source.next_check = None
|
||||
|
||||
for field, value in updates.items():
|
||||
setattr(source, field, value)
|
||||
await db.commit()
|
||||
await db.refresh(source)
|
||||
return source
|
||||
|
||||
|
||||
async def delete_source(db: AsyncSession, user_id: uuid.UUID, source_id: uuid.UUID) -> None:
|
||||
source = await _get_owned_source(db, user_id, source_id)
|
||||
await SourceRepository(db).delete(source)
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def test_source(
|
||||
db: AsyncSession, settings: Settings, user_id: uuid.UUID, source_id: uuid.UUID
|
||||
) -> CollectionResult:
|
||||
source = await _get_owned_source(db, user_id, source_id)
|
||||
company = await CompanyRepository(db).get_for_user(source.company_id, user_id)
|
||||
if company is None: # pragma: no cover - defensive, implied by _get_owned_source
|
||||
raise NotFoundError("Company not found")
|
||||
return await collect_source(db, settings, source, company)
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Server-wide secrets an admin can configure from the Settings page
|
||||
instead of only via .env - today just the Cloudflare Turnstile site key
|
||||
and secret (app/services/turnstile_service.py). Storage is encrypted at
|
||||
rest (app/core/crypto.py). Unlike per-user API keys
|
||||
(user_api_key_service.py), there's exactly one value per key, shared by
|
||||
the whole app - visible/editable only to admins (see
|
||||
app/api/v1/system.py's require_admin gate), never per-user.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.crypto import decrypt_secret, encrypt_secret
|
||||
from app.models.enums import SecurityEventType, SystemSecretKey
|
||||
from app.repositories.system_secret_repository import SystemSecretRepository
|
||||
from app.repositories.user_security_event_repository import UserSecurityEventRepository
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SecretMeta:
|
||||
label: str
|
||||
settings_field: str
|
||||
|
||||
|
||||
META: dict[SystemSecretKey, SecretMeta] = {
|
||||
SystemSecretKey.TURNSTILE_SITE_KEY: SecretMeta(
|
||||
label="Cloudflare Turnstile Site Key", settings_field="turnstile_site_key"
|
||||
),
|
||||
SystemSecretKey.TURNSTILE_SECRET: SecretMeta(
|
||||
label="Cloudflare Turnstile Secret Key", settings_field="turnstile_secret"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
async def list_status(db: AsyncSession, settings: Settings) -> list[dict[str, Any]]:
|
||||
repo = SystemSecretRepository(db)
|
||||
stored = {row.key: row for row in await repo.list_all()}
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
for key, meta in META.items():
|
||||
row = stored.get(key)
|
||||
value = decrypt_secret(row.encrypted_value, settings) if row is not None else None
|
||||
results.append(
|
||||
{
|
||||
"key": key.value,
|
||||
"label": meta.label,
|
||||
"configured": row is not None,
|
||||
"value": value,
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
async def set_secret(
|
||||
db: AsyncSession,
|
||||
key: SystemSecretKey,
|
||||
plaintext_value: str,
|
||||
settings: Settings,
|
||||
*,
|
||||
admin_user_id: uuid.UUID,
|
||||
client_ip: str,
|
||||
) -> None:
|
||||
"""A blank value clears the stored override, falling back to the
|
||||
server's .env-configured value again. Every update - including a clear -
|
||||
is logged to the acting admin's own Account activity, since this is a
|
||||
security-sensitive, app-wide change (Turnstile keys today)."""
|
||||
repo = SystemSecretRepository(db)
|
||||
stripped = plaintext_value.strip()
|
||||
if not stripped:
|
||||
await repo.delete(key)
|
||||
else:
|
||||
await repo.upsert(key, encrypt_secret(stripped, settings))
|
||||
await UserSecurityEventRepository(db).create(
|
||||
user_id=admin_user_id,
|
||||
event_type=SecurityEventType.SERVER_SECRET_UPDATED,
|
||||
ip_address=client_ip,
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def get_effective_settings(db: AsyncSession, settings: Settings) -> Settings:
|
||||
"""A copy of the global settings with any admin-stored secret
|
||||
substituted in for the matching field - keys with no stored override
|
||||
keep using the server's .env-configured default."""
|
||||
rows = await SystemSecretRepository(db).list_all()
|
||||
if not rows:
|
||||
return settings
|
||||
overrides = {
|
||||
META[row.key].settings_field: decrypt_secret(row.encrypted_value, settings) for row in rows
|
||||
}
|
||||
return settings.model_copy(update=overrides)
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Cloudflare Turnstile server-side verification. Canonical siteverify
|
||||
contract per Cloudflare's own reference: POST {secret, response, remoteip}
|
||||
to the fixed challenges.cloudflare.com endpoint, check `success === true`.
|
||||
Fails closed on any network error or non-2xx response - a Cloudflare outage
|
||||
must never silently let requests through unverified.
|
||||
|
||||
One deliberate exception: a misconfigured *secret* (typo'd/invalid, as
|
||||
opposed to a genuinely bad/expired user token) fails open instead. Cloudflare
|
||||
reports this distinctly via `error-codes` (`invalid-input-secret` /
|
||||
`missing-input-secret`) rather than as an ambiguous non-2xx/network failure,
|
||||
so it's a real, detectable "the admin's config is broken" signal, not "we
|
||||
couldn't tell if this passed." Locking out every real register/login/
|
||||
password-reset attempt because of an admin's own copy-paste mistake is a
|
||||
worse outcome than briefly running with reduced bot protection - especially
|
||||
since Turnstile is one layer among several here (see SECURITY.md's IP
|
||||
throttle/ban and account-lockout layers, which stay fully active either
|
||||
way).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
_SITEVERIFY_URL = "https://challenges.cloudflare.com/turnstile/v0/siteverify"
|
||||
|
||||
# Cloudflare's own error-code vocabulary for a broken *secret* - distinct
|
||||
# from user-token-level codes like invalid-input-response/timeout-or-
|
||||
# duplicate/missing-input-response, which are legitimate rejections and
|
||||
# must keep failing closed.
|
||||
_SECRET_MISCONFIGURED_CODES = {"invalid-input-secret", "missing-input-secret"}
|
||||
|
||||
|
||||
async def verify_turnstile(token: str, remote_ip: str, settings: Settings) -> bool:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=10) as client:
|
||||
response = await client.post(
|
||||
_SITEVERIFY_URL,
|
||||
data={
|
||||
"secret": settings.turnstile_secret,
|
||||
"response": token,
|
||||
"remoteip": remote_ip,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
if data.get("success") is True:
|
||||
return True
|
||||
|
||||
error_codes = set(data.get("error-codes") or [])
|
||||
if error_codes & _SECRET_MISCONFIGURED_CODES:
|
||||
logger.error(
|
||||
"turnstile_secret_misconfigured",
|
||||
error_codes=sorted(error_codes),
|
||||
)
|
||||
return True # fail open - this is a config problem, not the caller's
|
||||
return False
|
||||
except Exception as exc: # noqa: BLE001 - fail closed on any error
|
||||
logger.warning("turnstile_verify_failed", error=str(exc))
|
||||
return False
|
||||
|
||||
|
||||
def turnstile_required(is_localhost: bool, settings: Settings) -> bool:
|
||||
"""Skipped entirely for a loopback caller, or when no secret is
|
||||
configured at all (matches this app's usual optional-provider
|
||||
convention - e.g. NinjaPear/USPTO/Brave all no-op when unset)."""
|
||||
return not is_localhost and bool(settings.turnstile_secret)
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Public unban-request intake + admin resolution. One request per IP per
|
||||
24h (enforced here), notifying every admin account's own email (Resend if
|
||||
configured, else SMTP - same provider selection as the rest of security
|
||||
email, see security_email_service.resolve_provider) and via the existing
|
||||
admin-only Redis log feed (app/core/logging.py), so it surfaces in the
|
||||
Settings page's Logging box too.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import ConflictError, NotFoundError, RateLimitedError
|
||||
from app.core.logging import get_logger
|
||||
from app.models.ip_ban import IpBan
|
||||
from app.notifications.base import NotificationMessage
|
||||
from app.repositories.ip_throttle_repository import IpThrottleRepository
|
||||
from app.repositories.unban_request_repository import UnbanRequestRepository
|
||||
from app.repositories.user_repository import UserRepository
|
||||
from app.services.security_email_service import resolve_provider
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
UNBAN_REQUEST_COOLDOWN_HOURS = 24
|
||||
|
||||
|
||||
async def submit_unban_request(
|
||||
db: AsyncSession, settings: Settings, ip_address: str, message: str | None
|
||||
) -> None:
|
||||
repo = UnbanRequestRepository(db)
|
||||
if await repo.within_cooldown(ip_address, UNBAN_REQUEST_COOLDOWN_HOURS):
|
||||
raise RateLimitedError(
|
||||
f"Only one unban request is allowed per {UNBAN_REQUEST_COOLDOWN_HOURS} hours."
|
||||
)
|
||||
|
||||
await repo.create(ip_address, message)
|
||||
await db.commit()
|
||||
|
||||
logger.warning("unban_request_received", ip=ip_address, message=message or "")
|
||||
|
||||
admin_emails = await UserRepository(db).list_admin_emails()
|
||||
if not admin_emails:
|
||||
logger.warning("unban_request_no_admin_to_notify", ip=ip_address)
|
||||
return
|
||||
|
||||
provider = resolve_provider(settings)
|
||||
for admin_email in admin_emails:
|
||||
email_message = NotificationMessage(
|
||||
destination_value=admin_email,
|
||||
subject=f"Unban request from {ip_address}",
|
||||
body_text=f"IP: {ip_address}\n\nMessage:\n{message or '(none)'}",
|
||||
)
|
||||
await provider.send(email_message)
|
||||
|
||||
|
||||
async def list_ip_bans(db: AsyncSession) -> list[IpBan]:
|
||||
return await IpThrottleRepository(db).list_bans()
|
||||
|
||||
|
||||
async def unban_ip(db: AsyncSession, ip_address: str) -> bool:
|
||||
repo = IpThrottleRepository(db)
|
||||
cleared = await repo.clear_ban_and_state(ip_address)
|
||||
await db.commit()
|
||||
return cleared
|
||||
|
||||
|
||||
async def ban_ip(db: AsyncSession, ip_address: str, reason: str = "manual_admin_ban") -> IpBan:
|
||||
"""Admin-initiated ban, bypassing the usual offense-count escalation
|
||||
ladder (ip_throttle_service) entirely - a deliberate manual override,
|
||||
not something the automated abuse-detection path produces."""
|
||||
repo = IpThrottleRepository(db)
|
||||
if await repo.get_ban(ip_address) is not None:
|
||||
raise ConflictError(f"{ip_address} is already banned.")
|
||||
ban = await repo.create_ban(ip_address, reason, datetime.now(UTC))
|
||||
await db.commit()
|
||||
return ban
|
||||
|
||||
|
||||
async def accept_unban_request(db: AsyncSession, request_id: uuid.UUID) -> None:
|
||||
"""Unbans the requester's IP and clears the request from the pending
|
||||
queue - a real pardon (see clear_ban_and_state), not just acknowledging
|
||||
the request was read."""
|
||||
repo = UnbanRequestRepository(db)
|
||||
request = await repo.get(request_id)
|
||||
if request is None:
|
||||
raise NotFoundError("Unban request not found.")
|
||||
await IpThrottleRepository(db).clear_ban_and_state(request.ip_address)
|
||||
await repo.delete(request_id)
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def reject_unban_request(db: AsyncSession, request_id: uuid.UUID) -> None:
|
||||
"""Dismisses the request without touching the ban - the IP stays
|
||||
banned."""
|
||||
repo = UnbanRequestRepository(db)
|
||||
if await repo.get(request_id) is None:
|
||||
raise NotFoundError("Unban request not found.")
|
||||
await repo.delete(request_id)
|
||||
await db.commit()
|
||||
@@ -0,0 +1,132 @@
|
||||
"""Per-user API keys - each user can supply their own key for a provider,
|
||||
used in place of the server's global .env-configured key for their own
|
||||
requests (see get_effective_settings, and its call sites in companies.py,
|
||||
reports.py, tasks/collection.py, tasks/enrichment.py, and
|
||||
collection_service.to_company_context for USPTO patents).
|
||||
|
||||
Storage is encrypted at rest (app/core/crypto.py). A key is only ever
|
||||
decrypted for the owning user's own list/set calls or to actually place a
|
||||
provider call on their behalf - never exposed to any other user, admin or
|
||||
not (this is a deliberately different visibility model from the
|
||||
admin-only, localhost-only *server* key box in app/api/v1/system.py).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.crypto import decrypt_secret, encrypt_secret
|
||||
from app.models.enums import ApiKeyProvider, SecurityEventType
|
||||
from app.repositories.user_api_key_repository import UserApiKeyRepository
|
||||
from app.repositories.user_security_event_repository import UserSecurityEventRepository
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProviderMeta:
|
||||
label: str
|
||||
settings_field: str
|
||||
credits_note: str
|
||||
free: bool = False
|
||||
requires_government_id: bool = False
|
||||
|
||||
|
||||
PROVIDER_META: dict[ApiKeyProvider, ProviderMeta] = {
|
||||
ApiKeyProvider.ANTHROPIC: ProviderMeta(
|
||||
label="Anthropic",
|
||||
settings_field="anthropic_api_key",
|
||||
credits_note="Anthropic doesn't expose a credit/usage-balance API.",
|
||||
),
|
||||
ApiKeyProvider.BRAVE_SEARCH: ProviderMeta(
|
||||
label="Brave Search",
|
||||
settings_field="brave_search_api_key",
|
||||
credits_note=(
|
||||
"Brave Search API has no metered balance endpoint (plan-based, not prepaid credits)."
|
||||
),
|
||||
),
|
||||
ApiKeyProvider.NINJAPEAR: ProviderMeta(
|
||||
label="NinjaPear",
|
||||
settings_field="ninjapear_api_key",
|
||||
credits_note="Credit balance shown in System configuration below.",
|
||||
),
|
||||
ApiKeyProvider.USPTO: ProviderMeta(
|
||||
label="USPTO",
|
||||
settings_field="uspto_api_key",
|
||||
credits_note="Free - USPTO Open Data Portal has no usage limit or credit cost.",
|
||||
free=True,
|
||||
requires_government_id=True,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
async def list_status(
|
||||
db: AsyncSession, user_id: uuid.UUID, settings: Settings
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Never fetches a live NinjaPear credit balance itself - the frontend
|
||||
sources that number from /system/status's own already-fetched
|
||||
ninjapear_credit_balance (see the Settings page's System configuration
|
||||
box) instead of this endpoint making a second, redundant live call."""
|
||||
repo = UserApiKeyRepository(db)
|
||||
stored = {row.provider: row for row in await repo.list_for_user(user_id)}
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
for provider, meta in PROVIDER_META.items():
|
||||
row = stored.get(provider)
|
||||
value = decrypt_secret(row.encrypted_key, settings) if row is not None else None
|
||||
results.append(
|
||||
{
|
||||
"provider": provider.value,
|
||||
"label": meta.label,
|
||||
"configured": row is not None,
|
||||
"value": value,
|
||||
"credits": None,
|
||||
"credits_note": meta.credits_note,
|
||||
"free": meta.free,
|
||||
"requires_government_id": meta.requires_government_id,
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
async def set_key(
|
||||
db: AsyncSession,
|
||||
user_id: uuid.UUID,
|
||||
provider: ApiKeyProvider,
|
||||
plaintext_key: str,
|
||||
settings: Settings,
|
||||
*,
|
||||
client_ip: str,
|
||||
) -> None:
|
||||
"""An empty/blank key clears the user's override, falling back to the
|
||||
server's global key for that provider again. Every update - including a
|
||||
clear - is logged to this same user's own Account activity."""
|
||||
repo = UserApiKeyRepository(db)
|
||||
stripped = plaintext_key.strip()
|
||||
if not stripped:
|
||||
await repo.delete(user_id, provider)
|
||||
else:
|
||||
await repo.upsert(user_id, provider, encrypt_secret(stripped, settings))
|
||||
await UserSecurityEventRepository(db).create(
|
||||
user_id=user_id, event_type=SecurityEventType.API_KEY_UPDATED, ip_address=client_ip
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def get_effective_settings(
|
||||
db: AsyncSession, user_id: uuid.UUID, settings: Settings
|
||||
) -> Settings:
|
||||
"""A copy of the global settings with any of this user's own stored
|
||||
keys substituted in for the matching field - providers they haven't
|
||||
set their own key for keep using the server's global default."""
|
||||
rows = await UserApiKeyRepository(db).list_for_user(user_id)
|
||||
if not rows:
|
||||
return settings
|
||||
overrides = {
|
||||
PROVIDER_META[row.provider].settings_field: decrypt_secret(row.encrypted_key, settings)
|
||||
for row in rows
|
||||
}
|
||||
return settings.model_copy(update=overrides)
|
||||
Reference in New Issue
Block a user