From 1a4c80958f8733c1088337d5838031d4910a9072 Mon Sep 17 00:00:00 2001 From: Saksham Das Date: Wed, 5 Aug 2026 10:48:20 -0400 Subject: [PATCH] Initial commit: CI Agent competitive-intelligence monitoring app FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification). --- .env.example | 152 + .gitignore | 59 + ARCHITECTURE.md | 82 + KNOWN_LIMITATIONS.md | 181 + PLAN.md | 28 + README.md | 93 + SECURITY.md | 65 + TASKS.md | 171 + apps/api/alembic.ini | 38 + apps/api/app/__init__.py | 0 apps/api/app/analysis/__init__.py | 0 apps/api/app/analysis/llm/__init__.py | 0 .../app/analysis/llm/anthropic_provider.py | 82 + apps/api/app/analysis/llm/base.py | 33 + apps/api/app/analysis/llm/factory.py | 31 + apps/api/app/analysis/llm/gemini_provider.py | 73 + apps/api/app/analysis/llm/mock.py | 327 + apps/api/app/analysis/llm/ollama_provider.py | 85 + apps/api/app/api/__init__.py | 0 apps/api/app/api/v1/__init__.py | 0 apps/api/app/api/v1/admin.py | 91 + apps/api/app/api/v1/alerts.py | 78 + apps/api/app/api/v1/auth.py | 161 + apps/api/app/api/v1/companies.py | 155 + apps/api/app/api/v1/dashboard.py | 23 + apps/api/app/api/v1/monitoring.py | 55 + .../app/api/v1/notification_destinations.py | 91 + apps/api/app/api/v1/reports.py | 80 + apps/api/app/api/v1/router.py | 35 + apps/api/app/api/v1/snapshots.py | 27 + apps/api/app/api/v1/sources.py | 83 + apps/api/app/api/v1/system.py | 196 + apps/api/app/api/v1/user_api_keys.py | 46 + apps/api/app/auth/__init__.py | 0 apps/api/app/auth/dependencies.py | 69 + apps/api/app/change_detection/__init__.py | 0 apps/api/app/change_detection/extractors.py | 25 + .../api/app/change_detection/noise_filters.py | 32 + apps/api/app/change_detection/scoring.py | 103 + .../app/change_detection/structured_diff.py | 33 + apps/api/app/change_detection/text_diff.py | 56 + apps/api/app/collectors/__init__.py | 0 apps/api/app/collectors/base.py | 96 + apps/api/app/collectors/custom_url.py | 87 + apps/api/app/collectors/extraction.py | 76 + apps/api/app/collectors/github.py | 147 + apps/api/app/collectors/gov_contracts.py | 131 + apps/api/app/collectors/jobs.py | 156 + apps/api/app/collectors/patents.py | 245 + apps/api/app/collectors/registry.py | 36 + apps/api/app/collectors/reviews.py | 85 + apps/api/app/collectors/robots.py | 54 + apps/api/app/collectors/rss.py | 111 + apps/api/app/collectors/sec_edgar.py | 167 + apps/api/app/collectors/website.py | 176 + apps/api/app/core/__init__.py | 0 apps/api/app/core/config.py | 178 + apps/api/app/core/crypto.py | 25 + apps/api/app/core/errors.py | 46 + apps/api/app/core/http.py | 176 + apps/api/app/core/logging.py | 149 + apps/api/app/core/rate_limit.py | 28 + apps/api/app/core/security.py | 171 + apps/api/app/core/text.py | 13 + apps/api/app/db/__init__.py | 0 apps/api/app/db/base.py | 38 + apps/api/app/db/session.py | 50 + apps/api/app/enrichment/__init__.py | 0 apps/api/app/enrichment/base.py | 93 + apps/api/app/enrichment/factory.py | 21 + apps/api/app/enrichment/mock.py | 50 + apps/api/app/enrichment/ninjapear.py | 213 + apps/api/app/main.py | 136 + apps/api/app/models/__init__.py | 35 + apps/api/app/models/alert.py | 32 + apps/api/app/models/company.py | 76 + apps/api/app/models/company_enrichment.py | 42 + apps/api/app/models/detected_change.py | 44 + apps/api/app/models/email_code.py | 37 + apps/api/app/models/enums.py | 176 + apps/api/app/models/ip_ban.py | 21 + apps/api/app/models/ip_throttle_state.py | 29 + apps/api/app/models/monitor_configuration.py | 40 + apps/api/app/models/monitoring_run.py | 34 + apps/api/app/models/notification_delivery.py | 30 + .../app/models/notification_destination.py | 57 + apps/api/app/models/password_history.py | 28 + apps/api/app/models/refresh_token.py | 32 + apps/api/app/models/report.py | 29 + apps/api/app/models/snapshot.py | 31 + apps/api/app/models/source.py | 45 + apps/api/app/models/source_document.py | 40 + apps/api/app/models/system_secret.py | 24 + apps/api/app/models/unban_request.py | 17 + apps/api/app/models/user.py | 47 + apps/api/app/models/user_api_key.py | 36 + apps/api/app/models/user_known_ip.py | 36 + apps/api/app/models/user_security_event.py | 31 + apps/api/app/notifications/__init__.py | 0 apps/api/app/notifications/base.py | 31 + apps/api/app/notifications/console.py | 23 + apps/api/app/notifications/factory.py | 31 + apps/api/app/notifications/message_builder.py | 68 + apps/api/app/notifications/resend_email.py | 50 + apps/api/app/notifications/smtp_email.py | 53 + apps/api/app/notifications/telnyx_sms.py | 50 + apps/api/app/notifications/twilio_sms.py | 52 + apps/api/app/prompts/__init__.py | 0 apps/api/app/prompts/alert_summarization.py | 46 + apps/api/app/prompts/base.py | 39 + apps/api/app/prompts/change_significance.py | 60 + apps/api/app/prompts/company_profile.py | 72 + apps/api/app/prompts/extraction.py | 54 + apps/api/app/prompts/relevance.py | 51 + apps/api/app/prompts/report_generation.py | 135 + apps/api/app/prompts/schemas.py | 35 + apps/api/app/prompts/synthesis.py | 54 + apps/api/app/repositories/__init__.py | 0 apps/api/app/repositories/alert_repository.py | 55 + .../company_enrichment_repository.py | 57 + .../app/repositories/company_repository.py | 173 + .../detected_change_repository.py | 51 + .../app/repositories/email_code_repository.py | 69 + .../repositories/ip_throttle_repository.py | 64 + .../repositories/monitoring_run_repository.py | 124 + .../notification_delivery_repository.py | 24 + .../notification_destination_repository.py | 149 + .../password_history_repository.py | 25 + .../repositories/refresh_token_repository.py | 55 + .../api/app/repositories/report_repository.py | 46 + .../api/app/repositories/source_repository.py | 225 + .../repositories/system_secret_repository.py | 37 + .../repositories/unban_request_repository.py | 53 + .../repositories/user_api_key_repository.py | 43 + .../repositories/user_known_ip_repository.py | 47 + apps/api/app/repositories/user_repository.py | 48 + .../user_security_event_repository.py | 42 + apps/api/app/schemas/__init__.py | 0 apps/api/app/schemas/alert.py | 44 + apps/api/app/schemas/auth.py | 81 + apps/api/app/schemas/company.py | 163 + apps/api/app/schemas/dashboard.py | 32 + apps/api/app/schemas/discovery.py | 35 + apps/api/app/schemas/monitoring.py | 26 + .../app/schemas/notification_destination.py | 84 + apps/api/app/schemas/report.py | 28 + apps/api/app/schemas/snapshot.py | 20 + apps/api/app/schemas/source.py | 73 + apps/api/app/schemas/system_secret.py | 16 + apps/api/app/schemas/unban.py | 41 + apps/api/app/schemas/user.py | 20 + apps/api/app/schemas/user_api_key.py | 20 + apps/api/app/search/__init__.py | 0 apps/api/app/search/base.py | 26 + apps/api/app/search/brave.py | 42 + apps/api/app/search/factory.py | 21 + apps/api/app/search/mock.py | 62 + apps/api/app/services/__init__.py | 0 apps/api/app/services/alert_service.py | 202 + apps/api/app/services/analytics_service.py | 105 + apps/api/app/services/auth_service.py | 428 + .../app/services/change_detection_service.py | 172 + apps/api/app/services/collection_service.py | 198 + apps/api/app/services/company_service.py | 243 + apps/api/app/services/discovery_service.py | 190 + apps/api/app/services/enrichment_service.py | 176 + apps/api/app/services/ip_throttle_service.py | 151 + apps/api/app/services/monitoring_service.py | 66 + .../notification_destination_service.py | 109 + apps/api/app/services/report_markdown.py | 117 + apps/api/app/services/report_service.py | 188 + apps/api/app/services/scheduling.py | 83 + .../app/services/security_email_service.py | 77 + apps/api/app/services/snapshot_service.py | 20 + apps/api/app/services/source_service.py | 115 + .../api/app/services/system_secret_service.py | 97 + apps/api/app/services/turnstile_service.py | 71 + apps/api/app/services/unban_service.py | 103 + apps/api/app/services/user_api_key_service.py | 132 + apps/api/app/tasks/__init__.py | 0 apps/api/app/tasks/base.py | 33 + apps/api/app/tasks/celery_app.py | 60 + apps/api/app/tasks/collection.py | 265 + apps/api/app/tasks/enrichment.py | 55 + apps/api/app/tasks/maintenance.py | 43 + apps/api/app/tasks/scheduler.py | 73 + apps/api/celerybeat-schedule | Bin 0 -> 4096 bytes apps/api/migrations/env.py | 60 + apps/api/migrations/script.py.mako | 27 + apps/api/migrations/versions/.gitkeep | 0 ...cfe49b9_create_users_and_refresh_tokens.py | 64 + ...a36_add_per_source_scheduling_override_.py | 57 + ...57_create_sources_source_documents_and_.py | 155 + ...add_notification_destination_companies_.py | 120 + .../6d296b533f7c_add_user_api_keys_table.py | 47 + ...041131_create_company_enrichments_table.py | 64 + ...3b3b_add_password_history_entries_table.py | 45 + ...3b8aa1_create_companies_monitoring_and_.py | 186 + .../8ceefbd0a6a5_create_monitoring_runs.py | 96 + .../8db5bd1c31f8_add_system_secrets_table.py | 46 + ...93c89ac_create_alerts_and_notification_.py | 121 + .../versions/c06c845cdd10_create_reports.py | 68 + .../c34c769afc07_add_user_known_ips_table.py | 41 + .../cb61ba9570fb_create_detected_changes.py | 107 + ...add_email_verification_ip_throttle_ban_.py | 159 + ...e9_add_company_headquarters_and_public_.py | 39 + apps/api/pyproject.toml | 77 + apps/api/tests/__init__.py | 0 apps/api/tests/collectors/__init__.py | 0 apps/api/tests/collectors/conftest.py | 24 + .../collectors/test_custom_url_collector.py | 72 + .../collectors/test_fixture_collectors.py | 75 + .../tests/collectors/test_github_collector.py | 73 + .../test_gov_contracts_collector.py | 94 + .../collectors/test_job_posting_collector.py | 73 + .../collectors/test_patents_collector.py | 160 + .../tests/collectors/test_rss_collector.py | 85 + .../collectors/test_sec_edgar_collector.py | 91 + .../collectors/test_website_collector.py | 139 + apps/api/tests/conftest.py | 87 + apps/api/tests/enrichment/__init__.py | 0 .../tests/enrichment/test_mock_provider.py | 25 + .../enrichment/test_ninjapear_provider.py | 186 + .../fixtures/acme_mobility/v1/about.html | 13 + .../fixtures/acme_mobility/v1/careers.html | 12 + .../fixtures/acme_mobility/v1/press.html | 12 + .../fixtures/acme_mobility/v1/pricing.html | 11 + .../fixtures/acme_mobility/v1/products.html | 11 + .../fixtures/acme_mobility/v2/about.html | 14 + .../fixtures/acme_mobility/v2/careers.html | 13 + .../fixtures/acme_mobility/v2/press.html | 15 + .../fixtures/acme_mobility/v2/pricing.html | 11 + .../fixtures/acme_mobility/v2/products.html | 11 + .../tests/fixtures/patents/acme_mobility.json | 11 + .../tests/fixtures/reviews/acme_mobility.json | 20 + apps/api/tests/integration/__init__.py | 0 .../integration/test_acme_fixture_demo.py | 117 + .../tests/integration/test_alert_service.py | 326 + .../integration/test_analytics_service.py | 169 + .../test_change_detection_service.py | 372 + .../integration/test_collection_service.py | 221 + .../integration/test_discovery_service.py | 189 + .../tests/integration/test_enrichment_task.py | 105 + .../integration/test_maintenance_task.py | 76 + .../integration/test_per_source_scheduling.py | 126 + .../tests/integration/test_report_service.py | 153 + .../tests/integration/test_scheduler_task.py | 138 + apps/api/tests/unit/__init__.py | 0 apps/api/tests/unit/test_alerting_e2e.py | 173 + apps/api/tests/unit/test_alerts_api.py | 200 + apps/api/tests/unit/test_auth.py | 254 + apps/api/tests/unit/test_companies.py | 254 + apps/api/tests/unit/test_correlation_id.py | 22 + apps/api/tests/unit/test_dashboard_api.py | 38 + apps/api/tests/unit/test_discover_endpoint.py | 95 + .../api/tests/unit/test_enrichment_service.py | 170 + apps/api/tests/unit/test_extraction.py | 67 + apps/api/tests/unit/test_extractors.py | 22 + apps/api/tests/unit/test_health.py | 25 + .../tests/unit/test_ip_throttle_service.py | 245 + apps/api/tests/unit/test_llm_providers.py | 179 + apps/api/tests/unit/test_mock_llm_provider.py | 160 + apps/api/tests/unit/test_monitoring_smoke.py | 220 + apps/api/tests/unit/test_noise_filters.py | 30 + .../unit/test_notification_destinations.py | 331 + ...notification_destinations_test_endpoint.py | 121 + .../tests/unit/test_notification_providers.py | 256 + apps/api/tests/unit/test_rate_limit.py | 89 + .../unit/test_report_generation_prompt.py | 69 + apps/api/tests/unit/test_reports_api.py | 91 + apps/api/tests/unit/test_scoring.py | 130 + apps/api/tests/unit/test_search_providers.py | 78 + apps/api/tests/unit/test_security_flows.py | 654 + apps/api/tests/unit/test_snapshots_api.py | 98 + apps/api/tests/unit/test_sources_api.py | 190 + apps/api/tests/unit/test_ssrf.py | 77 + apps/api/tests/unit/test_structured_diff.py | 26 + apps/api/tests/unit/test_system_endpoints.py | 121 + apps/api/tests/unit/test_system_secrets.py | 294 + apps/api/tests/unit/test_text_diff.py | 34 + apps/api/tests/unit/test_unban_admin.py | 263 + apps/api/tests/unit/test_user_api_keys.py | 289 + apps/api/tests/unit/test_user_known_ips.py | 118 + apps/web/.eslintrc.json | 6 + apps/web/.prettierignore | 7 + apps/web/.prettierrc.json | 7 + apps/web/app/(app)/alerts/[id]/page.tsx | 138 + apps/web/app/(app)/alerts/page.tsx | 152 + apps/web/app/(app)/companies/[id]/page.tsx | 1269 ++ apps/web/app/(app)/companies/new/page.tsx | 680 + apps/web/app/(app)/companies/page.tsx | 162 + apps/web/app/(app)/dashboard/page.tsx | 147 + apps/web/app/(app)/layout.tsx | 75 + apps/web/app/(app)/reports/[id]/page.tsx | 55 + apps/web/app/(app)/settings/page.tsx | 766 ++ apps/web/app/forgot-password/page.tsx | 126 + apps/web/app/globals.css | 41 + apps/web/app/layout.tsx | 19 + apps/web/app/login/page.tsx | 188 + apps/web/app/page.tsx | 135 + apps/web/app/providers.tsx | 20 + apps/web/app/register/page.tsx | 152 + apps/web/app/reset-password/page.tsx | 141 + apps/web/app/unban-request/page.tsx | 93 + apps/web/app/verify-email/page.tsx | 165 + .../dashboard/analytics-section.tsx | 268 + apps/web/components/local-mode-banner.tsx | 22 + apps/web/components/page-transition.tsx | 19 + apps/web/components/report-view.tsx | 249 + apps/web/components/ui/badge.tsx | 41 + apps/web/components/ui/company-pill.tsx | 41 + apps/web/components/ui/copyable-email.tsx | 39 + apps/web/components/ui/form-field.tsx | 44 + .../ui/notification-channel-box.tsx | 90 + apps/web/components/ui/password-field.tsx | 76 + .../components/ui/password-strength-meter.tsx | 43 + apps/web/components/ui/select.tsx | 84 + apps/web/components/ui/stat-card.tsx | 24 + apps/web/components/ui/system-secret-row.tsx | 106 + apps/web/components/ui/turnstile-widget.tsx | 104 + apps/web/components/ui/user-api-key-row.tsx | 134 + apps/web/hooks/use-alerts.ts | 49 + apps/web/hooks/use-analytics.ts | 11 + apps/web/hooks/use-auth.ts | 265 + apps/web/hooks/use-companies.ts | 80 + apps/web/hooks/use-countdown.ts | 17 + apps/web/hooks/use-discovery.ts | 11 + apps/web/hooks/use-monitoring-runs.ts | 43 + .../hooks/use-notification-destinations.ts | 61 + apps/web/hooks/use-reports.ts | 35 + apps/web/hooks/use-snapshots.ts | 11 + apps/web/hooks/use-sources.ts | 42 + apps/web/lib/api-client.ts | 415 + apps/web/lib/auth.ts | 13 + apps/web/lib/config.ts | 1 + apps/web/lib/format.ts | 76 + apps/web/lib/types.ts | 588 + apps/web/next-env.d.ts | 6 + apps/web/next.config.js | 7 + apps/web/package-lock.json | 10505 ++++++++++++++++ apps/web/package.json | 51 + apps/web/playwright.config.ts | 19 + apps/web/postcss.config.js | 6 + apps/web/tailwind.config.ts | 53 + apps/web/tests/add-company-wizard.test.tsx | 131 + apps/web/tests/alerts-page.test.tsx | 71 + apps/web/tests/analytics-section.test.tsx | 64 + apps/web/tests/api-client.test.ts | 96 + apps/web/tests/auth-forms.test.tsx | 65 + apps/web/tests/badges.test.tsx | 23 + apps/web/tests/landing-page.test.tsx | 70 + apps/web/tests/security-pages.test.tsx | 112 + apps/web/tests/setup.ts | 17 + apps/web/tests/system-secret-row.test.tsx | 87 + apps/web/tests/test-utils.tsx | 10 + apps/web/tests/turnstile-widget.test.tsx | 55 + apps/web/tests/user-api-key-row.test.tsx | 137 + apps/web/tsconfig.json | 30 + apps/web/vitest.config.ts | 18 + docker-compose.yml | 110 + docs/FIREBASE_MIGRATION.md | 38 + infrastructure/docker/api.Dockerfile | 23 + infrastructure/docker/web.Dockerfile | 12 + packages/shared/package.json | 8 + packages/shared/src/index.ts | 40 + 365 files changed, 43541 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 ARCHITECTURE.md create mode 100644 KNOWN_LIMITATIONS.md create mode 100644 PLAN.md create mode 100644 README.md create mode 100644 SECURITY.md create mode 100644 TASKS.md create mode 100644 apps/api/alembic.ini create mode 100644 apps/api/app/__init__.py create mode 100644 apps/api/app/analysis/__init__.py create mode 100644 apps/api/app/analysis/llm/__init__.py create mode 100644 apps/api/app/analysis/llm/anthropic_provider.py create mode 100644 apps/api/app/analysis/llm/base.py create mode 100644 apps/api/app/analysis/llm/factory.py create mode 100644 apps/api/app/analysis/llm/gemini_provider.py create mode 100644 apps/api/app/analysis/llm/mock.py create mode 100644 apps/api/app/analysis/llm/ollama_provider.py create mode 100644 apps/api/app/api/__init__.py create mode 100644 apps/api/app/api/v1/__init__.py create mode 100644 apps/api/app/api/v1/admin.py create mode 100644 apps/api/app/api/v1/alerts.py create mode 100644 apps/api/app/api/v1/auth.py create mode 100644 apps/api/app/api/v1/companies.py create mode 100644 apps/api/app/api/v1/dashboard.py create mode 100644 apps/api/app/api/v1/monitoring.py create mode 100644 apps/api/app/api/v1/notification_destinations.py create mode 100644 apps/api/app/api/v1/reports.py create mode 100644 apps/api/app/api/v1/router.py create mode 100644 apps/api/app/api/v1/snapshots.py create mode 100644 apps/api/app/api/v1/sources.py create mode 100644 apps/api/app/api/v1/system.py create mode 100644 apps/api/app/api/v1/user_api_keys.py create mode 100644 apps/api/app/auth/__init__.py create mode 100644 apps/api/app/auth/dependencies.py create mode 100644 apps/api/app/change_detection/__init__.py create mode 100644 apps/api/app/change_detection/extractors.py create mode 100644 apps/api/app/change_detection/noise_filters.py create mode 100644 apps/api/app/change_detection/scoring.py create mode 100644 apps/api/app/change_detection/structured_diff.py create mode 100644 apps/api/app/change_detection/text_diff.py create mode 100644 apps/api/app/collectors/__init__.py create mode 100644 apps/api/app/collectors/base.py create mode 100644 apps/api/app/collectors/custom_url.py create mode 100644 apps/api/app/collectors/extraction.py create mode 100644 apps/api/app/collectors/github.py create mode 100644 apps/api/app/collectors/gov_contracts.py create mode 100644 apps/api/app/collectors/jobs.py create mode 100644 apps/api/app/collectors/patents.py create mode 100644 apps/api/app/collectors/registry.py create mode 100644 apps/api/app/collectors/reviews.py create mode 100644 apps/api/app/collectors/robots.py create mode 100644 apps/api/app/collectors/rss.py create mode 100644 apps/api/app/collectors/sec_edgar.py create mode 100644 apps/api/app/collectors/website.py create mode 100644 apps/api/app/core/__init__.py create mode 100644 apps/api/app/core/config.py create mode 100644 apps/api/app/core/crypto.py create mode 100644 apps/api/app/core/errors.py create mode 100644 apps/api/app/core/http.py create mode 100644 apps/api/app/core/logging.py create mode 100644 apps/api/app/core/rate_limit.py create mode 100644 apps/api/app/core/security.py create mode 100644 apps/api/app/core/text.py create mode 100644 apps/api/app/db/__init__.py create mode 100644 apps/api/app/db/base.py create mode 100644 apps/api/app/db/session.py create mode 100644 apps/api/app/enrichment/__init__.py create mode 100644 apps/api/app/enrichment/base.py create mode 100644 apps/api/app/enrichment/factory.py create mode 100644 apps/api/app/enrichment/mock.py create mode 100644 apps/api/app/enrichment/ninjapear.py create mode 100644 apps/api/app/main.py create mode 100644 apps/api/app/models/__init__.py create mode 100644 apps/api/app/models/alert.py create mode 100644 apps/api/app/models/company.py create mode 100644 apps/api/app/models/company_enrichment.py create mode 100644 apps/api/app/models/detected_change.py create mode 100644 apps/api/app/models/email_code.py create mode 100644 apps/api/app/models/enums.py create mode 100644 apps/api/app/models/ip_ban.py create mode 100644 apps/api/app/models/ip_throttle_state.py create mode 100644 apps/api/app/models/monitor_configuration.py create mode 100644 apps/api/app/models/monitoring_run.py create mode 100644 apps/api/app/models/notification_delivery.py create mode 100644 apps/api/app/models/notification_destination.py create mode 100644 apps/api/app/models/password_history.py create mode 100644 apps/api/app/models/refresh_token.py create mode 100644 apps/api/app/models/report.py create mode 100644 apps/api/app/models/snapshot.py create mode 100644 apps/api/app/models/source.py create mode 100644 apps/api/app/models/source_document.py create mode 100644 apps/api/app/models/system_secret.py create mode 100644 apps/api/app/models/unban_request.py create mode 100644 apps/api/app/models/user.py create mode 100644 apps/api/app/models/user_api_key.py create mode 100644 apps/api/app/models/user_known_ip.py create mode 100644 apps/api/app/models/user_security_event.py create mode 100644 apps/api/app/notifications/__init__.py create mode 100644 apps/api/app/notifications/base.py create mode 100644 apps/api/app/notifications/console.py create mode 100644 apps/api/app/notifications/factory.py create mode 100644 apps/api/app/notifications/message_builder.py create mode 100644 apps/api/app/notifications/resend_email.py create mode 100644 apps/api/app/notifications/smtp_email.py create mode 100644 apps/api/app/notifications/telnyx_sms.py create mode 100644 apps/api/app/notifications/twilio_sms.py create mode 100644 apps/api/app/prompts/__init__.py create mode 100644 apps/api/app/prompts/alert_summarization.py create mode 100644 apps/api/app/prompts/base.py create mode 100644 apps/api/app/prompts/change_significance.py create mode 100644 apps/api/app/prompts/company_profile.py create mode 100644 apps/api/app/prompts/extraction.py create mode 100644 apps/api/app/prompts/relevance.py create mode 100644 apps/api/app/prompts/report_generation.py create mode 100644 apps/api/app/prompts/schemas.py create mode 100644 apps/api/app/prompts/synthesis.py create mode 100644 apps/api/app/repositories/__init__.py create mode 100644 apps/api/app/repositories/alert_repository.py create mode 100644 apps/api/app/repositories/company_enrichment_repository.py create mode 100644 apps/api/app/repositories/company_repository.py create mode 100644 apps/api/app/repositories/detected_change_repository.py create mode 100644 apps/api/app/repositories/email_code_repository.py create mode 100644 apps/api/app/repositories/ip_throttle_repository.py create mode 100644 apps/api/app/repositories/monitoring_run_repository.py create mode 100644 apps/api/app/repositories/notification_delivery_repository.py create mode 100644 apps/api/app/repositories/notification_destination_repository.py create mode 100644 apps/api/app/repositories/password_history_repository.py create mode 100644 apps/api/app/repositories/refresh_token_repository.py create mode 100644 apps/api/app/repositories/report_repository.py create mode 100644 apps/api/app/repositories/source_repository.py create mode 100644 apps/api/app/repositories/system_secret_repository.py create mode 100644 apps/api/app/repositories/unban_request_repository.py create mode 100644 apps/api/app/repositories/user_api_key_repository.py create mode 100644 apps/api/app/repositories/user_known_ip_repository.py create mode 100644 apps/api/app/repositories/user_repository.py create mode 100644 apps/api/app/repositories/user_security_event_repository.py create mode 100644 apps/api/app/schemas/__init__.py create mode 100644 apps/api/app/schemas/alert.py create mode 100644 apps/api/app/schemas/auth.py create mode 100644 apps/api/app/schemas/company.py create mode 100644 apps/api/app/schemas/dashboard.py create mode 100644 apps/api/app/schemas/discovery.py create mode 100644 apps/api/app/schemas/monitoring.py create mode 100644 apps/api/app/schemas/notification_destination.py create mode 100644 apps/api/app/schemas/report.py create mode 100644 apps/api/app/schemas/snapshot.py create mode 100644 apps/api/app/schemas/source.py create mode 100644 apps/api/app/schemas/system_secret.py create mode 100644 apps/api/app/schemas/unban.py create mode 100644 apps/api/app/schemas/user.py create mode 100644 apps/api/app/schemas/user_api_key.py create mode 100644 apps/api/app/search/__init__.py create mode 100644 apps/api/app/search/base.py create mode 100644 apps/api/app/search/brave.py create mode 100644 apps/api/app/search/factory.py create mode 100644 apps/api/app/search/mock.py create mode 100644 apps/api/app/services/__init__.py create mode 100644 apps/api/app/services/alert_service.py create mode 100644 apps/api/app/services/analytics_service.py create mode 100644 apps/api/app/services/auth_service.py create mode 100644 apps/api/app/services/change_detection_service.py create mode 100644 apps/api/app/services/collection_service.py create mode 100644 apps/api/app/services/company_service.py create mode 100644 apps/api/app/services/discovery_service.py create mode 100644 apps/api/app/services/enrichment_service.py create mode 100644 apps/api/app/services/ip_throttle_service.py create mode 100644 apps/api/app/services/monitoring_service.py create mode 100644 apps/api/app/services/notification_destination_service.py create mode 100644 apps/api/app/services/report_markdown.py create mode 100644 apps/api/app/services/report_service.py create mode 100644 apps/api/app/services/scheduling.py create mode 100644 apps/api/app/services/security_email_service.py create mode 100644 apps/api/app/services/snapshot_service.py create mode 100644 apps/api/app/services/source_service.py create mode 100644 apps/api/app/services/system_secret_service.py create mode 100644 apps/api/app/services/turnstile_service.py create mode 100644 apps/api/app/services/unban_service.py create mode 100644 apps/api/app/services/user_api_key_service.py create mode 100644 apps/api/app/tasks/__init__.py create mode 100644 apps/api/app/tasks/base.py create mode 100644 apps/api/app/tasks/celery_app.py create mode 100644 apps/api/app/tasks/collection.py create mode 100644 apps/api/app/tasks/enrichment.py create mode 100644 apps/api/app/tasks/maintenance.py create mode 100644 apps/api/app/tasks/scheduler.py create mode 100644 apps/api/celerybeat-schedule create mode 100644 apps/api/migrations/env.py create mode 100644 apps/api/migrations/script.py.mako create mode 100644 apps/api/migrations/versions/.gitkeep create mode 100644 apps/api/migrations/versions/05a3ccfe49b9_create_users_and_refresh_tokens.py create mode 100644 apps/api/migrations/versions/06e7f03cea36_add_per_source_scheduling_override_.py create mode 100644 apps/api/migrations/versions/484419ccd357_create_sources_source_documents_and_.py create mode 100644 apps/api/migrations/versions/60a25ddfc6a3_add_notification_destination_companies_.py create mode 100644 apps/api/migrations/versions/6d296b533f7c_add_user_api_keys_table.py create mode 100644 apps/api/migrations/versions/79e3aa041131_create_company_enrichments_table.py create mode 100644 apps/api/migrations/versions/7e0adee63b3b_add_password_history_entries_table.py create mode 100644 apps/api/migrations/versions/8880af3b8aa1_create_companies_monitoring_and_.py create mode 100644 apps/api/migrations/versions/8ceefbd0a6a5_create_monitoring_runs.py create mode 100644 apps/api/migrations/versions/8db5bd1c31f8_add_system_secrets_table.py create mode 100644 apps/api/migrations/versions/b5e0d93c89ac_create_alerts_and_notification_.py create mode 100644 apps/api/migrations/versions/c06c845cdd10_create_reports.py create mode 100644 apps/api/migrations/versions/c34c769afc07_add_user_known_ips_table.py create mode 100644 apps/api/migrations/versions/cb61ba9570fb_create_detected_changes.py create mode 100644 apps/api/migrations/versions/ef56f181dbb9_add_email_verification_ip_throttle_ban_.py create mode 100644 apps/api/migrations/versions/f01919a99ee9_add_company_headquarters_and_public_.py create mode 100644 apps/api/pyproject.toml create mode 100644 apps/api/tests/__init__.py create mode 100644 apps/api/tests/collectors/__init__.py create mode 100644 apps/api/tests/collectors/conftest.py create mode 100644 apps/api/tests/collectors/test_custom_url_collector.py create mode 100644 apps/api/tests/collectors/test_fixture_collectors.py create mode 100644 apps/api/tests/collectors/test_github_collector.py create mode 100644 apps/api/tests/collectors/test_gov_contracts_collector.py create mode 100644 apps/api/tests/collectors/test_job_posting_collector.py create mode 100644 apps/api/tests/collectors/test_patents_collector.py create mode 100644 apps/api/tests/collectors/test_rss_collector.py create mode 100644 apps/api/tests/collectors/test_sec_edgar_collector.py create mode 100644 apps/api/tests/collectors/test_website_collector.py create mode 100644 apps/api/tests/conftest.py create mode 100644 apps/api/tests/enrichment/__init__.py create mode 100644 apps/api/tests/enrichment/test_mock_provider.py create mode 100644 apps/api/tests/enrichment/test_ninjapear_provider.py create mode 100644 apps/api/tests/fixtures/acme_mobility/v1/about.html create mode 100644 apps/api/tests/fixtures/acme_mobility/v1/careers.html create mode 100644 apps/api/tests/fixtures/acme_mobility/v1/press.html create mode 100644 apps/api/tests/fixtures/acme_mobility/v1/pricing.html create mode 100644 apps/api/tests/fixtures/acme_mobility/v1/products.html create mode 100644 apps/api/tests/fixtures/acme_mobility/v2/about.html create mode 100644 apps/api/tests/fixtures/acme_mobility/v2/careers.html create mode 100644 apps/api/tests/fixtures/acme_mobility/v2/press.html create mode 100644 apps/api/tests/fixtures/acme_mobility/v2/pricing.html create mode 100644 apps/api/tests/fixtures/acme_mobility/v2/products.html create mode 100644 apps/api/tests/fixtures/patents/acme_mobility.json create mode 100644 apps/api/tests/fixtures/reviews/acme_mobility.json create mode 100644 apps/api/tests/integration/__init__.py create mode 100644 apps/api/tests/integration/test_acme_fixture_demo.py create mode 100644 apps/api/tests/integration/test_alert_service.py create mode 100644 apps/api/tests/integration/test_analytics_service.py create mode 100644 apps/api/tests/integration/test_change_detection_service.py create mode 100644 apps/api/tests/integration/test_collection_service.py create mode 100644 apps/api/tests/integration/test_discovery_service.py create mode 100644 apps/api/tests/integration/test_enrichment_task.py create mode 100644 apps/api/tests/integration/test_maintenance_task.py create mode 100644 apps/api/tests/integration/test_per_source_scheduling.py create mode 100644 apps/api/tests/integration/test_report_service.py create mode 100644 apps/api/tests/integration/test_scheduler_task.py create mode 100644 apps/api/tests/unit/__init__.py create mode 100644 apps/api/tests/unit/test_alerting_e2e.py create mode 100644 apps/api/tests/unit/test_alerts_api.py create mode 100644 apps/api/tests/unit/test_auth.py create mode 100644 apps/api/tests/unit/test_companies.py create mode 100644 apps/api/tests/unit/test_correlation_id.py create mode 100644 apps/api/tests/unit/test_dashboard_api.py create mode 100644 apps/api/tests/unit/test_discover_endpoint.py create mode 100644 apps/api/tests/unit/test_enrichment_service.py create mode 100644 apps/api/tests/unit/test_extraction.py create mode 100644 apps/api/tests/unit/test_extractors.py create mode 100644 apps/api/tests/unit/test_health.py create mode 100644 apps/api/tests/unit/test_ip_throttle_service.py create mode 100644 apps/api/tests/unit/test_llm_providers.py create mode 100644 apps/api/tests/unit/test_mock_llm_provider.py create mode 100644 apps/api/tests/unit/test_monitoring_smoke.py create mode 100644 apps/api/tests/unit/test_noise_filters.py create mode 100644 apps/api/tests/unit/test_notification_destinations.py create mode 100644 apps/api/tests/unit/test_notification_destinations_test_endpoint.py create mode 100644 apps/api/tests/unit/test_notification_providers.py create mode 100644 apps/api/tests/unit/test_rate_limit.py create mode 100644 apps/api/tests/unit/test_report_generation_prompt.py create mode 100644 apps/api/tests/unit/test_reports_api.py create mode 100644 apps/api/tests/unit/test_scoring.py create mode 100644 apps/api/tests/unit/test_search_providers.py create mode 100644 apps/api/tests/unit/test_security_flows.py create mode 100644 apps/api/tests/unit/test_snapshots_api.py create mode 100644 apps/api/tests/unit/test_sources_api.py create mode 100644 apps/api/tests/unit/test_ssrf.py create mode 100644 apps/api/tests/unit/test_structured_diff.py create mode 100644 apps/api/tests/unit/test_system_endpoints.py create mode 100644 apps/api/tests/unit/test_system_secrets.py create mode 100644 apps/api/tests/unit/test_text_diff.py create mode 100644 apps/api/tests/unit/test_unban_admin.py create mode 100644 apps/api/tests/unit/test_user_api_keys.py create mode 100644 apps/api/tests/unit/test_user_known_ips.py create mode 100644 apps/web/.eslintrc.json create mode 100644 apps/web/.prettierignore create mode 100644 apps/web/.prettierrc.json create mode 100644 apps/web/app/(app)/alerts/[id]/page.tsx create mode 100644 apps/web/app/(app)/alerts/page.tsx create mode 100644 apps/web/app/(app)/companies/[id]/page.tsx create mode 100644 apps/web/app/(app)/companies/new/page.tsx create mode 100644 apps/web/app/(app)/companies/page.tsx create mode 100644 apps/web/app/(app)/dashboard/page.tsx create mode 100644 apps/web/app/(app)/layout.tsx create mode 100644 apps/web/app/(app)/reports/[id]/page.tsx create mode 100644 apps/web/app/(app)/settings/page.tsx create mode 100644 apps/web/app/forgot-password/page.tsx create mode 100644 apps/web/app/globals.css create mode 100644 apps/web/app/layout.tsx create mode 100644 apps/web/app/login/page.tsx create mode 100644 apps/web/app/page.tsx create mode 100644 apps/web/app/providers.tsx create mode 100644 apps/web/app/register/page.tsx create mode 100644 apps/web/app/reset-password/page.tsx create mode 100644 apps/web/app/unban-request/page.tsx create mode 100644 apps/web/app/verify-email/page.tsx create mode 100644 apps/web/components/dashboard/analytics-section.tsx create mode 100644 apps/web/components/local-mode-banner.tsx create mode 100644 apps/web/components/page-transition.tsx create mode 100644 apps/web/components/report-view.tsx create mode 100644 apps/web/components/ui/badge.tsx create mode 100644 apps/web/components/ui/company-pill.tsx create mode 100644 apps/web/components/ui/copyable-email.tsx create mode 100644 apps/web/components/ui/form-field.tsx create mode 100644 apps/web/components/ui/notification-channel-box.tsx create mode 100644 apps/web/components/ui/password-field.tsx create mode 100644 apps/web/components/ui/password-strength-meter.tsx create mode 100644 apps/web/components/ui/select.tsx create mode 100644 apps/web/components/ui/stat-card.tsx create mode 100644 apps/web/components/ui/system-secret-row.tsx create mode 100644 apps/web/components/ui/turnstile-widget.tsx create mode 100644 apps/web/components/ui/user-api-key-row.tsx create mode 100644 apps/web/hooks/use-alerts.ts create mode 100644 apps/web/hooks/use-analytics.ts create mode 100644 apps/web/hooks/use-auth.ts create mode 100644 apps/web/hooks/use-companies.ts create mode 100644 apps/web/hooks/use-countdown.ts create mode 100644 apps/web/hooks/use-discovery.ts create mode 100644 apps/web/hooks/use-monitoring-runs.ts create mode 100644 apps/web/hooks/use-notification-destinations.ts create mode 100644 apps/web/hooks/use-reports.ts create mode 100644 apps/web/hooks/use-snapshots.ts create mode 100644 apps/web/hooks/use-sources.ts create mode 100644 apps/web/lib/api-client.ts create mode 100644 apps/web/lib/auth.ts create mode 100644 apps/web/lib/config.ts create mode 100644 apps/web/lib/format.ts create mode 100644 apps/web/lib/types.ts create mode 100644 apps/web/next-env.d.ts create mode 100644 apps/web/next.config.js create mode 100644 apps/web/package-lock.json create mode 100644 apps/web/package.json create mode 100644 apps/web/playwright.config.ts create mode 100644 apps/web/postcss.config.js create mode 100644 apps/web/tailwind.config.ts create mode 100644 apps/web/tests/add-company-wizard.test.tsx create mode 100644 apps/web/tests/alerts-page.test.tsx create mode 100644 apps/web/tests/analytics-section.test.tsx create mode 100644 apps/web/tests/api-client.test.ts create mode 100644 apps/web/tests/auth-forms.test.tsx create mode 100644 apps/web/tests/badges.test.tsx create mode 100644 apps/web/tests/landing-page.test.tsx create mode 100644 apps/web/tests/security-pages.test.tsx create mode 100644 apps/web/tests/setup.ts create mode 100644 apps/web/tests/system-secret-row.test.tsx create mode 100644 apps/web/tests/test-utils.tsx create mode 100644 apps/web/tests/turnstile-widget.test.tsx create mode 100644 apps/web/tests/user-api-key-row.test.tsx create mode 100644 apps/web/tsconfig.json create mode 100644 apps/web/vitest.config.ts create mode 100644 docker-compose.yml create mode 100644 docs/FIREBASE_MIGRATION.md create mode 100644 infrastructure/docker/api.Dockerfile create mode 100644 infrastructure/docker/web.Dockerfile create mode 100644 packages/shared/package.json create mode 100644 packages/shared/src/index.ts diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..b6d1106 --- /dev/null +++ b/.env.example @@ -0,0 +1,152 @@ +## --------------------------------------------------------------------------- +## CI Agent environment configuration +## Copy to .env and adjust. Nothing here is a real secret. +## --------------------------------------------------------------------------- + +# --- App ----------------------------------------------------------------- +APP_ENV=development +APP_NAME=CI Agent +# Comma-separated list of allowed CORS origins. Add your LAN address here +# (e.g. http://localhost:3000,http://192.168.1.190:3000) to reach the app +# from another device on your network - find your LAN IP with `ipconfig` +# (Windows) or `ifconfig`/`ip a` (Mac/Linux). +FRONTEND_URL=http://localhost:3000 +BACKEND_URL=http://localhost:8000 +# What the *browser* uses to reach the API - baked into the frontend build, +# so for LAN access this must be the host's LAN IP, not localhost (e.g. +# http://192.168.1.190:8000). Leave unset for localhost-only access. +NEXT_PUBLIC_API_URL=http://localhost:8000 + +# --- Reverse proxy (only relevant once deployed behind Cloudflare/Nginx) ----- +# Empty = trust the direct connection for client-IP resolution (correct for +# this Docker/local setup - no proxy exists). Set to CF-Connecting-IP once +# behind Cloudflare's proxy, or the IP throttle/ban system and is_localhost +# will treat every visitor as sharing the proxy's own address. +TRUSTED_PROXY_IP_HEADER= + +# --- Local-dev convenience -------------------------------------------------- +# Comma-separated extra IPs that is_localhost treats as loopback-equivalent. +# Needed under Docker Desktop, where even host-originated traffic arrives at +# the containers via the bridge gateway rather than real loopback - find +# yours from a recent ip_throttle_state/user_security_events row, or leave +# blank to keep the strict default (only real 127.0.0.1/::1 count as local). +# Never set this in a real deployment. +ADDITIONAL_TRUSTED_LOCAL_IPS= + +# --- Auth ------------------------------------------------------------------ +# local = single fixed dev user, no login screen required +# jwt = real email/password accounts +AUTH_MODE=local +JWT_SECRET=dev-only-change-me-32-characters-minimum +JWT_ACCESS_TOKEN_MINUTES=15 +JWT_REFRESH_TOKEN_DAYS=7 + +# --- Database ---------------------------------------------------------------- +# Postgres (Docker Compose default): +DATABASE_URL=postgresql+psycopg://ciagent:ciagent@postgres:5432/ciagent +# SQLite fallback for running the API without Docker: +# DATABASE_URL=sqlite+aiosqlite:///./ciagent_dev.db + +# --- Redis / Celery -------------------------------------------------------- +REDIS_URL=redis://redis:6379/0 +CELERY_TASK_ALWAYS_EAGER=false + +# --- LLM --------------------------------------------------------------------- +# mock | anthropic | ollama | gemini +LLM_PROVIDER=mock +ANTHROPIC_API_KEY= +ANTHROPIC_MODEL=claude-sonnet-5 +OLLAMA_BASE_URL=http://host.docker.internal:11434 +OLLAMA_MODEL=llama3.1 +# Free-tier option: create a key at https://aistudio.google.com/apikey +GEMINI_API_KEY= +GEMINI_MODEL=gemini-2.0-flash +LLM_MAX_TOKENS_PER_REQUEST=4000 +LLM_MAX_RETRIES=2 + +# --- Search / company discovery -------------------------------------------- +# mock | brave +SEARCH_PROVIDER=mock +BRAVE_SEARCH_API_KEY= +SERPAPI_API_KEY= +BING_SEARCH_API_KEY= + +# --- Patents ----------------------------------------------------------------- +# Free key via account registration at data.uspto.gov/apis/getting-started +# (now requires ID.me identity verification). PatentSourceCollector falls +# back to its honest disabled/fixture behavior when this is empty. +USPTO_API_KEY= + +# --- Company enrichment (NinjaPear / nubela.co) ------------------------------- +# Paid, per-credit API - get a key from nubela.co/dashboard after +# registering. Only ever called once per company, at creation time (never +# on a recurring schedule) - see app/services/enrichment_service.py. +# Leave empty to skip this feature entirely; nothing else in the app +# depends on it. +NINJAPEAR_API_KEY= +NINJAPEAR_MAX_LEADERSHIP_LOOKUPS=5 + +# --- Email (SMTP) ------------------------------------------------------------ +# Fallback transport when RESEND_API_KEY (below) isn't set - alert emails +# and, if Resend is unconfigured, security emails go through this. Point it +# at any real SMTP relay (e.g. your own mail server, or Resend's own SMTP +# endpoint at smtp.resend.com). Nothing in this stack runs a local catch-all +# mail sink - a real relay (or a real Resend account) is required to +# actually test email delivery locally. +SMTP_HOST= +SMTP_PORT=587 +SMTP_USERNAME= +SMTP_PASSWORD= +SMTP_FROM_EMAIL=alerts@ci-agent.local +SMTP_USE_TLS=true + +# --- Resend (transactional security email: verify/reset/lockout) ------------ +# Unset by default - falls back to the SMTP block above. Get a key from +# resend.com after verifying your sending domain. +RESEND_API_KEY= +RESEND_SECURITY_FROM_EMAIL=security@ciagent.org + +# --- Cloudflare Turnstile (CAPTCHA on register/login/password-reset) -------- +# Unset by default - skipped when the caller is on loopback, or when this +# isn't configured at all (neither here nor via the Settings page's admin +# "Server secrets" box, which takes priority over these when set - see +# app/services/system_secret_service.py). The site key is safe to expose +# publicly; /system/status serves it live to the frontend, so there's no +# separate NEXT_PUBLIC_* build-time variable for it. +TURNSTILE_SITE_KEY= +TURNSTILE_SECRET= + +# --- SMS (optional) ----------------------------------------------------------- +NOTIFICATION_SMS_ENABLED=false +# twilio | telnyx +SMS_PROVIDER=twilio +TWILIO_ACCOUNT_SID= +TWILIO_AUTH_TOKEN= +TWILIO_FROM_NUMBER= +# Telnyx: portal.telnyx.com -> API Keys for TELNYX_API_KEY; the number must +# be assigned to a Messaging Profile (portal.telnyx.com -> Messaging). +TELNYX_API_KEY= +TELNYX_FROM_NUMBER= +SMS_MONTHLY_CAP=50 + +# --- GitHub collector (optional, raises rate limit) -------------------------- +GITHUB_TOKEN= + +# --- Scheduling defaults ----------------------------------------------------- +DEFAULT_TIMEZONE=America/New_York +DEFAULT_MONITORING_FREQUENCY=weekly +MINIMUM_MONITORING_INTERVAL_MINUTES=60 + +# --- Scraper behavior --------------------------------------------------------- +SCRAPER_USER_AGENT=CIAgentBot/1.0 (+https://ci-agent.local/bot) +MAX_PAGES_PER_DOMAIN=25 +SCRAPER_REQUEST_TIMEOUT_SECONDS=30 +SCRAPER_DOMAIN_DELAY_SECONDS=2 + +# --- Cost / abuse controls ----------------------------------------------------- +MAX_COMPANIES_PER_USER=25 +MAX_MANUAL_RUNS_PER_DAY=10 + +# --- Retention & logging ------------------------------------------------------- +DATA_RETENTION_DAYS=365 +LOG_LEVEL=INFO diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..094375a --- /dev/null +++ b/.gitignore @@ -0,0 +1,59 @@ +# --- env / secrets --- +.env +.env.local +*.pem +*.key + +# --- Python --- +__pycache__/ +*.py[cod] +*.egg-info/ +.venv/ +venv/ +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +htmlcov/ +.coverage +*.db +*.sqlite3 +.fixture_state + +# --- Node / Next.js --- +node_modules/ +.next/ +out/ +dist/ +build/ +.turbo/ +coverage/ +*.tsbuildinfo +.eslintcache + +# --- Playwright --- +test-results/ +playwright-report/ +playwright/.cache/ + +# --- Docker / OS --- +.DS_Store +Thumbs.db +# Stray file Git Bash on Windows sometimes creates from a `> nul` redirect +# that doesn't map to the real NUL device the way cmd.exe's does. +nul + +# --- IDE --- +.vscode/* +!.vscode/extensions.json +.idea/ + +# --- Logs --- +*.log +logs/ + +# --- Local dev data --- +apps/api/ciagent_dev.db +mailpit-data/ + +# --- Local agent tooling state (not app source) --- +.claude/ diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..ab70336 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,82 @@ +# Architecture + +## Overview + +``` +apps/web (Next.js) <---REST/JSON---> apps/api (FastAPI) <---> PostgreSQL + | \ + | ---> Redis (Celery broker + cache) + | + Celery worker(s) + Celery Beat + | + collectors -> LLM analysis -> change detection -> notifications +``` + +Everything runs via `docker-compose.yml`: `postgres`, `redis`, `api`, `worker`, `beat`, `web`. Each backend concern (`apps/api/app/*`) is organized by responsibility, not by HTTP route, so services can be reused from Celery tasks and from route handlers alike. + +## Provider abstraction pattern + +Every external dependency that costs money, requires a credential, or might later move to a different vendor (including a future Firebase/GCP migration) sits behind a small `Protocol` interface with a `Mock`/`Console` implementation that requires no configuration and is what the app uses by default: + +| Concern | Interface | Implementations | +|---|---|---| +| Auth | `AuthProvider` (`app/auth/base.py`) | `LocalAuthProvider`, `JWTAuthProvider` | +| LLM | `LLMProvider` (`app/analysis/llm/base.py`) | `MockLLMProvider`, `AnthropicLLMProvider`, `OllamaLLMProvider`, `GeminiLLMProvider` | +| Search | `SearchProvider` (`app/search/base.py`) | `MockSearchProvider`, `BraveSearchProvider` | +| Notifications | `NotificationProvider` (`app/notifications/base.py`) | `ConsoleProvider`, `SmtpEmailProvider`, `TwilioSmsProvider` | +| Source collection | `SourceCollector` (`app/collectors/base.py`) | `WebsiteCollector`, `RssCollector` (incl. Google News auto-discovery), `CustomUrlCollector`, `SecEdgarCollector`, `GithubCollector`, `JobPostingCollector`, `GovContractCollector` (USASpending.gov, free/keyless), `PatentSourceCollector` (real USPTO ODP call when `USPTO_API_KEY` is set, fixture fallback otherwise), `ReviewSourceCollector` (fixture) | +| Company enrichment | `EnrichmentProvider` (`app/enrichment/base.py`) | `MockEnrichmentProvider`, `NinjaPearProvider` (nubela.co, paid/per-credit, real call when `NINJAPEAR_API_KEY` is set) | + +Selection is via environment variables (`LLM_PROVIDER`, `SEARCH_PROVIDER`, `AUTH_MODE`, etc.), read once in `app/core/config.py` (a pydantic-settings `Settings` singleton) and resolved through small factory functions — never imported directly by call sites. This is what makes the Firebase/GCP migration path (see `docs/FIREBASE_MIGRATION.md`) tractable: swapping `AuthProvider` for a Firebase-backed one, or the DB session for a Firestore client, doesn't ripple through route handlers or Celery tasks. + +## Data flow: monitoring run + +1. `MonitorConfiguration.next_run` is reached, or an individual `Source` has its own faster `frequency_type` override that's independently due (Beat), or a user hits "Run Now" (API) → a `MonitoringRun` row is created (`status=queued`) and a `collection.run_monitoring` Celery task is enqueued with the run ID. +2. The task loads active `Source` rows for the company. For a `SCHEDULED` trigger, only sources actually due right now are collected (a source with no override rides the company's own cadence; one with an override uses its own `next_check`, via `SourceRepository.list_due_for_company`) - a `MANUAL` "Run now" always collects every active source regardless of individual cadence. For each, the task calls the matching `SourceCollector.collect()`, going through `app/core/http.safe_fetch()` (SSRF-guarded) for anything network-bound. +3. Each successful collection produces a `SourceDocument` (raw extracted text + metadata) and updates the run's `sources_successful` / `sources_failed` counters. A `Snapshot` (structured summary + hash) is derived per source. +4. `change_detection` compares the new `Snapshot` against the most recent prior one for the same source, layer by layer (hash → structured fields → bounded text diff → optional LLM semantic check), producing a `DetectedChange` with a `significance_score`, `confidence`, and `severity`. +5. `analysis` (LLM) generates/updates the `Report` for the company from the accumulated evidence (`SourceDocument`s + `DetectedChange`s), never inventing facts outside that evidence. +6. For each `DetectedChange` above the user's severity threshold, an `Alert` is created (after dedup/cooldown checks) and `notifications` delivers it to each enabled `NotificationDestination` *linked to that company* (via the `notification_destination_companies` join table - a destination can be shared across several companies, e.g. one email registered once but linked to every company the user monitors), recording a `NotificationDelivery`. A destination with zero remaining company links is garbage-collected when the last linking company is deleted. +7. The frontend polls `MonitoringRun` status and then renders the `Report`/`Alert` once available. + +## Data flow: company discovery (onboarding) + +`POST /api/v1/companies/discover` (`app/services/discovery_service.py`) turns a bare company name into a pre-filled, user-editable profile, and is deliberately kept separate from the two pipelines above: + +- **`SearchProvider`** answers "where should I look" — it returns URLs/snippets, never facts about the company itself. +- **`SourceCollector`** (the same collectors the monitoring pipeline uses) answers "what's actually there" by fetching real pages. +- **`LLMProvider`** only ever *analyzes* evidence already fetched by the two above — it is never prompted with "tell me about company X" from its own training data, matching the evidence-grounded pattern every other analysis task in this app follows (`app/prompts/*`). + +Steps: (1) resolve `official_website` from the user's hint or `search.search(f"{name} official website")`; (2) fetch that homepage via the existing `fetch_with_retries`/`extract_readable_text` helpers; (3) run a few targeted searches (headquarters, competitors, aliases) for more evidence snippets; (4) call `app/prompts/company_profile.py::extract_company_profile` once over all of it; (5) merge — any user-supplied hint wins over the discovered value; (6) call the `website`/`github`/`sec_edgar`/`job_posting` collectors' existing `.discover()` methods against a synthetic, unsaved `CompanyContext` to preview likely sources. **Nothing is persisted by this endpoint** — the wizard's Review step shows the result for editing, and only the final `POST /companies` call (unchanged) writes a `Company` row. Source rows themselves still come from the pre-existing lazy discovery gate in `tasks/collection.py` (first monitoring run, not onboarding) — discovery preview and source persistence intentionally stay two different code paths so a user can preview/abandon without any DB writes. + +## Data flow: company enrichment (paid, onboarding-only) + +`app/services/enrichment_service.py` is a third, deliberately separate data-gathering path from the two above, for a paid third-party vendor (NinjaPear/nubela.co) that bills per-field per-request: + +- Only ever runs **once**, right after a `Company` row is actually committed in `company_service.create_company` — never on a recurring schedule, and never during the discovery *preview* step (which would spend real credits on companies a user might abandon before creating). The enqueue itself (`enrich_company.delay(...)`, routed to its own `enrichment` Celery queue) is skipped entirely unless `NINJAPEAR_API_KEY` is configured, so a user who never opts in gets zero extra background-task volume — same "gate the enqueue on the key, not just the provider" pattern as `PatentSourceCollector.discover()`. +- A `CompanyEnrichment` row (1:1 with `Company`, same shape as `MonitorConfiguration`) is created with `status=pending` synchronously, in the same transaction as company creation, *before* the Celery task even starts — this gives the frontend something real to poll on (`useCompany`'s `refetchInterval`), since without it "not yet enriched" and "never configured" would both look like `enrichment: null`. +- The task (`app/tasks/enrichment.py`) calls several independent NinjaPear endpoints (company details, funding, competitors, products, customer/updates listings, plus capped per-leadership-member work-email/profile lookups) through `EnrichmentProvider`; one failed call is recorded in `errors` and never sinks the others, same principle as `tasks/collection.py`'s per-source loop. Overall `status` becomes `complete`/`partial`/`failed` depending on how many of those independent calls actually succeeded. +- Once present, `CompanyEnrichment.data` feeds into report generation as a `company_enrichment` evidence block (`report_service.py`, `prompts/report_generation.py`) exactly like `company_profile` does — real fetched data, never invented, and skipped entirely (falls back to an honest empty block) if enrichment never ran, is still pending, or failed outright. + +## Change significance scoring (documented, testable formula) + +Implemented in `app/change_detection/scoring.py`. Each detected difference starts with a base weight from its category (e.g. `leadership_change=0.9`, `pricing_change=0.6`, `wording_change=0.15`), then is adjusted by: + +``` +significance = base_weight + * source_trust_score (0.3 - 1.0) + * min(1.0, independent_sources / 2) # corroboration bonus, caps at 2 sources + * focus_match_multiplier (1.3 if matches user's stated focus, else 1.0) + * recency_multiplier (1.0 if new, 0.5 if a repeat of a prior alert) +confidence = f(extraction_confidence, source_trust_score, corroboration) +``` + +`severity` is then a deterministic bucket over `significance * confidence` (see `SEVERITY_THRESHOLDS` in the same module), with a hard rule that `critical` requires `confidence >= 0.7` regardless of score — an uncorroborated single-source signal cannot be labeled Critical. Full unit tests live in `apps/api/tests/unit/test_scoring.py`. + +## Directory layout + +See `PLAN.md` for the top-level file tree. Within `apps/api/app`, packages are: `api/v1` (routers only — thin, no business logic), `auth`, `core` (config, logging, http/SSRF, security), `db` (session, base model), `models` (SQLAlchemy ORM), `schemas` (Pydantic request/response), `repositories` (query layer, one per aggregate), `services` (business logic orchestration), `collectors`, `analysis` (LLM provider + prompt tasks), `change_detection`, `notifications`, `tasks` (Celery task definitions, thin wrappers around `services`), `prompts` (prompt templates + response schemas, one module per analysis task). + +## Why FastAPI routes stay thin + +Route handlers only: parse/validate input (Pydantic), call one service method, map the result/exception to an HTTP response. All ownership checks, business rules, and orchestration live in `services/`, which is what both the API and Celery tasks call — this avoids duplicating authorization or business logic between the two entry points. diff --git a/KNOWN_LIMITATIONS.md b/KNOWN_LIMITATIONS.md new file mode 100644 index 0000000..7c808cc --- /dev/null +++ b/KNOWN_LIMITATIONS.md @@ -0,0 +1,181 @@ +# Known Limitations + +This file is updated as each phase lands. It exists so nothing is silently claimed to work when it doesn't. + +## Scope decisions (see PLAN.md) + +- **Customer review source**: implemented as a real `SourceCollector` interface with a documented fixture/mock adapter, not a live scraper - there's no reliable free public API, and the spec explicitly instructs against fabricating results for sources without one. Swapping in a real provider later means implementing one class against the existing `SourceCollector` protocol. +- **Patent source**: as of Phase 14, `PatentSourceCollector` attempts a real call to USPTO's Open Data Portal when `USPTO_API_KEY` is configured (a free key via account registration), falling back to the same honest fixture/disabled behavior as before when it isn't. See the Phase 14 section below. +- **Nubela/NinjaPear company-enrichment API**: considered and dropped in Phase 14 (believed enterprise-only), then wired up in Phase 15 once the user found and paid for an individual tier that Phase 14's research missed - see the Phase 15 section below. +- **Job posting collector**: generic HTML extraction from a company's own careers page is real. Board-specific APIs (LinkedIn, Indeed, etc.) are not implemented — most require paid access or prohibit automated collection in their terms. +- **JS-rendered page collection (Playwright)**: the spec calls for Playwright for JS-heavy pages. The website/custom-URL collectors in this build use static HTML fetching (httpx + trafilatura/BeautifulSoup) only. A `PlaywrightRenderCollector` would slot in behind the same `SourceCollector` interface; not implemented in this pass to keep the collector container lightweight. Sites that require JS rendering will collect an empty/thin document and the source will be flagged rather than silently failing. +- **PDF report export**: Markdown and JSON export are implemented; PDF export is documented as a later enhancement per the spec. +- **OAuth social login**: not implemented; the spec marks this as an optional later enhancement. `AUTH_MODE=jwt` (email/password) and `AUTH_MODE=local` (dev) cover the MVP. + +## Frontend dependency notes + +- `npm audit` reports a moderate esbuild advisory (dev-server-only, requires a malicious site to reach your local Vite dev/test server while it's running — not exploitable in production or CI) and two high advisories inside Next.js's own vendored `postcss`/`sharp` (used only by the Image Optimization pipeline, which this app does not currently use via `next/image`). Tracked upstream; will clear on Next's next patch release. Re-run `npm audit` after `npm install` to check current status. + +## Infrastructure + +- Docker Desktop must be running before `docker compose up`; this was verified during Phase 1 (see TASKS.md). +- On Windows + Docker Desktop, the frontend dev container's file watcher does not always pick up newly *created* route files through the bind mount (edits to existing files hot-reload fine). If a new page 404s right after adding it, `docker compose restart web` picks it up. This is a bind-mount/watcher quirk, not an app bug. +- **`web`'s `node_modules` *and* `.next` are both anonymous Docker volumes** (`docker-compose.yml`'s `- /app/node_modules` and `- /app/.next` lines, needed so the container's Linux-built native modules and build cache don't get clobbered by the host's bind-mounted `apps/web`). This means adding a new npm dependency and running `docker compose build web` is **not enough** - the anonymous volumes from the container's first-ever `up` persist across rebuilds *and* plain `docker compose restart`, so a new package still 404s as `Module not found`, and (observed live during Phase 11) an **edited existing file can silently keep serving its pre-edit output** even after a restart, because Next dev's on-disk `.next` cache survives the restart untouched. `docker compose restart web` alone is not a reliable way to pick up source changes made while the container was already running under Docker Desktop on Windows - the fix in both cases is the same: `docker compose rm -f -s -v web` (removes the container *and* its anonymous volumes) then `docker compose up -d web` to recreate it clean. The mirror-image gotcha to the Python one below - same root cause (a persistent layer masking a rebuilt image/changed source), different mechanism (anonymous volume vs. stale image tag). + +## Auth (Phase 2) + +- Access/refresh tokens are stored in `localStorage` on the frontend for simplicity (see `apps/web/lib/api-client.ts`). This is a reasonable tradeoff for a local-first MVP but is XSS-exposed compared to httpOnly cookies; a production multi-user deployment should move to cookie-based storage with CSRF protection before going live publicly. +- There is no email verification, password reset, or account lockout after repeated failed logins yet (rate limiting mitigates brute force but doesn't lock the account). Password reset is called out in the spec as a "documented future capability" for the MVP. +- Admin role (`User.is_admin`) exists on the model and the local dev user is admin, but no admin-only endpoints exist yet to gate with `require_admin`. + +## Collection pipeline (Phase 4) + +- Auto-discovery (`collection_service.discover_sources_for_company`) is **not** wired into `POST /companies` yet — it makes several live outbound HTTP calls (robots.txt, sitemap, GitHub search, SEC EDGAR search) which don't belong on the synchronous request path. It's called directly today (see integration tests) and will be triggered as a background job once Celery exists (Phase 5), matching the spec's "System discovers company sources" step in the async pipeline diagram. +- The `SCRAPER_DOMAIN_DELAY_SECONDS` per-domain rate limit and the robots.txt cache are in-process dictionaries. Fine for a single API/worker process; a multi-worker Celery deployment would need a shared store (Redis) for these to be true global limits rather than per-worker ones. +- SEC EDGAR collector stores filing *metadata* (form type, date, accession number, link) rather than parsing full filing document bodies — full-text filing parsing is a meaningfully larger scope (SEC filings are large, structurally inconsistent HTML/XBRL documents) left for a future pass. +- No collector renders JavaScript (no Playwright integration in this pass — see the Non-goals note in PLAN.md). A JS-heavy page will collect whatever static HTML is served, which may be thin or empty; the source will still report success with fewer/emptier documents rather than silently failing. + +## Background processing (Phase 5) + +- **Docker image drift on new Python deps**: `api`, `worker`, and `beat` all build from the same `infrastructure/docker/api.Dockerfile`, but each is a separate image. Adding a dependency to `apps/api/pyproject.toml` and only running `docker compose restart api` leaves `worker`/`beat` on stale images (they'll crash with `ModuleNotFoundError`). Run `docker compose build api worker beat` (or `docker compose up --build`) after any dependency change, not just a restart. +- **`@celery_app.task` vs `@shared_task`**: this codebase defines exactly one Celery app (`app.tasks.celery_app.celery_app`) and every task is bound to it explicitly with `@celery_app.task(...)`. Using `@shared_task` here silently binds the task to Celery's implicit default app instead — which doesn't have our Redis broker or `task_always_eager` config — and `.delay()` will try to connect to a default RabbitMQ broker and fail. Discovered and fixed during this phase; don't reintroduce `@shared_task` in `app/tasks/*`. +- **Nested event loop under eager mode**: Celery's synchronous task functions bridge into the app's async service layer via `asyncio.run(...)`. Under `CELERY_TASK_ALWAYS_EAGER=true` (tests, or any `.delay()` call made from inside an async FastAPI route handler), the task body executes synchronously *inside* the caller's already-running event loop, and a plain `asyncio.run()` raises `RuntimeError`. `app/tasks/base.py::run_async_task` detects this and falls back to running the coroutine in a dedicated thread with its own loop. This only matters for eager/in-process execution; a real worker process never has this problem. +- The per-domain rate limiter and robots.txt cache noted in the Phase 4 section are still in-process only — a multi-worker Celery deployment (`--concurrency` > 1, or multiple worker containers) would need a shared store (Redis) for these to be true global limits. +- No task deduplication beyond the single-active-run-per-company check (`MonitoringRunRepository.get_active_for_company`). There's no distributed lock, so a very tight race (two beat ticks within milliseconds) could theoretically enqueue two runs; not observed in testing and low-impact if it happened (the second run just collects again). + +## Change detection (Phase 6) + +- **No semantic (Layer 4) comparison yet.** The spec describes an optional LLM/embedding-based layer for cases where the text diff is ambiguous. That needs the LLM provider interface, which is Phase 7. Layers 1-3 (hash/structured/text) plus deterministic scoring already produce real, correctly-classified detections; Layer 4 would improve recall on subtle wording-only changes, not correctness of what's already detected. +- **Single-source only.** `independent_source_count` is hardcoded to `1` in `change_detection_service.py` - cross-source corroboration (e.g. a press release *and* a job posting both pointing at the same expansion) is the spec's "cross-source synthesis" task, which is Phase 7 (LLM analysis) scope. Every current detection is scored as uncorroborated, which the formula already discounts significantly by design. +- **Leadership/price extraction are regex heuristics**, not NLP - see the docstring in `app/change_detection/extractors.py`. They catch common phrasings ("named CEO", "$49/month") and will miss unusual ones or produce occasional false positives on lookalike phrasing. Phase 7's LLM-based extraction task is the higher-fidelity replacement/supplement. +- **Structured diff only tracks the URL set**, not deeper structured fields (specific job titles changing wording, individual price line-items on a page with several). A URL/title appearing or disappearing is caught; a price changing on a page that lists ten prices, where the page's overall URL set doesn't change, is caught by the text-diff price extractor instead (which does work across the whole page, not per-item). + +## LLM analysis (Phase 7) + +- **`AnthropicLLMProvider` is now verified live end-to-end** - the user supplied a real `ANTHROPIC_API_KEY` (`claude-sonnet-5`) and, combined with a real `BRAVE_SEARCH_API_KEY`, the full pipeline was driven for real: company discovery (Discover -> Review, real search + real structured extraction), company creation, the pre-existing lazy source-discovery-on-first-run crawl of `stripe.com` (23 items collected across 3 sources, 0 failures), and baseline report generation via a real `api.anthropic.com/v1/messages` call. The resulting report was coherent and correctly epistemically humble - it explicitly named what it could and couldn't substantiate from the evidence actually collected (no fabricated financials/leadership claims). This is the strongest live-verification signal in the whole app: real evidence in, real grounded analysis out, exactly as the architecture intends. **`Ollama` remains structurally complete but unverified** - no local Ollama instance is available in this environment; unit-tested with the client mocked only. Set `LLM_PROVIDER=ollama` + a running Ollama instance to exercise it for real. +- **MockLLMProvider is deliberately non-generic** - each of the six analysis tasks has a purpose-built builder function that reads the evidence block and produces genuinely evidence-derived output (real counts, titles, severities), rather than a reflection-based generic filler. This is intentional (see `app/analysis/llm/mock.py` docstring) but means adding a *new* analysis task requires writing its own mock builder, not just a new Pydantic schema. +- **Tasks A (relevance), B (extraction), and C (synthesis) are implemented and tested but not yet wired into the collection/report pipeline** - Task D (report generation) currently consumes raw `SourceDocument`/`DetectedChange` rows directly rather than pre-filtering through Task A or pre-extracting via Task B/C. Wiring them in would improve report quality (e.g. filtering irrelevant documents before they reach the report prompt) but the report already works correctly without it, since `report_service.py`'s evidence-gathering only pulls documents/changes already scoped to the company. +- **A report is regenerated only on baseline (first evidence) or when a run detects an actual change** - not on every scheduled run, to keep LLM cost proportional to real activity rather than to schedule cadence. See `app/tasks/collection.py`. +- **No token/cost usage tracking or per-user LLM budget yet** - `LLM_MAX_TOKENS_PER_REQUEST` and `LLM_MAX_RETRIES` bound a single call, but there's no aggregate usage dashboard (spec section 30's "Display usage statistics in settings" is Phase 10/not-yet-done). + +## Notifications (Phase 8) + +- **`TwilioSmsProvider` was verified live against a real Twilio account, and the auth/transport layer works correctly** - the user supplied real `TWILIO_ACCOUNT_SID`/`TWILIO_AUTH_TOKEN`/`TWILIO_FROM_NUMBER`, added a real SMS notification destination through the Settings UI, and triggered a real "Send test notification." The request reached `api.twilio.com` with correct auth and was rejected with a real Twilio error (`400`, code `572006`, `"Invalid template name. Trial accounts can only use predefined SMS templates."`), which the app surfaced cleanly inline on the destination row rather than crashing or swallowing it - `TwilioSmsProvider.send()`'s error-passthrough path (previously only respx-mocked) is now confirmed correct end-to-end. + **This exposed a real product-level blocker, not a code bug**: Twilio trial accounts no longer support arbitrary custom SMS body text at all - they restrict outbound SMS to Twilio's own predefined templates (order confirmations, appointment reminders, etc.), confirmed via [Twilio's trial-restrictions docs](https://www.twilio.com/docs/usage/trials). Since every alert message this app sends is dynamically generated (company name, severity, summary - see `app/notifications/message_builder.py`), **custom-body SMS cannot work on a Twilio trial account at all, for any app**, not just this one. There is no code-side workaround. The account needs to be upgraded off trial (a payment method added, converting it to a standard pay-as-you-go account) before real SMS alerts can be verified end-to-end. Once upgraded, the provider itself is expected to work as-is - the failure observed was purely Twilio's trial content policy, not a malformed request. +- **`TelnyxSmsProvider` added as a second SMS vendor, selected via `SMS_PROVIDER=twilio|telnyx`** (`app/notifications/telnyx_sms.py`, `app/notifications/factory.py`) - a plain Bearer-authenticated REST POST to `https://api.telnyx.com/v2/messages`, same no-SDK shape as `TwilioSmsProvider`. Unit-tested with the HTTP call respx-mocked (success, API-error with Telnyx's `errors[]` shape, and not-configured paths). The user then supplied real `TELNYX_API_KEY`/`TELNYX_FROM_NUMBER` credentials and drove a real "Send test notification" through the Settings UI. First attempt correctly surfaced a real `403` (`"Only pre-verified destinations are allowed at this account level"` - Telnyx trial accounts cap outbound SMS to a single [verified destination number](https://support.telnyx.com/en/articles/6988813-verified-numbers)). After the user verified the destination number in the Telnyx portal, a retry got a real `200 OK` from `api.telnyx.com` and the UI showed "Test sent" - **but the text never actually arrived on the phone**. +- **This exposed a real gap: a `200 OK` from `POST /v2/messages` only means "Telnyx accepted the send request," not "the carrier delivered it."** Polling `GET /v2/messages/{id}` afterward (not something the app does automatically - no webhook receiver exists in this dev stack) showed the true outcome: `status: "delivery_failed"`, error `40010` - `"The sending number is not 10DLC-registered but is required to be by the carrier."` US carriers (T-Mobile/AT&T/Verizon) require standard 10-digit long-code numbers sending A2P SMS to go through [10DLC campaign registration](https://developers.telnyx.com/docs/account-setup/levels-and-capabilities/trial) - this is an industry-wide carrier requirement, not a Telnyx or app-specific issue, and Twilio has the identical requirement. There is no code-side workaround; the sending number's 10DLC campaign (or a toll-free number's Toll-Free Verification, a separate and often-faster alternative) must complete registration in the Telnyx portal before delivery will actually succeed. **The app's `send()` methods for both SMS providers report success based only on the initial API response, not eventual carrier delivery status** - a known simplification, not fixed here, since confirming real delivery would require either polling every send (adds latency/cost per message) or a publicly-reachable webhook endpoint (not available in this local dev stack). Worth revisiting if/when this deploys somewhere with a public URL. +- **Per the user's direction, SMS is switched back off (`NOTIFICATION_SMS_ENABLED=false`) while 10DLC registration is pending**, and this now applies consistently everywhere SMS could be triggered - a real gap was found and fixed in the same pass: `alert_service.send_test_notification` (the "Test" button) previously ignored `NOTIFICATION_SMS_ENABLED` entirely and would call a configured SMS provider for real regardless of the kill switch; only the real-alert-dispatch path (`create_alert_for_change`) checked it. Fixed so both paths short-circuit identically - verified live (the "Test" button now returns "SMS delivery is currently disabled..." with zero calls to `api.telnyx.com`, confirmed via API logs before/after). The Settings page's "Add destination" form also now shows a specific inline notice when SMS is selected while disabled, explaining *why* (10DLC registration pending) rather than just that it won't work - `apps/web/app/(app)/settings/page.tsx`. 211 backend tests (up from 210), 18 frontend tests unchanged. +- **No delivery retry.** `NotificationDelivery.attempt_count` exists in the schema for future use, but `alert_service.create_alert_for_change` currently makes exactly one send attempt per destination and records whatever result comes back - a transient SMTP/Twilio failure is recorded as `FAILED` and not retried. A retry task (e.g. exponential backoff via the existing `notifications` Celery queue) would be the natural follow-up. +- **No alert-level dedup/digest across runs.** Change-level dedup/cooldown already exists (Phase 6 - exact-repeat suppression within 24h), but there's no "don't re-alert on the same underlying issue" or "batch several changes from one run into a single digest email" logic; every alert-worthy `DetectedChange` produces its own Alert and its own set of notification sends. +- **Email/SMS body text is not localized or user-customizable** - `app/notifications/message_builder.py` produces one fixed English format per channel. Per-user notification templates are not implemented. +- **`NotificationDestination.verified` is tracked on the model but never set to `true` anywhere** - there's no verification-code/confirmation-link flow yet (a destination is usable for real sends as soon as it's created, gated only by `enabled` + `minimum_severity`). Verification is a natural Phase 10+ hardening item, especially for SMS where sending to an unverified number has cost/abuse implications. + +## Fixture demo + tests (Phase 9) + +- **The switchable live-demo mechanism built in this phase (the dev-only `app/dev/fixtures.py` HTTP-serving/version-switch endpoints, the Settings page's "Acme Mobility demo" panel, `scripts/seed_acme_demo.py`, `scripts/switch_fixture.py`, the SSRF allowlist setting that existed only to support it, and the Playwright E2E spec that drove it through the UI) has since been removed entirely** - it added a live, always-mounted-outside-production HTTP surface and a dedicated SSRF-guard carve-out for a one-off manual demo workflow that wasn't worth the ongoing maintenance/security surface. The underlying fixture HTML files (`apps/api/tests/fixtures/acme_mobility/v1|v2/*.html`) and the backend regression test that reads them straight off disk (`apps/api/tests/integration/test_acme_fixture_demo.py`) are unaffected and remain the automated way this pipeline's baseline→change→alert behavior gets locked in (leadership/price/content-modified/no-change, all four outcomes). + +## Hardening & docs (Phase 10) + +- **Rate limits are per-IP (`slowapi`'s default `get_remote_address` key func), not per-user.** Multiple users behind the same NAT/corporate proxy share a limit bucket. Fine for a local-first MVP; a real multi-tenant deployment behind a load balancer would want to key on the authenticated user id instead (falling back to IP for unauthenticated routes like register/login). +- **The daily manual-run cap (`MAX_MANUAL_RUNS_PER_DAY`) counts per company, not per user.** A user with several companies gets the cap on each independently rather than a combined daily budget. This matches how the setting reads ("manual runs per day") but is worth confirming against actual intent if usage patterns suggest otherwise. +- **Correlation-id propagation into Celery's threaded eager-mode fallback required a real fix, not just a wrapper.** `ThreadPoolExecutor` doesn't copy `contextvars` into the new thread by default, which would have silently dropped `request_id`/`run_id`/`task_id` from log lines whenever a task ran via the threaded fallback path in `app/tasks/base.py::run_async_task` (used under `CELERY_TASK_ALWAYS_EAGER=true` or when `.delay()` is called from inside an already-running event loop). Fixed by explicitly capturing `contextvars.copy_context()` and running the executor call through it - see the comment in `run_async_task`. Worth knowing about if a *new* thread-hopping code path is added elsewhere in the task layer; the same gap would reappear unless it goes through the same helper. +- **Data retention only covers `SourceDocument`.** `Snapshot`, `DetectedChange`, `Report`, `Alert`, and `NotificationDelivery` all persist indefinitely regardless of `DATA_RETENTION_DAYS` - a deliberate scope decision (see `SourceDocumentRepository.delete_older_than`'s docstring and the FK-cascade risk it documents), not an oversight, but it does mean actual storage growth from those tables is unbounded in a long-running deployment. Extending retention to them safely would need cascade-aware deletion (e.g. only delete a `DetectedChange` if no `Alert` references it) rather than a blind age-based purge. +- **No admin/ops endpoint to trigger the retention purge on demand** - it only runs on the Celery Beat schedule (3am UTC daily). Verified live by invoking `app.tasks.maintenance.purge_expired_data.delay()` directly against the Docker stack rather than through an HTTP route. + +## Company discovery & provider completion (Phase 11) + +- **`BraveSearchProvider` is now verified live** - the user supplied a real `BRAVE_SEARCH_API_KEY` and the discover flow was driven through the browser against the live Docker stack; all real `api.search.brave.com` queries (official website, headquarters, competitors, "formerly known as") returned `200 OK` with real result data. This is the one external provider in the whole app that has actually been confirmed working against its real API, not just unit-tested with the HTTP layer mocked. +- **`GeminiLLMProvider`'s structured-output schema had a real bug, found and fixed via this live verification**: `CompanyProfileExtraction.public_identifiers` was originally `dict[str, str]`, which Pydantic turns into an open-ended (`additionalProperties`) JSON schema - the Gemini *Developer* API (free-tier key auth, as opposed to Vertex AI Enterprise mode) rejects that shape outright with `ValueError: additionalProperties is only supported in Gemini Enterprise Agent Platform mode`. This is exactly the class of bug a mocked-SDK unit test cannot catch, since the mock never runs the real schema-conversion code path. Fixed by changing the field to `list[PublicIdentifier]` (a fixed `{key, value}` object shape) in `app/prompts/company_profile.py`, with `discovery_service.py` converting the list back to a `dict[str, str]` for the (unaffected) public API response shape - see `ARCHITECTURE.md`. **Any future Gemini-targeted structured-output schema must avoid free-form `dict`/`Mapping` fields** for this same reason. +- **After that fix, a real `GEMINI_API_KEY` still could not be verified end-to-end** - Google returned `429 RESOURCE_EXHAUSTED` with `limit: 0` for both `generate_content_free_tier_requests` and `generate_content_free_tier_input_token_count` on `gemini-2.0-flash`. A `limit: 0` (not "quota exceeded from usage") typically means the free tier isn't actually active for that API key's Google Cloud project - e.g. the Generative Language API needs to be explicitly enabled in Cloud Console, or the AI Studio account needs to complete a one-time terms/region check - rather than anything this app's code can work around. If you hit this: check [ai.dev/rate-limit](https://ai.dev/rate-limit) for that project, confirm the key was generated from an AI Studio project with the free tier actually enabled, and retry - the request/response plumbing itself (schema, retries, error surfacing) is confirmed correct up to the point Google's API rejects the call. The endpoint fails gracefully either way: `POST /companies/discover` returns a clean error response and the wizard shows "Something went wrong. Please try again." rather than crashing. +- **`MockSearchProvider`'s domain guess is a plausibility heuristic, not a real lookup** (`app/search/mock.py::_guess_domain` - lowercases the name and strips all non-alphanumeric characters, then appends `.com`). For a real company name this often happens to resolve to that company's actual site; for a fictional/test name it can resolve to an unrelated real domain (observed live: `"Acme Mobility"` → `acmemobility.com`, a real parked/for-sale domain). This is expected mock behavior, not a bug - the mock is explicitly documented as synthetic/dev-only, and the monitoring pipeline correctly handles the resulting real-world 0-evidence case by finishing the run with `status=failed` rather than crashing. +- **The discovery endpoint's LLM extraction quality is entirely a function of `MockLLMProvider`'s regex heuristics under the sandbox default** (`app/analysis/llm/mock.py::_build_company_profile`, e.g. `_HQ_RE`/`_FORMERLY_RE`) - it only fills in `headquarters`/`aliases` when the fetched evidence text happens to contain a matching phrasing ("headquartered in...", "formerly known as..."). Real company websites frequently don't phrase things this way, so most discovered profiles will legitimately come back with several blank fields under mock providers - this is why the Review step's blank-field placeholders ("Not found - fill in") are load-bearing UI, not just cosmetic. A real `LLM_PROVIDER` (anthropic/gemini/ollama) with real search evidence would fill these in far more often. +- **No confidence indicator per discovered field.** The wizard shows *that* a field came back blank (via placeholder text) but doesn't distinguish "the LLM was highly confident" from "the LLM guessed from a weak snippet" for fields that were filled in - deliberately deferred (see the plan's out-of-scope section) to avoid scope creep; `sources_consulted` gives basic transparency into what evidence existed without a full confidence UI. +- **Discovery is a real, metered API call** (one `SearchProvider.search()` per targeted query plus one `LLMProvider` call) **and has no per-user quota beyond the flat `5/minute` rate limit** on `POST /companies/discover` - a user re-running discovery repeatedly (e.g. tweaking hints and re-submitting) re-pays the full search+LLM cost each time rather than there being any caching/memoization of prior discovery calls for the same name. +- **`_resolve_official_website` skips a fixed list of reference/social hosts (Wikipedia, LinkedIn, Crunchbase, etc.) when picking from real search results, but only when a better-ranked alternative exists in the same result set** (`app/services/discovery_service.py::_NON_CORPORATE_HOSTS`). Found live: Brave's top result for `"Stripe official website"` was Stripe's Wikipedia article, not `stripe.com`; taking it at face value fed the wrong base domain into every downstream source-preview collector (observed: a broken `en.wikipedia.org/careers` URL in the "sources to monitor" preview). Fixed by preferring the first non-reference-host result when one is present in the top 3, falling back to the top result otherwise (a genuinely obscure company with only a Wikipedia page as a real hit should still resolve to it, not `None`). This is a heuristic over a fixed hostname list, not a general solution - a well-known company whose actual corporate site is itself hosted on one of these domains, or whose only real presence is a listed host not in `_NON_CORPORATE_HOSTS`, wouldn't be caught by this fix. + +## UI polish & bug-fix round (Phase 12) + +- **Competitor cross-linking matches by exact (case-insensitive) company name only** (`apps/web/app/(app)/companies/[id]/page.tsx::monitoredByName`) - it does not check the target company's aliases, nor does it do fuzzy/partial matching. A competitor listed as `"PayPal, Inc."` would not match an already-monitored company named plain `"PayPal"`, and would incorrectly route to the wizard's "add new" path instead of the existing company. Since competitor names come from LLM extraction (or user-typed hints), this can happen whenever the discovered/typed competitor name doesn't exactly match the monitored company's `name` field. Worth revisiting if this becomes a frequent papercut - matching against `aliases` too, or a normalized/fuzzy comparison, would close most of the gap. +- **The "generate report before any evidence exists" case is now flagged with a warning, not prevented.** The Latest Report tab shows an inline notice when `runs.length === 0`, but "Generate report now" is never actually disabled - a user can still click through and get a thin, evidence-free (but honestly-labeled) report. This is intentional: there's no strong reason to hard-block report generation, just to make sure the user understands what they're about to get before they ask for it. +- **The new Snapshots tab has no pagination** (`GET /companies/{id}/snapshots` caps at 50, newest-first, same pattern as monitoring-run history) - for a company with a long monitoring history across many sources, older snapshots beyond the cap are simply not shown. No UI affordance to page past the cap yet; would need one if 50 turns out to be too few for real usage patterns. +- **Snapshot text/structured summaries render as plain text/JSON, not a diff view.** The Snapshots tab shows what a snapshot *contained*, not what *changed* between it and the previous snapshot for the same source - that comparison already exists internally (`DetectedChange` rows, shown via Alerts/Monitoring history) but isn't cross-linked from a snapshot row to the `DetectedChange` it produced (if any). A "view what changed from the prior snapshot" link would be a natural follow-up. + +## Second UI polish & bug-fix round (Phase 13) + +- **Duplicate-name detection is a client-side heuristic, not authoritative.** `findPossibleDuplicate` (`apps/web/app/(app)/companies/new/page.tsx`) strips punctuation and common legal suffixes (Inc/LLC/Corp/etc.) then checks exact or substring match against the user's *currently-loaded* company list - it won't catch a genuinely different-looking name for the same company (e.g. "Facebook" vs "Meta"), and a slow/failed `useCompanies()` fetch means the check silently finds nothing rather than blocking. The backend's `_unique_display_name` is the actual hard guarantee (uniqueness per user, filesystem-style auto-suffix) - the frontend warning is a courtesy to catch obvious cases *before* paying for a real search+LLM discovery call, not a substitute for it. +- **Notification-destination company-linking requires at least one company to exist before a destination can be created** (`company_ids` has `min_length=1` in `NotificationDestinationCreate`) - a brand-new user with zero companies can't pre-register a notification destination; the Settings page's Add form correctly reflects this ("Add a company first…") but it's a real ordering constraint, not just a UI nicety. +- **The one-time migration's dedup step is lossy for `NotificationDelivery` history**: when two destination rows shared the same (user, type, value), the newer duplicate(s) were deleted and their delivery history cascaded away with them (see `TASKS.md` Phase 13 and the migration's own docstring, `60a25ddfc6a3`). Acceptable for this app's current scale/stage - delivery history isn't relied on for anything beyond the Alert detail view - but worth knowing if delivery audit history ever becomes load-bearing. +- **`Company.notification_links` and `NotificationDestination.company_links` both need explicit ORM `cascade="all, delete-orphan"` (not just the DB-level `ON DELETE CASCADE` in the migration) because SQLite - used for local dev and the entire test suite - doesn't enforce foreign-key constraints without an explicit `PRAGMA foreign_keys=ON` this app doesn't set.** This was caught before it became a real bug (tests exercise the SQLite path and would have caught silent cleanup failures), but it's a sharp edge worth remembering for any *future* cascade-delete relationship added to this codebase: an `ondelete="CASCADE"` in the migration alone is not sufficient, the ORM relationship needs its own cascade declaration and the parent object needs the child eager-loaded before deletion (see `CompanyRepository._with_relations`'s comment). +- **Report generation's new `company_profile` evidence block has no per-field confidence/staleness signal.** If a company's discovered profile turns out to be wrong (e.g. a bad search result at onboarding time) or goes stale (the company rebrands, moves HQ), the report will confidently repeat that stale/wrong fact indefinitely, since there's no re-verification step and no way for the report prompt to know the profile data's age or original confidence. Editing the company's profile fields (Configuration tab) is the only way to correct this today. + +## First UI polish & bug-fix round (Phase 12) + +- **Competitor cross-linking matches by exact (case-insensitive) company name only** (`apps/web/app/(app)/companies/[id]/page.tsx::monitoredByName`) - it does not check the target company's aliases, nor does it do fuzzy/partial matching. A competitor listed as `"PayPal, Inc."` would not match an already-monitored company named plain `"PayPal"`, and would incorrectly route to the wizard's "add new" path instead of the existing company. Since competitor names come from LLM extraction (or user-typed hints), this can happen whenever the discovered/typed competitor name doesn't exactly match the monitored company's `name` field. Worth revisiting if this becomes a frequent papercut - matching against `aliases` too, or a normalized/fuzzy comparison, would close most of the gap. +- **The "generate report before any evidence exists" case is now flagged with a warning, not prevented.** The Latest Report tab shows an inline notice when `runs.length === 0`, but "Generate report now" is never actually disabled - a user can still click through and get a thin, evidence-free (but honestly-labeled) report. This is intentional: there's no strong reason to hard-block report generation, just to make sure the user understands what they're about to get before they ask for it. +- **The new Snapshots tab has no pagination** (`GET /companies/{id}/snapshots` caps at 50, newest-first, same pattern as monitoring-run history) - for a company with a long monitoring history across many sources, older snapshots beyond the cap are simply not shown. No UI affordance to page past the cap yet; would need one if 50 turns out to be too few for real usage patterns. +- **Snapshot text/structured summaries render as plain text/JSON, not a diff view.** The Snapshots tab shows what a snapshot *contained*, not what *changed* between it and the previous snapshot for the same source - that comparison already exists internally (`DetectedChange` rows, shown via Alerts/Monitoring history) but isn't cross-linked from a snapshot row to the `DetectedChange` it produced (if any). A "view what changed from the prior snapshot" link would be a natural follow-up. + +## New intelligence sources & per-source scheduling (Phase 14) + +- **`PatentSourceCollector`'s real USPTO branch is now live-verified (Phase 15), and the live pass found real schema bugs that mocked tests couldn't catch** - fixed: the sort field needed to be `applicationMetaData.filingDate`, not `filingDate` (was causing a hard `500`); `inventionTitle`/`filingDate`/`abstractText` all live under `applicationMetaData`, not at the entry's top level (was producing "Untitled patent filing" with no dates for every real result); USPTO returns `404` for "no matching records" rather than `200` with an empty array, now treated as an honest empty `ACTIVE` result instead of a `FAILED` one (same non-error empty-result precedent as `GovContractCollector`). +- **Querying USPTO by assignee/company name is not supported by this endpoint at all** - confirmed by inspecting a real response's full field list (for a query that *did* return 110k+ real results by inventor name), which contains no assignee/organization field anywhere. USPTO's Patent Application Search reliably supports inventor-name and application-number lookups only. **Worked around, not fixed**: `PatentSourceCollector.collect()` now searches USPTO by each of the company's known leadership names instead (from NinjaPear enrichment, `CompanyContext.leadership_names`, threaded through in `collection_service.to_company_context`) - live-verified against Stripe, returning 19 real patent documents by searching its executives' names. This is a real, working signal, but a heuristic one: **there is no way to confirm a patent found this way actually belongs to the monitored company** (vs. a same-named person, or work done at a prior employer) - every such document is trust-scored at 0.5 (vs. what a verified-assignee match would warrant) and its content explicitly states which leadership name it matched on, so the report LLM's confidence labeling reflects the uncertainty rather than treating it as confirmed fact. Two real constraints this implies: (1) a company gets zero patent results until NinjaPear enrichment has actually completed and found a leadership team (no `NINJAPEAR_API_KEY` configured means patents stays empty too, even with a valid USPTO key) - self-heals on the next scheduled collection once enrichment finishes, since leadership names are re-read from the DB on every run, not cached at discovery time; (2) capped at the first 5 leadership names per company (`_MAX_INVENTOR_SEARCHES` in `app/collectors/patents.py`) to bound the number of USPTO calls per run - USPTO itself is free/keyless, so this cap is about politeness and run time, not cost. +- **`GovContractCollector` always reports a source as discoverable for every company**, including obviously-private ones - matching `SecEdgarCollector`'s existing precedent for non-public companies. A private company simply gets zero contract results (not an error); there's no attempt to guess whether a company is likely to have federal contracts before offering the source. +- **Google News RSS's query is just the company's `name` field, url-encoded** - no disambiguation for a company name that collides with something unrelated (e.g. a common word, or a same-named but different company). A company with a highly generic name will get noisy/irrelevant results; the existing evidence-grounded report generation still won't fabricate claims from noise, but the raw collected documents themselves aren't filtered for relevance at the collector layer. +- **Per-source scheduling has no cron/custom-interval UI** - the Sources tab's "Check frequency" `Select` only offers the fixed cadences (hourly through monthly), matching `MONITORING_FREQUENCIES` minus `custom`; the backend (`SourceUpdate` schema, `validate_and_compute_next_run`) fully supports a per-source `custom` override with `interval_minutes`/`cron_expression`, but there's no frontend control to set one - would need the same extra interval/cron inputs the company-level Configuration tab doesn't have either (it also has no dedicated custom-schedule UI beyond raw form fields). +- **A newly-set or newly-cleared per-source override resets `next_check` to `None`, meaning "due on the very next scheduler tick"** rather than computing a real future `next_check` immediately (unlike the company-level `MonitorConfiguration`, which does recompute `next_run` immediately on a schedule change). This is a deliberate choice to match how a brand-new source already behaves (checked ASAP), but it does mean setting a slow cadence (e.g. monthly) on a source still triggers one immediate check before the monthly cadence actually takes effect - there's no way to say "start counting from now, don't check immediately." +- **The scheduler's due-ness check queries every enabled `MonitorConfiguration` and then does a Python-side per-company source scan** (`SourceRepository.company_has_due_work`, called once per enabled company per Beat tick) rather than a single SQL query joining across companies and sources. Fine at this app's scale (a personal/small-team tool with a handful to dozens of companies); would need to move the OR-with-NULL-fallback due-ness logic into SQL if the company count grew large enough for this to become a real per-tick cost. + +## NinjaPear company enrichment (Phase 15) + +- **Live-verified against a real NinjaPear account and a real company (Stripe)** - and the initial implementation, written from a scraped/summarized reading of the JS-rendered `nubela.co/docs`, had several real schema bugs the live pass caught and fixed: identity is `website`-only (there is no name-based company lookup at all - a company with no `official_website` now fails fast with a clear error instead of attempting doomed calls, see `enrichment_service.enrich_company`'s upfront guard); `employee_count` and `industry` come back as raw numbers, not strings; funding's `investors` are objects (`{name, type, website}`), not plain strings; response field names throughout differ from the initial guesses (`executives` not `leadership_team`, `total_funds_raised`/`funding_rounds`/`round_type` not `total_raised`/`rounds`/`round_name`, `competitors[].website`/`competition_reason` not `name`/`reason`, `x_profile_url` not `profile_url`, work-email/profile lookups take `first_name`/`last_name`/`domain` not a single name string); and the credit-balance endpoint is `/api/v1/meta/credit-balance` returning `credit_balance`, not `/company/credit-balance` returning `balance`. All fixed and re-verified live - a real run against Stripe returned real leadership bios, work emails, X profiles, funding history, competitors-with-reasons, live blog updates, and inferred customers, landing on `status: partial` (one funding-parsing bug, since fixed, and one legitimate 404 for a board member not in NinjaPear's database) for 34 real credits. +- **Credit-cost tracking (`CompanyEnrichment.credits_spent`, the Settings-page balance display) is an estimate, not billing-accurate.** Per-call costs in `enrichment_service._CREDIT_COSTS` are transcribed from NinjaPear's published pricing page at a point in time and don't account for per-item add-on charges NinjaPear may apply (e.g. funding's "+1 per investor," customer listing's "+2 per company returned") - these are approximated as flat per-call costs. Useful for a rough sense of spend, not a source of truth; NinjaPear's own dashboard/credit-balance endpoint is the actual authority (which is why `/system/status` also surfaces the real balance separately, not just the estimated spend). +- **The "Similar People" endpoint is not wired in at all** - it's a role/company-anchored prospecting tool ("find people like X at competitor Y") with no natural trigger at onboarding time, since no specific role is ever selected. Would need its own UI (e.g. "find people like this one" on a leadership-team entry) to make sense, not an automatic onboarding call. +- **NinjaPear's suggested competitors are never merged into `Company.competitors`** (the user-typed/reviewed list that drives Phase 12's competitor cross-linking) - shown only in the Enrichment tab, with their stated reasons, kept deliberately separate so a background API call can never silently change what the user explicitly reviewed. A "add as competitor" action would be a reasonable follow-up if this becomes a papercut. +- **Person-level lookups (work email, profile) are capped at `NINJAPEAR_MAX_LEADERSHIP_LOOKUPS` (default 5) per company**, applied to however many leadership members NinjaPear's Company Details call happens to return, in whatever order it returns them - there's no ranking by seniority/relevance before the cap is applied, so for a company with a large leadership team, which specific people get resolved is effectively arbitrary. +- **No re-enrichment mechanism** - by design (see Phase 15's Context in `TASKS.md`, onboarding-only was the explicit scoping decision to bound cost), but it does mean `CompanyEnrichment.data` can go stale indefinitely (a funding round happens, a leadership change occurs) with no way to refresh it short of manually deleting the row and re-triggering via the API/DB directly - there's no "re-enrich this company" button anywhere in the UI. +- **`GET /system/status`'s live credit-balance check adds one outbound HTTPS call (to NinjaPear) to every load of that endpoint** whenever a key is configured - guarded so a failure there can never break the rest of system status (returns `null` for the balance, logs a warning), but it does mean the Settings page's load time now has a dependency on `nubela.co`'s availability/latency for that one field, not just the app's own DB/Redis health. + +## Live logging, API-key visibility, and enrichment-tab formatting (Phase 17) + +- **Localhost detection (`_is_localhost` in `apps/api/app/api/v1/system.py`) does not work as intended under Docker Desktop for Windows/Mac.** It checks `request.client.host in ("127.0.0.1", "::1")` - correct and sufficient for a bare-metal/non-Dockerized deployment, or Docker on native Linux with `network_mode: host`. But under Docker Desktop's default networking (this project's actual `docker compose up` setup on Windows), every published-port connection - whether it originates from the host machine itself (`curl localhost:8000`) or from another machine on the LAN - gets NATed through `docker-proxy` and arrives at the API container sourced from the bridge network's gateway address (observed live: `172.18.0.1`), never from a literal loopback address. Live-verified: a `curl http://localhost:8000/api/v1/system/status` run directly on the Windows host still reports `is_localhost: false`. This was a deliberate fail-closed choice once discovered mid-implementation, not an oversight left unfixed: the alternative (treating the bridge gateway IP as "trusted local") would be actively insecure, since a genuine LAN neighbor hitting the same published port is indistinguishable from the host machine at that same gateway IP - Docker's NAT erases the distinction this feature exists to draw. (The Settings page's old admin-only API Keys box, gated on this check alone, has since been replaced by per-user API keys plus an admin-only-but-not-localhost-gated "Server secrets" box - see the addendum on that below - so this finding's practical impact today is narrower: `ADDITIONAL_TRUSTED_LOCAL_IPS`, see the Phase 19 addendum, is the intended opt-in fix for genuinely-local dev traffic that needs to look local.) +- **NinjaPear competitor/customer `name` fields are provider-supplied and inconsistent in shape** - sometimes a bare URL (`https://paypal.com`), sometimes already a clean name. `companyNameFromUrl` (`apps/web/lib/format.ts`) does a best-effort strip-and-title-case on anything URL-shaped; there's no verification the derived label (e.g. "Staxpayments" from `staxpayments.com`) matches the company's actual public branding (e.g. "Stax Payments") - it's a formatting heuristic, not a lookup against real company names. +- **The Redis-backed log capture (`apps/api/app/core/logging.py`, `app:logs` list, capped at 500) is best-effort and ungated by user/tenant** - any authenticated user of this single-tenant app can see every log line the whole application produced (across all companies, all users, if this app were ever extended to multi-tenant), including other users' company names and error details in the `context` field (secrets are redacted by `_redact_secrets`, but business data isn't). Fine for this app's current single-operator scope; would need per-tenant log scoping before this pattern could ship in a real multi-user product. It's also a plain capped list with no admin-facing way to clear/filter it beyond what's newest, and a Redis flush (`FLUSHDB`/restart) silently empties the whole feed with no persistence. +- **The four API-key-status categories (`internal_error`/`api_error`/`important`/`normal`) are derived purely from structlog level** (`error`/`critical` → red, `warning` → yellow, `info` with an explicit `important=True` kwarg → blue, else white) - this is a coarse mapping, not a semantic classification. A `logger.warning(...)` call that isn't actually about an external API failure (there are a few in this codebase that aren't) still shows up labeled "API error"; nothing currently opts into `important=True`, so the blue category has no real entries yet until a call site is deliberately updated to pass it. + +## Per-connection auth: loopback stays free, everyone else logs in (Phase 18) + +- **Under Docker Desktop's networking, the developer also sees the login screen.** `get_current_user` (`apps/api/app/auth/dependencies.py`) now only hands out the fixed "Local Developer" account when both `AUTH_MODE=local` *and* the request's actual TCP peer is literal loopback (`app.core.security.is_localhost`, `127.0.0.1`/`::1`). Per the Phase 17 finding this same check was built for (API-key visibility), Docker Desktop's port-publishing NATs every request - including the host machine's own browser hitting `localhost:3000` via the Docker-mapped port - through the bridge gateway IP, never literal loopback. That means **running this app via `docker compose` (the documented setup) requires a real registered account for everyone, including the person running the stack**, not just LAN/WAN visitors. This is the correct, secure outcome (the alternative - trusting the Docker bridge IP as "local" - would let a LAN neighbor bypass login too, since they'd be indistinguishable from the host at that IP) but is worth knowing going in: the fastest way to get the zero-login convenience back for local development is running the API directly (`uvicorn app.main:app`, bypassing Docker's NAT hop entirely) rather than through `docker compose`. +- **Accounts are fully data-isolated per user** (existing behavior, unchanged by this phase - every `Company`, `NotificationDestination`, etc. has always belonged to a `user_id`) - a new WAN/LAN visitor who registers starts with zero companies, not a copy of the "Local Developer" account's existing demo data (Stripe, Nvidia, the Acme Mobility fixture, etc.). There's no account-linking or data-migration path from the local-dev user to a real account. +- **The password-strength meter on the register page is a purely client-side visual affordance**, not a second source of truth - it scores length/case/symbol variety heuristically to give live feedback, but the actual enforced policy is still just the existing Zod schema (10+ characters, at least one letter and one digit) mirrored server-side by `RegisterRequest`'s validators. A password can show as "Fair" or "Good" on the meter and still be the minimum-viable accepted password; the meter never blocks or requires more than the schema does. + +## Email verification, escalating lockout/ban, Turnstile, split logging (Phase 19) + +- **`get_client_ip`'s `trusted_proxy_ip_header` must be configured correctly before this app ever sits behind Cloudflare (or any reverse proxy), or IP-based throttling/banning breaks in one of two ways.** Today (`TRUSTED_PROXY_IP_HEADER=` unset), every consumer - `ip_throttle_service`, `is_localhost`, the unban-request cooldown - reads `request.client.host` directly, which is correct only when nothing sits between the client and this app. Once deployed behind Cloudflare's proxy, every request's `request.client.host` becomes Cloudflare's edge IP, not the real visitor's - unset, this either bans/throttles *everyone* behind Cloudflare together as if they were one IP (one abusive visitor locks out every legitimate one), or - if Cloudflare's IP itself gets banned - locks out the entire app. Setting `TRUSTED_PROXY_IP_HEADER=CF-Connecting-IP` fixes this by reading the real visitor IP Cloudflare forwards - but this header must **only** be trusted once it's actually known the proxy is in front (never trust a client-suppliable header blindly); this is a deploy-time config change, not something the code can safely auto-detect. +- **A real production bug was found and fixed during this phase, not just a test artifact**: the original implementation called `ip_throttle_service.record_attempt(..., RESEND_VERIFICATION, ...)` unconditionally on every `POST /auth/register`, intending to start the "first resend allowed in 30s" cooldown immediately. Since IP-level throttling is deliberately IP-scoped (not account-scoped, so one abusive account can't be worked around by re-registering), this meant **every registration from a shared IP counted against the same ladder** - an office, a NAT'd household, or (as caught by the backend test suite sharing one fake IP across ~200 tests) any high-volume signup source would eventually exhaust the resend ladder, escalate through all 6 timeouts, and permanently ban that shared IP, taking down registration/login/reset for everyone behind it. Fixed by removing that call entirely: the escalation ladder for `resend_verification` now only starts from the first *manual* resend click (`POST /auth/resend-verification`), and "first resend allowed in 30s" is enforced client-side only (a UI cooldown) for the very first send, not backend-enforced. This is a real, intentional deviation from the original plan's literal wording ("first resend should be allowed in 30 seconds" was originally read as governing the very first send; it now governs resend #2 onward). +- **IP-level throttling/banning is IP-scoped by design, which has the flip side of the bug above**: a genuinely malicious actor sharing a NAT/VPN egress IP with legitimate users can still get that whole shared IP banned through repeated failed logins or resend/reset spam against real accounts - `record_attempt` on `failed_login`/`resend_reset` is still called for every attempt, since those actions can't skip IP-level protection without losing the abuse-prevention this feature exists for. `TRUSTED_PROXY_IP_HEADER` (see above) narrows the blast radius once real per-visitor IPs are available; a corporate/university NAT with no such header will always share fate at this layer, and the manual `/unban-request` flow is the intended (if annoying) escape hatch. +- **Account lockout and email verification are both keyed by account (`User.email_verified`/`User.locked_at`), not IP** - a locked or unverified account stays locked/unverified from *every* IP until the user resets their password or verifies their email, regardless of which IP most recently triggered the lockout. Only the throttle/ban layer (`ip_throttle_state`/`ip_bans`) is IP-scoped; these are two independent dimensions that happen to escalate in lockstep for the specific case of `LOGIN_BACKOFF_SECONDS`/`LOGIN_LOCKOUT_THRESHOLD` (both derived from the same 11-stage array, see `auth_service.py`), but a determined attacker rotating IPs against one specific account is still stopped by the account-level lock, not just the (bypassable-by-IP-rotation) IP throttle. +- **Security transactional email (verification codes, reset codes, lockout notices) picks the Resend HTTP API when `RESEND_API_KEY` is set, else falls back to the SMTP provider** (`security_email_service.py`) - a misconfigured/expired `RESEND_API_KEY` in production silently degrades to attempting SMTP instead (which will itself fail loudly if `SMTP_HOST` isn't configured either) rather than surfacing a specific "Resend is broken" signal anywhere in the UI - failures are logged (`get_logger`) but not otherwise surfaced to the registering/resetting user beyond the generic success message the enumeration-safe endpoints always return. +- **The bundled local Mailpit container was removed** (originally added in Phase 8/9 as a zero-config local mail sink for alert email, and reused by Phase 19's security email as the no-Resend-account fallback) - the project's real `.env` had already been pointed at Resend's own SMTP relay for both alerts and security email, making Mailpit dead weight in practice (never actually reached by the running app; the ~500 stray messages found in it during Phase 19 testing turned out to be an unrelated side effect of running the local `pytest` suite outside Docker, which defaults `SMTP_HOST` to `localhost` and happened to reach Mailpit's host-published port). Net effect: there is no bundled way to test real email delivery locally anymore without a real SMTP relay or a real Resend account - `.env.example`'s SMTP block now ships blank rather than pointing at a fake `mailpit` hostname. The unban-request admin notification (`unban_service.py`) also moved off a fixed `admin_notification_email` setting onto **every account with `is_admin=True`** (`UserRepository.list_admin_emails`) - multiple admins now all get notified, and the previous fake `admin@ci-agent.local` default (which only ever worked because Mailpit intercepted it) is gone. +- **Turnstile enforcement could not be live-verified by this agent** - Cloudflare Turnstile is explicitly designed to be unsolvable by automation, and this dev environment (Docker) means even the agent's own browser-tool traffic isn't loopback (see the Phase 17/18 Docker-NAT finding above), so the widget is always presented once a `turnstile_secret` is configured (via `.env` or the admin-set override, see the system-secrets addendum below). Full live verification (register/login/forgot-password through the actual widget, plus a real Resend-delivered code if desired) requires a human to solve the challenge - tracked as the final Phase 19 verification step. +- **The public `POST /unban-requests` endpoint has no Turnstile/CAPTCHA of its own** - only a flat `3/minute` rate limit (`slowapi`) and a 24-hour per-IP cooldown enforced at the DB level (`UnbanRequestRepository.within_cooldown`). An IP that is *not yet* banned could still be used to spam the admin's inbox up to the rate limit before the cooldown kicks in on the second request; low-impact (each request is a single email + one log line, and the cooldown caps sustained abuse to one message per IP per day) but a deliberate scope decision, not an oversight - adding Turnstile here would require serving the challenge to a possibly-already-banned visitor, which is awkward given Turnstile itself is skipped for banned/localhost callers elsewhere in this app's model. +- **`GET /auth/security-events` is capped at the 100 most recent events per user** (`UserSecurityEventRepository.list_for_user`, hardcoded `limit=100`) with no pagination - a long-lived, frequently-logged-in account will eventually have older events silently fall off the visible list. No admin-facing purge/retention policy exists for this table either (unlike `SourceDocument`'s `DATA_RETENTION_DAYS`, see the Phase 10 section above) - it grows unboundedly in the DB even though only the newest 100 rows are ever shown. +- **`IpBan.reason` is a short fixed string naming which action type triggered the ban** (`resend_verification`, `resend_reset`, `failed_login`, or `manual_admin_ban` for an admin-initiated ban - see below) rather than a detailed forensic record (no timestamp history of the individual attempts that led to it, no association with which account(s) were being targeted) - sufficient for the admin Settings-page IP Bans panel to show *why*, not a full audit trail. The `user_security_events` table is the closer thing to an audit trail, but it's scoped per-account, not per-IP, so correlating "this IP got banned" with "these were the accounts it was hammering" requires manually cross-referencing `ip_address` across both tables today. +- **Admins can now ban an IP directly** (`POST /admin/ip-bans`, the Settings page's "Ban an IP manually" field) - bypasses the offense-count escalation ladder entirely (`ip_throttle_service`), a deliberate manual override rather than something the automated abuse-detection path produces. Rejects an already-banned IP with 409 rather than silently no-oping, and validates the address is a real IPv4/IPv6 literal (422 otherwise) - it does not accept CIDR ranges or hostnames. +- **Unban requests now have real Accept/Reject actions** (`POST /admin/unban-requests/{id}/accept`, `DELETE /admin/unban-requests/{id}`) instead of being a read-only queue an admin had to separately go find the IP in the bans list to act on. Accept unbans the IP (full pardon - clears throttle state too, not just the ban row) and removes the request; Reject removes the request without touching the ban, so the IP stays blocked. Either way the request is gone from the pending list afterward - there's no "resolved but kept for history" state, so once acted on, a request leaves no trace of the decision beyond whatever happened to the ban/throttle rows themselves. +- **`ADDITIONAL_TRUSTED_LOCAL_IPS` (`app.core.security.is_localhost`) is a narrow, opt-in escape hatch for the Docker-NAT findings above** - a comma-separated list of extra IPs treated as loopback-equivalent, on top of real `127.0.0.1`/`::1`. Added because Cloudflare Turnstile crashes both the Browser pane and the Claude Code process itself when it renders (observed live; the widget is designed to be unsolvable by automation, and this failure mode is worse than a stuck challenge), so an agent's own browser-tool traffic against `localhost:3000`/`8000` needs `is_localhost` to actually resolve true to avoid ever rendering it - matching how a real developer running the stack via `docker compose` would want Turnstile skipped too. Left empty by default (strict). This project's `.env` sets it to the observed Docker bridge gateway IP (`172.18.0.1`) for local dev. **This must never be set in a real deployment** - unlike `TRUSTED_PROXY_IP_HEADER` (which reads a header a trusted proxy controls), this is a flat IP allowlist; if that IP is ever reachable by anyone other than the actual host machine (e.g. a misconfigured network, or a bridge subnet shared with untrusted containers), it would incorrectly grant them the same loopback-only privileges (Turnstile bypass, and - if `AUTH_MODE=local` - the fixed local-dev-user auto-login too, see the Phase 18 section above). + +## Admin-managed server secrets: Turnstile site key/secret move off .env + +- **The old admin-only, localhost-only "API keys" Settings box (`/system/api-keys`, Phase 17) has been removed entirely**, replaced by two independent mechanisms with a deliberately different visibility model each: per-user API keys (Anthropic/Brave/NinjaPear/USPTO - every user manages their own, no admin/localhost gate at all, since each user only ever sees their own value) and a new admin-only (but **not** localhost-gated) "Server secrets" box for values that are genuinely global rather than per-user - today just the Cloudflare Turnstile site key and secret. `SystemSecret` (`app/models/system_secret.py`) is a true singleton-per-key table (one row per `SystemSecretKey`, not scoped to any user), encrypted at rest the same way as `UserApiKey`, with the identical fallback-to-`.env`-when-unset pattern via `system_secret_service.get_effective_settings`. +- **The Turnstile site key is no longer delivered to the frontend via a `NEXT_PUBLIC_*` build-time env var.** It used to be baked into the Next.js bundle at container-build time (`NEXT_PUBLIC_TURNSTILE_SITE_KEY`), which meant an admin-updated value could never take effect without a full frontend rebuild - defeating the point of making it admin-editable. It's now served live by the already-public, unauthenticated `GET /system/status` endpoint (`turnstile_site_key`, resolved through the same effective-settings fallback), and `TurnstileWidget` takes it as a prop instead of reading `process.env` directly. This does mean the site key round-trips through one more network hop (an extra field on a call the login/register pages already make) rather than being inlined - a negligible cost for a value that isn't itself secret. +- **`system_secrets` rows are true global singletons with no natural per-test isolation** (unlike `user_api_keys`, which is naturally isolated by a randomized per-test `user_id`) - `test_system_secrets.py` has to explicitly clear the table before and after every test via an autouse fixture to avoid cross-test pollution within a full-suite run. Worth remembering if this pattern is extended to more server-wide keys later: any new key added to `SystemSecretKey` inherits the same test-isolation requirement. +- **The real `.env` `TURNSTILE_SITE_KEY`/`TURNSTILE_SECRET` values were copied into this DB-backed storage** (both now `configured: true` via the admin Settings box) rather than removed from `.env` outright - the user plans to purge the `.env` values manually once satisfied the DB-backed path works, so both currently agree and nothing changes behaviorally either way. `NEXT_PUBLIC_TURNSTILE_SITE_KEY` specifically *was* removed from `.env`/`.env.example`/`docker-compose.yml`, since nothing reads it anymore (see above) - that one wasn't a "purge later" judgment call, it was genuinely dead as soon as the frontend stopped reading it. + +## Account activity logging expansion: local-dev sign-ins, secret/key updates, known IPs + +- **The local-dev bypass (`AUTH_MODE=local` + loopback) now logs a `login_success` event too**, even though it has no real login step - `get_or_create_local_user` (`app/services/auth_service.py`) runs on every authenticated request for that account, so logging unconditionally would flood Account activity with one event per request. A 30-minute cooldown (`LOCAL_DEV_LOGIN_LOG_COOLDOWN_MINUTES`) treats "no login_success logged for this account in the last 30 minutes" as a proxy for "a new session," rather than trying to detect real session boundaries that don't exist in this bypass. This is a heuristic, not a precise signal - reopening the app twice within 30 minutes shows one sign-in, not two, and an idle tab left open for hours making periodic background requests wouldn't show repeated sign-ins either unless a genuine 30+ minute gap occurs. +- **Updating a Server secret (admin) or one of your own per-user API keys now writes a `server_secret_updated`/`api_key_updated` event to Account activity** (`system_secret_service.set_secret` / `user_api_key_service.set_key`, both now require the acting user's id and IP). Deliberately logged under the *acting* user's own account either way - for Server secrets that's whichever admin made the change, not some app-wide "system" pseudo-account - so an admin's Account activity is also a partial audit trail of their own admin actions. **Clearing a key/secret (setting it blank) is logged identically to setting a real value** - the event type doesn't distinguish "set" from "cleared," only the row's own `configured` state (visible via the Settings page, not the event log itself) tells you which happened. +- **New `user_known_ips` table captures every distinct IP an account has signed in from** (`app/models/user_known_ip.py`), one row per `(user_id, ip_address)` pair with `first_seen_at`/`last_seen_at`, updated on every recorded sign-in (both real login and the local-dev bypass, gated by the same cooldown above for the latter). This is pure data capture for now - **nothing currently reads this table or surfaces it anywhere in the UI**; it exists as the foundation for a possible future "new device/location" security feature, per explicit request. `UserKnownIpRepository.record_login`'s return value (whether the IP was new) is already threaded through but currently unused by any caller. +- **IPs were already being recorded per-login inside `user_security_events`** (every `login_success`/`login_failed` row has always carried `ip_address`) - `user_known_ips` doesn't replace that, it's a deliberately separate, deduplicated view: the security-events log is an append-only history of every attempt, while `user_known_ips` answers "what's the current set of IPs this account has ever used" without needing to scan and dedupe the (much larger, unbounded-growth - see the Phase 19 addendum above) events table. + +Further limitations are appended per-phase below. diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 0000000..cab1d81 --- /dev/null +++ b/PLAN.md @@ -0,0 +1,28 @@ +# CI Agent — Project Plan + +## What we're building + +A Competitive Intelligence monitoring web application. Users register (or run in local single-user dev mode), add companies to monitor, describe what they care about, and choose a monitoring schedule. The system collects public information from multiple sources, stores historical snapshots, uses an LLM to analyze evidence into a structured report, detects meaningful changes between runs, scores them for severity and confidence, and notifies the user by email (and optionally SMS). Everything is viewable from a web dashboard. + +## Build strategy + +Given the size of the full specification, we are building in the phased order below. Each phase leaves the application in a runnable state. We start with the vertical slice explicitly called out in the spec (local auth → fixture company → baseline report → fixture change → alert → Mailpit email → dashboard) and expand outward from there. + +1. **Foundation** — monorepo, Docker Compose, FastAPI + Next.js skeletons, health checks, lint/format/test scaffolding. +2. **Auth & Users** — local dev-mode auth and JWT auth behind one interface. +3. **Company Management** — CRUD, monitoring configuration, dashboard, add-company wizard. +4. **Collection Pipeline** — collector interface, SSRF-safe fetching, website/RSS/custom-URL/SEC EDGAR/GitHub collectors, extraction & normalization. +5. **Background Processing** — Celery + Beat, dynamic per-company schedules, run-now, retries. +6. **Change Detection** — layered hash/structured/text/semantic diffing, significance scoring, severity classification. +7. **LLM Analysis** — provider-agnostic interface (Mock/Anthropic/Ollama), six discrete analysis tasks, evidence-linked reports. +8. **Notifications** — SMTP/Mailpit, Console, Twilio SMS, delivery tracking. +9. **Fixture Demo + Tests** — Acme Mobility Systems fixture company (v1/v2), full automated test suite, Playwright E2E. +10. **Hardening & Docs** — rate limiting, structured logging, retention, security review, Firebase migration doc. + +## Non-goals for this pass + +- Live scraping of sources with no reliable free public API (patents, customer reviews, most job boards) — these get a real interface + documented fixture adapter instead of a fabricated live integration, per the spec's own guidance. +- Payment/billing, multi-tenant admin console, OAuth social login — not in the spec's MVP. +- PDF export — documented as a later enhancement (Markdown/JSON export are implemented). + +See `TASKS.md` for the live checklist, `ARCHITECTURE.md` for system design, `SECURITY.md` for the threat model, and `KNOWN_LIMITATIONS.md` for what's stubbed vs. fully live. diff --git a/README.md b/README.md new file mode 100644 index 0000000..3f476ed --- /dev/null +++ b/README.md @@ -0,0 +1,93 @@ +# CI Agent — Competitive Intelligence Monitoring + +CI Agent monitors companies you choose, collects publicly available information from multiple sources on a schedule, analyzes it with an LLM into an evidence-linked report, detects meaningful changes between runs, and alerts you by email (and optionally SMS) with a severity and confidence score. + +See [`PLAN.md`](PLAN.md) for the build strategy, [`ARCHITECTURE.md`](ARCHITECTURE.md) for system design, [`SECURITY.md`](SECURITY.md) for the threat model, [`TASKS.md`](TASKS.md) for the live implementation checklist, [`KNOWN_LIMITATIONS.md`](KNOWN_LIMITATIONS.md) for what's stubbed vs. fully live, and [`docs/FIREBASE_MIGRATION.md`](docs/FIREBASE_MIGRATION.md) for a (skeptical) look at what moving to Firebase would take. + +## Quick start (Docker) + +```bash +git clone +cd ci-agent +cp .env.example .env +docker compose up --build +``` + +Then visit: + +| Service | URL | +|---|---| +| Frontend | http://localhost:3000 | +| Backend API | http://localhost:8000 | +| API docs (Swagger) | http://localhost:8000/docs | + +The default `.env.example` runs in `AUTH_MODE=local` (no login required, fixed dev user) with `LLM_PROVIDER=mock` and `SEARCH_PROVIDER=mock` — the whole demo workflow works with zero paid API keys. + +`api`, `worker`, and `beat` build from the same Dockerfile but are separate images — after adding a Python dependency, run `docker compose build api worker beat` (not just `restart`) or they'll crash with `ModuleNotFoundError` on stale images. + +Similarly, after adding an npm dependency to `apps/web`, a plain rebuild isn't enough either — `web`'s `node_modules` *and* `.next` are both persistent anonymous Docker volumes that survive `docker compose build web` and even a plain `docker compose restart web`. This means a new package can still 404, and — on Windows + Docker Desktop — an edited existing file can keep serving its pre-edit output after a restart, since the on-disk `.next` build cache isn't cleared by a restart. Run `docker compose rm -f -s -v web && docker compose up -d web` to actually pick up new packages or force a clean recompile. + +## Running without Docker + +**Backend:** + +```bash +cd apps/api +python -m venv .venv +./.venv/Scripts/activate # or `source .venv/bin/activate` on macOS/Linux +pip install -e ".[dev]" +cp ../../.env.example ../../.env # edit DATABASE_URL to the sqlite line if you don't have Postgres running +alembic upgrade head +uvicorn app.main:app --reload +``` + +**Frontend:** + +```bash +cd apps/web +npm install +npm run dev +``` + +## Common commands + +```bash +# Database migrations +cd apps/api && alembic upgrade head +cd apps/api && alembic revision --autogenerate -m "description" + +# Run a Celery worker + beat (only needed outside Docker) +cd apps/api && celery -A app.tasks.celery_app worker --loglevel=INFO -Q default,collection,analysis,notifications,maintenance +cd apps/api && celery -A app.tasks.celery_app beat --loglevel=INFO + +# Backend tests / lint / format +cd apps/api && pytest +cd apps/api && ruff check app tests +cd apps/api && black app tests + +# Frontend tests / lint / format / typecheck +cd apps/web && npm test +cd apps/web && npm run lint +cd apps/web && npm run format +cd apps/web && npm run typecheck + +# End-to-end test (requires `docker compose up -d` already running) +cd apps/web && npx playwright install --with-deps chromium # one-time +cd apps/web && npm run e2e +``` + +## Configuration + +All configuration is via environment variables — see [`.env.example`](.env.example) for the full list with comments. Highlights: + +- `AUTH_MODE=local|jwt` — local single-user dev mode vs. real email/password accounts. +- `LLM_PROVIDER=mock|anthropic|ollama|gemini` — set `ANTHROPIC_API_KEY`/`ANTHROPIC_MODEL`, `OLLAMA_BASE_URL`/`OLLAMA_MODEL`, or `GEMINI_API_KEY`/`GEMINI_MODEL` to go live. Gemini has a genuine free tier — grab a key at [aistudio.google.com/apikey](https://aistudio.google.com/apikey) — so it's the cheapest provider to actually try against a real model. +- `SEARCH_PROVIDER=mock|brave` — set `BRAVE_SEARCH_API_KEY` to go live. +- `SMTP_HOST`/`RESEND_API_KEY` — no bundled local mail sink; point SMTP at a real relay or set `RESEND_API_KEY` to actually test email delivery (alerts and security email both use this). +- `NOTIFICATION_SMS_ENABLED=false` by default — set to `true` and provide `TWILIO_*` to enable SMS. + +Every paid/external provider defaults to a mock/console implementation. Automated tests always run against mocks and never call a paid API. + +## Known limitations + +See [`KNOWN_LIMITATIONS.md`](KNOWN_LIMITATIONS.md). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..f6b6cd0 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,65 @@ +# Security + +## Threat model summary + +This app fetches user-submitted and discovered URLs from the public internet on a schedule, stores API keys for third-party providers, and sends emails/SMS. The main risks are: SSRF via URL fetching, credential leakage, cross-user data access, and abuse of the scraping/LLM/SMS pipeline as a cost or spam vector. + +## SSRF prevention (`app/core/http.py::safe_fetch`) + +All outbound collector/custom-URL requests go through one function. It: + +- Only allows `http`/`https` schemes (no `file://`, `ftp://`, `gopher://`, etc.). +- Resolves DNS itself before connecting, and rejects the request if any resolved address is loopback, link-local, private (RFC1918/RFC4193), multicast, or a known cloud metadata address (`169.254.169.254`, etc.). +- Disables automatic redirect following in the HTTP client and instead re-validates each redirect target against the same rules before following it, up to a small max-hop count. +- Applies a per-request timeout and a per-domain rate limit / delay (`SCRAPER_DOMAIN_DELAY_SECONDS`). +- Sends a configurable, identifying `User-Agent` (`SCRAPER_USER_AGENT`) rather than impersonating a browser. + +No collector or route handler is allowed to call `httpx`/`aiohttp` directly for a user- or discovery-supplied URL — code review should reject that pattern. The guard has zero exceptions: every resolved private/loopback/link-local/metadata address is rejected unconditionally, with no allowlist. + +## Secrets + +- All secrets are read from environment variables via `app/core/config.py`. Nothing is hard-coded. +- `.env` is git-ignored; `.env.example` ships with empty/placeholder values only. +- Secrets are never logged. The structured logger has a redaction filter for known secret-shaped keys (`*_key`, `*_secret`, `*_token`, `*password*`). +- The frontend never receives backend API keys; all third-party calls happen server-side. +- `NotificationDestination.destination_value` (email/phone) is treated as PII, not a secret, but is still excluded from audit logs beyond a masked form. + +## AuthN/AuthZ + +- Passwords hashed with Argon2 (`passlib[argon2]`), never stored or logged in plaintext. +- JWT access tokens are short-lived (default 15 min); refresh tokens are longer-lived, stored hashed, and rotated on use. +- Every service-layer query that loads a `Company`, `Report`, `Alert`, `Source`, or `NotificationDestination` filters by `owner_user_id` — enforced in the repository layer, not just the route layer, so a missed check in one route can't leak data. +- Auth endpoints are rate-limited (`slowapi`) to blunt credential stuffing. Per-IP request-rate limits also apply to every other mutation endpoint that creates data or costs money to call — company/source/notification-destination creation, report generation (LLM call), notification-destination test-send (real SMTP/Twilio call), and run-now — on top of (not instead of) the business-logic caps in "Cost/abuse controls" below. See `@limiter.limit(...)` usages across `app/api/v1/*.py`. +- `AUTH_MODE=local` is for local development only; the app refuses to start in this mode when `APP_ENV=production`. +- **Notification destinations are not verified.** `NotificationDestination.verified` exists on the model but nothing ever sets it to `true` - a destination is usable for real alert sends as soon as it's created, with no confirmation-code/link flow. Combined with SMS being real-money-per-message once `NOTIFICATION_SMS_ENABLED=true` and Twilio is configured, this means a user could point alerts at a phone number or inbox they don't control. Rate limiting on the `/test` endpoint (above) narrows the abuse window but doesn't close it; a verification flow is the real fix and is tracked in `KNOWN_LIMITATIONS.md` as unimplemented. + +## Ethical/legal collection rules + +Enforced by the collector layer, not left to convention: + +1. `robots.txt` is fetched and honored before crawling a domain (website collector). +2. No collector attempts to bypass a CAPTCHA, login wall, paywall, or anti-bot control. If a source requires that, the `Source.status` is set to `AUTH_REQUIRED`/`BLOCKED_BY_POLICY` and the failure is recorded, never silently skipped or faked. +3. Only publicly reachable URLs are collected — `safe_fetch` also serves as the enforcement point for "no private/internal targets." +4. Per-domain delay + retry-with-backoff avoids hammering a source. +5. Content hashing avoids redundant re-fetches of unchanged pages. +6. A run that partially fails is marked `partial`, not silently reported as complete — the report generator is told which sources failed so it doesn't imply completeness it doesn't have. + +## Input validation & injection + +- All request/response bodies are Pydantic models with explicit types and length limits. +- SQLAlchemy ORM with parameter binding throughout — no raw string-interpolated SQL. +- LLM output is only ever deserialized into a strict Pydantic schema (`generate_structured`); free-text LLM output is never concatenated into a shell command, SQL string, or HTML without escaping, and is rendered in the frontend as text, not `dangerouslySetInnerHTML`. +- Request bodies are size-limited at the ASGI layer. + +## Cost/abuse controls + +Per-user limits (configurable, enforced in `services/`): max monitored companies, max manual "run now" triggers per day, max pages crawled per run (`MAX_PAGES_PER_DOMAIN`), max LLM tokens per request, max LLM retries, monthly SMS cap. Paid providers (Anthropic, Brave, Twilio) are opt-in via explicit env configuration and default to their Mock/Console counterparts, so a fresh checkout cannot incur cost by accident, and automated tests never hit a paid provider (they run under `LLM_PROVIDER=mock`, `SEARCH_PROVIDER=mock`, `NOTIFICATION` console-only fixtures). + +## Observability & data retention + +- Every HTTP request and every Celery task run carries a correlation id (`request_id` for HTTP, `task_id` for Celery, plus `run_id` for monitoring runs) bound into structlog's contextvars for the duration of the request/task, so every log line emitted while handling it can be grepped together - see `app/main.py::correlation_id_middleware` and the `bound_contextvars(...)` calls in `app/tasks/*.py`. The HTTP middleware also echoes the id back as an `X-Request-ID` response header, reusing an inbound one from a gateway if present. +- `DATA_RETENTION_DAYS` (default 365) is enforced by a daily Celery Beat task (`app.tasks.maintenance.purge_expired_data`, 3am UTC) that deletes `SourceDocument` rows - the raw collected text - older than the window. Only `SourceDocument` is in scope: nothing else in the schema has a foreign key onto it, so this can never cascade-delete a `Snapshot`, `DetectedChange`, `Alert`, or `Report` a user might still want to see (see the docstring on `SourceDocumentRepository.delete_older_than`). + +## Reporting a vulnerability + +This is a local/dev-stage project; if you find an issue, open an issue in the repository describing the problem and reproduction steps rather than exploiting it further. diff --git a/TASKS.md b/TASKS.md new file mode 100644 index 0000000..18734ab --- /dev/null +++ b/TASKS.md @@ -0,0 +1,171 @@ +# Tasks + +Legend: `[ ]` pending, `[x]` done, `[~]` partial/stubbed (see KNOWN_LIMITATIONS.md). + +## Phase 1 — Foundation +- [x] Monorepo layout (`apps/web`, `apps/api`, `packages/shared`, `infrastructure`, `scripts`, `docs`) +- [x] PLAN.md, ARCHITECTURE.md, TASKS.md, SECURITY.md, README.md +- [x] `.env.example`, `.gitignore` +- [x] `docker-compose.yml` (postgres, redis, mailpit, api, worker, beat, web) — verified with `docker compose up --build` +- [x] FastAPI skeleton with `/health`, `/ready` (+ `/api/v1/system/status`) — pytest passing +- [x] Next.js skeleton with landing page shell — build/lint/typecheck/test passing +- [x] Backend lint/format (ruff, black) + pytest scaffolding +- [x] Frontend lint/format (eslint, prettier) + vitest scaffolding + +## Phase 2 — Auth & Users +- [x] User model + Alembic migration (`users`, `refresh_tokens`) +- [x] `AuthProvider`-equivalent dependency (`get_current_user`): `LocalAuthProvider` and `JWTAuthProvider` behavior behind one seam +- [x] `/auth/register`, `/auth/login`, `/auth/refresh` (rotating), `/auth/logout`, `/auth/me` +- [x] Protected-route dependency (`get_current_user`, `require_admin`) — per-user isolation enforced at repository layer going forward +- [x] Auth rate limiting (slowapi; disabled under `APP_ENV=test`, verified by a dedicated test) +- [x] Frontend: login, register (RHF+Zod), local-mode banner, `useAuth` hooks, dashboard shell with auth guard — verified live against Dockerized API (Postgres) and 14 backend + 6 frontend tests passing + +## Phase 3 — Company Management +- [x] Company, CompanyAlias, Competitor, MonitorConfiguration, NotificationDestination models + migrations +- [x] Companies CRUD + pause/resume endpoints (`/run` moves to Phase 5 alongside MonitoringRun/Celery) +- [x] Dashboard page (summary cards, schedule, system health, recent companies) — real data, no placeholders +- [x] Companies list page (table, pause/resume, delete with confirm) +- [x] Add-Company wizard (5 steps: company, focus templates, schedule, notifications+consent, review) +- [x] Company detail page (Overview + Configuration functional; Report/Alerts/Sources/History/Snapshots tabs show phase-appropriate empty states) — verified live end-to-end (create → dashboard → list → detail → configuration) against Dockerized Postgres API; 33 backend + 6 frontend tests passing + +## Phase 4 — Collection Pipeline +- [x] `SourceCollector` protocol + `safe_fetch`/`fetch_with_retries` SSRF guard (DNS pre-resolution, redirect re-validation, domain rate limiting) +- [x] Website collector (sitemap + heuristic pages, robots.txt) +- [x] RSS/Atom collector +- [x] Custom URL collector +- [x] SEC EDGAR collector (CIK lookup + recent 10-K/10-Q/8-K metadata) +- [x] GitHub collector (org search + repo metadata) +- [x] Job posting collector (generic HTML link-heuristic extraction real; board-specific APIs stubbed, see KNOWN_LIMITATIONS.md) +- [x] Patent source interface (fixture adapter only, documented, never fabricates) +- [x] Review source interface (fixture adapter only, documented, never fabricates) +- [x] Extraction/normalization (trafilatura + BeautifulSoup fallback, whitespace normalization, content hashing, URL canonicalization, cross-page dedup) +- [x] Source, SourceDocument, Snapshot models + migration +- [x] Sources API (list/create/update/delete/test) with ownership isolation, wired into the company detail page's Sources tab +- [x] `collection_service` (discover_sources_for_company, collect_source) — 22 collector tests + 9 integration tests, all respx-mocked (no live network in the suite); verified live end-to-end against Dockerized Postgres + real `example.com` fetch + +## Phase 5 — Background Processing +- [x] Celery app + queues (default/collection/analysis/notifications/maintenance) — bound explicitly via `celery_app.task(...)`, not `@shared_task` (see KNOWN_LIMITATIONS.md for why that distinction mattered) +- [x] Celery Beat dynamic schedule sync from `MonitorConfiguration` (`sync_schedules`, runs every minute, no static per-company entries needed) +- [x] MonitoringRun model + status lifecycle (queued → running → successful/partial/failed) + migration (incl. the deferred `snapshots.monitoring_run_id` FK from Phase 4) +- [x] Run-now endpoint → enqueue without disrupting schedule (`last_run` updates always; `next_run` only advances for scheduled-trigger runs) + idempotent (active run returned, not duplicated) +- [x] Retry/backoff policy (task-level `max_retries`; per-source failures caught and recorded without failing the whole run) +- [x] Run status polling in frontend (2s interval while queued/running) + Monitoring history tab + "Run now" wired on both list and detail pages +- [x] Verified live with the *real* worker/beat/Redis stack (not eager mode): task received, HTTP collection executed, run completed — confirmed via docker logs and API polling + +## Phase 6 — Change Detection +- [x] Hash comparison layer (short-circuits before any diffing when unchanged) +- [x] Structured field diff layer (added/removed item sets between consecutive snapshots) +- [x] Bounded text diff + noise filters (timestamps/cookies/copyright/counters stripped before diffing; capped output size) +- [~] Semantic comparison layer — deferred to Phase 7 (needs the LLM provider interface); Layers 1-3 + deterministic scoring are sufficient to ship real detections now +- [x] Significance scoring (documented formula in `app/change_detection/scoring.py`, unit tested — 9 tests covering trust/corroboration/focus-match/repeat/diff-ratio scaling) +- [x] Severity classification (deterministic buckets + hard confidence floor for Critical, unit tested) +- [x] DetectedChange model + migration +- [x] Change-level dedup/cooldown (exact-repeat suppression within a 24h window; repeats that aren't exact still recorded but dampened) — Alert-level dedup/digest is Phase 8 scope +- [x] Wired into the monitoring run task; verified live end-to-end in Docker (real worker, real Postgres) — 34 new tests (scoring/structured-diff/text-diff/noise-filters/extractors/service integration), 130 total passing + +## Phase 7 — LLM Analysis +- [x] `LLMProvider` interface, Mock/Anthropic/Ollama implementations (Anthropic via forced tool-use, Ollama via JSON mode; both have a bounded repair loop, tested with the SDK/HTTP layer mocked) +- [x] Task A: document relevance +- [x] Task B: fact/signal extraction +- [x] Task C: cross-source synthesis +- [x] Task D: report generation (evidence gathered from real DB rows, never invented) +- [x] Task E: change significance (narrative only — severity itself stays deterministic per ARCHITECTURE.md) +- [x] Task F: alert summarization +- [x] Report model (JSON + Markdown) + Report page UI (16 sections, copy/print/export-JSON, Markdown-with-sources toggle) — wired into monitoring runs (baseline on first evidence, update when a run detects a change) and verified live end-to-end in Docker (real Postgres, real report generated from real collected evidence) + +## Phase 8 — Notifications +- [x] `NotificationProvider` interface: Console, SMTP (Mailpit, stdlib `smtplib` via `asyncio.to_thread`), Twilio SMS (plain REST API, no SDK dependency) +- [x] Alert model + NotificationDelivery model + migration +- [x] `alert_service.create_alert_for_change`: two independent thresholds by design — `MonitorConfiguration.severity_threshold` gates whether an Alert is created at all; each `NotificationDestination.minimum_severity` separately gates whether that destination is notified — wired into the monitoring run task right after change detection +- [x] Alerts API (list with company/severity/read/resolved filters, detail with delivery statuses, PATCH + mark-read/resolve actions) — ownership-scoped +- [x] `POST /notification-destinations/{id}/test` endpoint for a real send-path test (not just CRUD) +- [x] Alerts page (filters, unread/resolved indicators, mark read/resolve actions, detail view with delivery status per destination) +- [x] Settings page (notification destination management incl. per-destination minimum-severity/enabled/test/delete, system configuration incl. LLM/search provider, SMS-enabled flag, DB/Redis health) +- [x] 26 new backend tests (providers, alert_service threshold/dispatch/failure-recording, alerts API ownership+filters, notification-destination test endpoint, one full HTTP-driven E2E: two real monitoring runs → detected change → alert → email) — 175 total passing +- [x] Verified live in Docker: Alembic migration applied against real Postgres, worker/beat rebuilt and healthy, a real SMTP send round-tripped through the live Mailpit container, and both new frontend pages verified in-browser against seeded data +- [x] `TwilioSmsProvider` re-verified against a real Twilio account (user supplied real credentials): auth/transport/error-surfacing all confirmed correct end-to-end (a real `400` from `api.twilio.com` was correctly caught and displayed inline, not swallowed). Uncovered a real product-level blocker rather than a bug: Twilio trial accounts only permit predefined message templates, not the dynamically-generated alert text this app sends — see `KNOWN_LIMITATIONS.md`. Sending a real SMS alert end-to-end requires the user's Twilio account to be upgraded off trial first. +- [x] Added `TelnyxSmsProvider` (`app/notifications/telnyx_sms.py`) as a second SMS vendor option, selected via new `SMS_PROVIDER=twilio|telnyx` setting (`app/notifications/factory.py` now routes on it — same pattern as `LLM_PROVIDER`/`SEARCH_PROVIDER`); `SystemStatusResponse`/`/system/status` and the Settings page's system-configuration panel now show which SMS provider is active. 4 new backend tests (not-configured, real-send, API-error-with-Telnyx's-`errors[]`-shape, factory routing by `sms_provider`) — 210 backend tests total, 18 frontend tests unchanged. +- [x] `TelnyxSmsProvider` driven live against a real Telnyx account: a real `403` ("only pre-verified destinations allowed") was caught and displayed cleanly; after the user verified the destination number, a retry got a real `200 OK` from `api.telnyx.com` — but the text never arrived. Polling the message afterward revealed the real cause: `delivery_failed`, error `40010`, the sending number isn't 10DLC-registered (a US carrier requirement, industry-wide, not app-specific). No code fix possible — needs 10DLC/toll-free registration completed in the Telnyx portal. See `KNOWN_LIMITATIONS.md`. +- [x] Fixed a real gap found in the same pass: `alert_service.send_test_notification` didn't respect `NOTIFICATION_SMS_ENABLED`, so the "Test" button could fire a real SMS API call even with SMS globally disabled — only real alert dispatch checked the flag. Fixed so both paths gate identically; verified live (API logs show zero calls to `api.telnyx.com` after the fix). Settings page now shows an inline notice on the phone-number field explaining SMS is paused for 10DLC registration, not just that it's off. `NOTIFICATION_SMS_ENABLED` set back to `false` per the user's direction while registration is pending. 211 backend tests (up from 210), 18 frontend tests unchanged. + +## Phase 9 — Fixture Demo + Tests +- [x] Acme Mobility Systems fixtures v1/v2 (about/products/careers/press/pricing) — real HTML under `apps/api/tests/fixtures/acme_mobility/`, versions differ by a leadership change, a price increase, a new job posting, and an expansion press release; `products` is unchanged v1→v2 to demonstrate the hash-based short-circuit +- [x] Dev-only fixture server (`app/dev/fixtures.py`, mounted only when `APP_ENV != production`) serving those pages over real HTTP, plus a narrow single-hostname SSRF allowlist (`Settings.demo_fixture_host`, unset by default, never honored in production) so the collectors can fetch them +- [x] `scripts/seed_acme_demo.py` (creates the demo company + 5 sources + notification destination, idempotent) and `scripts/switch_fixture.py` (v1/v2/status) + a matching dev-only "Acme Mobility demo" panel on the Settings page (only shown in local auth mode) +- [x] `tests/integration/test_acme_fixture_demo.py` — locks the exact demo scenario (real fixture files off disk, respx-mocked HTTP) into the automated suite: leadership/price/content changes detected correctly, `products` correctly produces no change +- [x] SSRF allowlist unit tests (`tests/unit/test_ssrf.py`) — exact-hostname-only bypass, production-mode ignores it entirely +- [x] Frontend component tests: `SeverityBadge`/`CompanyStatusBadge` rendering, `AlertsPage` (company-name resolution, severity-filter refetch) +- [x] Playwright E2E (`apps/web/e2e/acme-demo-flow.spec.ts`): full UI-driven workflow against the live Docker stack — add company via the wizard → add fixture sources → baseline run (v1) → switch fixture to v2 via the Settings panel → run again → real alert appears on the Alerts page → real email confirmed in Mailpit. Verified passing against the live stack, not just written. +- [x] Backend/frontend regression check: full suites re-run after every addition (179 backend, 14 frontend, all passing) + +## Phase 10 — Hardening & Docs +- [x] Rate limiting beyond auth — `@limiter.limit(...)` added to company/source/notification-destination creation, notification-destination test-send, and report generation; verified live (22 rapid company-creation calls → 20×201 then 2×429) +- [x] Manual "run now" daily cap actually enforced — `MAX_MANUAL_RUNS_PER_DAY` was defined in `Settings` since Phase 1 but never wired up until now (`monitoring_service.enqueue_run_now`, new `RateLimitedError` → HTTP 429) +- [x] Structured logging + correlation IDs — `request_id` (HTTP, echoed as `X-Request-ID`) and `run_id`/`task_id` (Celery tasks) bound via structlog contextvars; fixed a real context-propagation gap in the threaded eager-mode fallback (`app/tasks/base.py`) along the way — verified live (header round-trips, task log line carries `task_id`) +- [x] Data retention job — `app.tasks.maintenance.purge_expired_data`, daily Celery Beat task, purges `SourceDocument` rows older than `DATA_RETENTION_DAYS`; scope deliberately limited to the one table with no incoming FK (see KNOWN_LIMITATIONS.md) — verified live via direct `.delay()` invocation against the real worker +- [x] SECURITY.md review pass — added the demo-fixture SSRF allowlist exception, the expanded rate-limiting surface, the notification-destination verification gap, and a new Observability & data retention section +- [x] README run/setup instructions finalized — fixed stale `python -m scripts.*` references from Phase 1 planning that never matched what got built; added the real Acme demo walkthrough and E2E instructions +- [x] `docs/FIREBASE_MIGRATION.md` — honest architecture-fit assessment (auth maps over cleanly; scheduling, cascading deletes, and cross-collection filtering do not), effort estimate, and an explicit recommendation against a full migration +- [x] `KNOWN_LIMITATIONS.md` final pass — Phase 10 section added; all prior phases' sections already up to date +- [x] Full regression check: 185 backend tests, 14 frontend tests, all passing after every Phase 10 change + +## Phase 11 — Company Discovery & Provider Completion +- [x] `SearchProvider` interface (`app/search/base.py`) + `MockSearchProvider` (deterministic, honest about having no evidence) + `BraveSearchProvider` (`SEARCH_PROVIDER=brave`) + factory +- [x] Company-profile extraction LLM task (`app/prompts/company_profile.py`) — evidence-grounded (real search snippets + real fetched homepage text in, structured industry/country/region/headquarters/aliases/competitors/public_identifiers out), never asked to recall facts from training data +- [x] `discovery_service.discover_company_profile` — resolves official website, fetches it for real, runs a few targeted searches, calls the extraction task once, merges (user hints win over discovered values), and previews likely sources via the existing collectors' `.discover()` methods (no persistence) +- [x] `POST /api/v1/companies/discover` — rate-limited (`5/minute`), returns a `DiscoveredCompanyProfile`, writes nothing to the DB +- [x] `Company.headquarters` + `Company.public_identifiers` columns (migration `f01919a99ee9`) threaded through the repository/service/schema layers +- [x] `GeminiLLMProvider` (`app/analysis/llm/gemini_provider.py`) on the official `google-genai` SDK, native structured-output (`response_schema`) + the same repair-loop pattern as the Anthropic provider; `LLM_PROVIDER=gemini` +- [x] Add-Company wizard redesigned around discover → review: `STEPS = ["Discover", "Review", "Schedule", "Notifications", "Confirm"]`. Discover step only requires a company name (official website/focus/competitors/aliases are optional accuracy hints); Review step shows the discovered profile, fully editable, with a potential-sources list and a "sources consulted" transparency line +- [x] Company detail page's Details panel now shows `headquarters` and any `public_identifiers` (was previously captured by the wizard but had nowhere to display after creation — fixed during live verification) +- [x] 206 backend tests (up from 185), 18 frontend tests (up from 14) — new coverage for `SearchProvider` (mock + Brave respx-mocked), `discovery_service` (respx-mocked fetch + mock LLM, hint-overrides-discovery merge, plus a regression pair for the non-corporate-host resolution fix below), the `/companies/discover` endpoint, `GeminiLLMProvider` (mocked SDK client), and the wizard's discover→review flow +- [x] Verified live end-to-end against the Docker stack, first with mock providers (`api`/`worker`/`beat` rebuilt for the new `google-genai` dependency, Alembic migration applied to real Postgres, full wizard flow Discover → Review (edited fields survived) → Confirm → a real `Company` row created with `headquarters`/`public_identifiers` persisted; pre-existing "Run now" → lazy source-discovery pipeline confirmed unaffected), then re-verified with **real provider keys the user supplied**: `BraveSearchProvider` confirmed live (`api.search.brave.com`, all queries `200 OK`); `GeminiLLMProvider` uncovered and fixed a real bug (`public_identifiers: dict[str,str]` produced an `additionalProperties` JSON schema the Gemini Developer API rejects - changed to `list[PublicIdentifier]`), then hit an account-level `429 limit:0` unrelated to app code; user switched to `AnthropicLLMProvider`, which then verified the **entire pipeline live with real APIs**: real discovery (Brave + Claude) → real company creation → real `stripe.com` crawl on first run (23 items, 3/3 sources successful) → a real, evidence-grounded, correctly-hedged baseline report from `claude-sonnet-5`. Also found and fixed live: `_resolve_official_website` picking Brave's top-ranked Wikipedia result over the real corporate domain for well-known companies, which corrupted downstream source-preview URLs - see `KNOWN_LIMITATIONS.md` + +## Phase 12 — UI Polish & Bug-Fix Round (post-live-testing feedback) +User drove the live app (real Brave + Anthropic keys) and reported 9 concrete issues. All addressed: +- [x] Company detail page's delete-confirmation control now animates in/out with a `grid-template-columns` + opacity transition (200ms, matching the app's existing `fade-in` timing) instead of an instant DOM swap that also instantly shifted the Pause/Run now buttons sideways +- [x] Add-Company wizard's "What do you want to know?" field now shows placeholder text summarizing the categories the app actually monitors for (products/pricing, leadership, hiring, financial signals, M&A, patents, expansion, regulatory/legal, competitor positioning) — was previously blank with no guidance +- [x] `CompanyProfileExtraction`/`DiscoveredCompanyProfile` gained a `description` field (`app/prompts/company_profile.py`, `app/schemas/discovery.py`, `app/services/discovery_service.py`) - the Review step's Description box was always blank before because discovery never produced one; `MockLLMProvider`'s heuristic derives it from the fetched homepage's first real sentence, real providers extract it from evidence like every other field +- [x] Wizard's Confirm step no longer flashes the full confirm view before redirecting - a new `isFinalizing` state swaps in a "Setting up monitoring for X…" spinner the instant "Create company" is clicked and stays there through the redirect (previously `createCompany.isPending` flipped false before `router.push` completed, causing a one-frame flash of the re-enabled button/full confirm content) +- [x] Company detail page's Details panel switched from side-by-side `dt`/`dd` (which produced a hanging-indent wrap for long values like a multi-clause Headquarters string) to a stacked label-above-value layout with `break-words` +- [x] Competitors on the company detail page are now clickable: hovering highlights them, and clicking navigates to that competitor's own company page if it's already monitored (case-insensitive name match against the user's company list) or to `/companies/new?name=` (wizard reads the `name` query param and pre-fills the Discover step) if not - verified live both ways (Stripe → PayPal went to the wizard pre-filled, then after creating PayPal for real, the same link went straight to its company page) +- [x] Investigated "16 points but basically empty" report finding for the live Stripe company - **not a bug**: the report was generated before any monitoring run had ever collected evidence (0 source documents, 0 detected changes), and the real report correctly refused to fabricate findings, explicitly stating "insufficient evidence" throughout rather than hallucinating - exactly the evidence-grounded behavior this app is designed around. Fixed the actual gap, which was a missing warning: the Latest Report tab now shows an inline notice when generating a report with zero monitoring runs, and Generate now/"Run now" ordering is explained rather than silently producing a thin report +- [x] Investigated empty Sources tab for the same company - also **not a bug**: no monitoring run had ever executed for that company, and `Source` rows are only ever created lazily on first collection (by design, since Phase 4/5). Fixed the gap: the empty-state message now explicitly says sources appear after the first "Run now" rather than a bare "No sources configured yet." +- [x] Built the previously-unimplemented Snapshots tab end-to-end: new `GET /companies/{id}/snapshots` endpoint (`app/api/v1/snapshots.py`, `app/services/snapshot_service.py`, `SnapshotRepository.list_for_company` added to `app/repositories/source_repository.py`, newest-first, capped at 50), `SnapshotResponse` schema, frontend `useSnapshots` hook + `api.listSnapshots`, and an expandable list UI (source name, type, timestamp, char count when collapsed; hash, full text summary, and pretty-printed structured summary when expanded) - verified live against real PayPal snapshots (real scraped homepage/careers/GitHub text, expand/collapse working) +- [x] 214 backend tests (up from 211: 3 new for the snapshots endpoint - empty list, newest-first ordering via a directly-inserted `Snapshot` row, ownership isolation), 18 frontend tests unchanged (existing wizard test extended to assert the description field pre-fills) +- [x] Verified live end-to-end in Docker with real Brave + Anthropic keys: ran the full wizard for PayPal (Stripe's competitor), confirmed `paypal.com` resolved directly (not Wikipedia, confirming the Phase 11 fix still holds), description field populated with real Claude-extracted text, a real monitoring run (50 items, 3/3 sources) populated the Sources and Snapshots tabs with real data, and the Stripe↔PayPal competitor cross-link resolved correctly both before and after PayPal existed as a monitored company + +## Phase 13 — Second UI Polish & Bug-Fix Round (wizard nav, dedup, notification linking, report grounding) +User reported 4 more issues after driving the app further. All addressed: +- [x] Wizard's Back button on the Discover step no longer stays permanently disabled - `goBack()` now calls `router.back()` when `step === 0` instead of a no-op, so it correctly returns to whichever of the 3 entry points (Dashboard, Companies, or a competitor link) the user actually came from. Verified live via a real click-through chain (Companies → wizard → Back → landed back on `/companies`) +- [x] Add-Company wizard now detects likely duplicate company names client-side (`normalizeCompanyName`/`findPossibleDuplicate` in `apps/web/app/(app)/companies/new/page.tsx` - strips legal suffixes/punctuation, then exact/substring match against the user's existing companies) **before** spending a real search+LLM call on discovery, shows a "You might already be monitoring X" warning with a link to the existing company and a "Continue anyway" override, and re-checks automatically if the name is edited afterward. Backend now guarantees the persisted `name` is unique per user regardless (`company_service._unique_display_name`, mirroring the existing `_unique_slug` pattern) - "Stripe" → "Stripe (2)" → "Stripe (3)" on collision, filesystem-style. Verified live: typing "Stripe" warned correctly, no discovery API call fired until "Continue anyway", and the created company was actually named "Stripe (2)" +- [x] **Notification destinations are now linked to specific companies** instead of being flat per-user rows every destination implicitly applied to every company. New `notification_destination_companies` join table (migration `60a25ddfc6a3`, includes a data backfill+dedup step - see below), `NotificationDestinationRepository.list_for_company`/`link_company`/`find_by_value`/`delete_orphaned_for_user`, `POST /notification-destinations` now requires `company_ids: list[UUID]` (min 1) and reuses an existing destination by (user, type, value) instead of duplicating it - this is the actual fix for the wizard silently creating a fresh row per company even when the same email was already registered. `alert_service.create_alert_for_change` now dispatches via `list_for_company(company.id)` instead of `list_for_user`, so a destination only fires for companies it's actually linked to. Deleting a company now garbage-collects any destination left with zero remaining links (`company_service.delete_company` → `delete_orphaned_for_user`), with `Company.notification_links`/`NotificationDestination.company_links` given explicit ORM `cascade="all, delete-orphan"` since SQLite (used in dev/tests) doesn't enforce `ON DELETE CASCADE` without a pragma this app doesn't set - relying on the DB-level FK alone would've silently broken cleanup under SQLite while appearing to work on Postgres +- [x] The one-time migration backfills every existing destination onto every company the same user currently has (a no-op behavior change - it's exactly what already happened implicitly before the join table existed) and then deduplicates rows sharing the same (user, type, normalized value), keeping the earliest and deleting the rest (cascading their `NotificationDelivery` history, an acceptable one-time cleanup). Verified against the live Postgres DB: went from several duplicate `theminecraftboy...@gmail.com` rows down to exactly 4 unique destinations, with 48 backfilled links (4 destinations × 12 companies at migration time) - matching the pre-migration behavior exactly +- [x] Settings page redesigned: `AddDestinationForm` now has a required company multi-select (toggle-chip buttons, at least one required to submit); `DestinationRow` shows each unique destination once with a horizontally-scrollable row of clickable company chips (`overflow-x-auto`, hover-highlight, links to `/companies/{id}`) instead of no company visibility at all. Verified live: scrollWidth (1134px) exceeds clientWidth (384px) on the chip row, confirming it actually scrolls rather than wrapping/clipping; creating a destination linked to only one company and then deleting that company correctly removed the destination from the list (confirmed via direct API calls against the live stack) +- [x] Report generation now receives the company's discovered profile (`description`, `official_website`, `headquarters`, `country`, `region`, `public_identifiers`) as a `company_profile` evidence block (`app/prompts/report_generation.py`, threaded through from `report_service.py`), not just `source_documents`/`detected_changes` - this data is genuine evidence (fetched from the company's real website/search results at onboarding), just previously never wired into report generation, which is why reports generated before any monitoring run came back almost entirely "insufficient evidence" even when the company profile had real content. `MockLLMProvider._build_report` also rewritten to ground `company_overview`/`market_positioning` in profile fields. Verified live: generated a report for a brand-new "Airbnb" company with zero monitoring runs - `company_overview` and `market_positioning` came back as substantive, evidence-grounded paragraphs (real HQ, industry, named competitors, business model) instead of "insufficient evidence", while sections with genuinely no evidence (financials, hiring, leadership) still correctly said so +- [x] 223 backend tests (up from 214: company name-uniqueness ×3, notification-destination linking/dedup/GC ×6, mock report profile-grounding ×1, plus updates to existing destination/alert tests to pass `company_ids`), 19 frontend tests (up from 18: new duplicate-warning wizard test) +- [x] Verified live end-to-end against the Docker stack with real Brave + Anthropic keys and the live Postgres DB (migration applied, backfill/dedup confirmed via direct SQL) - see individual bullets above for what was checked + +## Phase 14 — New Intelligence Sources + Per-Source Scheduling +User compared the live app against the original ChatGPT-authored planning document that inspired it and found real gaps - not a policy problem (bypassing paywalls/logins was explicitly rejected as out of scope again), just free/public sources that were never wired up, plus no way to check a fast-moving source (news) more often than a slow one (patents) within the same company. +- [x] Google News RSS auto-discovery - `RssCollector.discover()` (`app/collectors/rss.py`) was previously a stub returning `[]`; now builds `https://news.google.com/rss/search?q={company}&hl=en-US&gl=US&ceid=US:en` and registers it as a real discoverable source, reusing `RssCollector.collect()`'s already-real `feedparser` fetch/parse +- [x] Government contracts via USASpending.gov - new `SourceType.GOV_CONTRACT` + `GovContractCollector` (`app/collectors/gov_contracts.py`), a free/keyless `POST /api/v2/search/spending_by_award/`, structurally mirroring `SecEdgarCollector` (always offered, a private company just gets zero results, not an error) +- [x] Patents wired to USPTO's Open Data Portal - new `USPTO_API_KEY` setting; `PatentSourceCollector.collect()` now attempts a real `POST /api/v1/patent/applications/search` call when a key is configured, with the existing honest fixture/disabled fallback completely unchanged (byte-for-byte) for the default no-key case, so none of the 12 existing companies with a dormant `PatentSourceCollector` regressed +- [x] Per-source check-frequency scheduling - `Source` gained 4 nullable columns (`frequency_type`/`interval_minutes`/`cron_expression`/`next_check`, migration `06e7f03cea36`); `NULL` means "inherit the company's default cadence" (the default for every source, zero behavior change unless a source opts in). `sync_schedules` now enqueues a company if *either* its own `MonitorConfiguration.next_run` is due *or* any of its sources has an independently-due override (`SourceRepository.company_has_due_work`/`list_due_for_company`). A `SCHEDULED` run only collects the sources actually due; a `MANUAL` "Run now" still collects every active source regardless of cadence, unchanged. Frontend: a per-row "Check frequency" `Select` on the Sources tab (`apps/web/app/(app)/companies/[id]/page.tsx`), defaulting to "Same as company", wired through `PATCH /sources/{id}` (`SourceUpdate` gained the same 3 fields; `source_service.update_source` validates the schedule via the existing `validate_and_compute_next_run` and resets `next_check` to `None` so a changed/cleared override takes effect on the very next scheduler tick rather than waiting out the old cadence) +- [x] Explicitly dropped: Nubela/NinjaPear company-enrichment API (confirmed enterprise-only pricing, not accessible to a normal user - no code written); dedicated PRNewswire/BusinessWire/GlobeNewswire collectors (redundant with what Google News RSS already surfaces) +- [x] 240 backend tests (up from 223: 2 RSS discovery-URL tests, 4 gov-contracts collector tests, 5 patents-live-branch tests, 4 scheduler/per-source-due-ness integration tests, 2 source-update-API tests for the new scheduling fields), frontend `tsc`/`eslint`/vitest (19 tests) all clean with the new `SourceUpdatePayload`/`useUpdateSource` additions +- [x] Verified live end-to-end against the Docker stack: `api`/`worker`/`beat`/`web` rebuilt, migration `06e7f03cea36` applied to the real Postgres DB, a real "Run now" against Stripe auto-discovered and successfully collected from all 5 sources including the two new ones (`GET https://news.google.com/rss/search?q=Stripe...` → `200 OK`, `POST https://api.usaspending.gov/api/v2/search/spending_by_award/` → `200 OK`), and the new per-row frequency Select was exercised live in-browser (set "Stripe — Google News" to Daily, confirmed via a re-fetch from the API; cleared it back to "Same as company", confirmed that round-tripped too, both backed by real `PATCH /api/v1/sources/{id}` → `200 OK` calls in the API logs). Patents' real-API branch is unit-tested against a mocked HTTP call only - not live-verified, since it requires a user-supplied `USPTO_API_KEY` that hasn't been provided yet; the existing no-key fixture path was already covered by the pre-existing patents tests and is unaffected + +## Phase 15 — NinjaPear Company Enrichment +Phase 14 dropped Nubela/NinjaPear as "enterprise-only" - the user found and paid for an individual $49/mo tier ($49-$1899/mo range) that Phase 14's research missed, then asked for it wired up. Scoped via `AskUserQuestion` before building: **all** endpoint categories including customer listing (not just company-level data), but **onboarding-only** timing (never a recurring per-cycle cost) - confirmed given the API bills real credits per field per request, unlike every other free/flat-rate provider in this app. +- [x] New `app/enrichment/` provider package (`base.py` Protocol + Pydantic result shapes, `mock.py` honest-empties default, `ninjapear.py` real per-endpoint `httpx` calls to `nubela.co`, `factory.py`), mirroring `app/search/`'s shape. New `NINJAPEAR_API_KEY`/`NINJAPEAR_MAX_LEADERSHIP_LOOKUPS` settings +- [x] New `CompanyEnrichment` model (1:1 with `Company`, migration `79e3aa041131`) - `status` (pending/partial/complete/failed), a single `data` JSON blob (employee count, leadership team, funding rounds, competitors-with-reasons, products, recent updates, customers), and an `errors` map so a partially-failed enrichment is never silently presented as complete +- [x] `app/services/enrichment_service.py` orchestrates ~6 independent per-endpoint calls plus capped per-leadership-member work-email/profile lookups (`NINJAPEAR_MAX_LEADERSHIP_LOOKUPS`, default 5) - one failed call is recorded in `errors` and never sinks the others, same principle as `tasks/collection.py`'s per-source loop. New `app/tasks/enrichment.py` Celery task (own `enrichment` queue, generous time limits given NinjaPear's documented up-to-5-minute endpoints) +- [x] `company_service.create_company` enqueues the task post-commit, **gated entirely on `NINJAPEAR_API_KEY` being set** - zero extra background-task volume for the overwhelming majority of users who haven't configured it, matching `PatentSourceCollector.discover()`'s "gate on the key, not the provider" precedent. A `status=pending` `CompanyEnrichment` row is created synchronously in the same transaction (not left implicit) so the frontend has something real to poll on +- [x] Report generation gains a `company_enrichment` evidence block (`app/prompts/report_generation.py`, threaded through `report_service.py`) right alongside the existing `company_profile` block - no new report schema needed, since funding/leadership/competitors/products/customers all map onto existing `ReportContent` sections (`financial_signals`, `leadership_changes`, `competitor_comparison`, `products_and_services`, `customer_sentiment`) +- [x] `GET /system/status` gains `ninjapear_configured`/`ninjapear_credit_balance` (a live, free credit-balance call), shown in Settings' System configuration panel - real-money cost visibility, same treatment the SMS provider status already gets +- [x] Frontend: new "Enrichment" tab on the company detail page (funding table, leadership list with resolved work-email/profile links, competitors-with-reasons kept visually separate from the user's own reviewed Competitors list, products, recent updates, customers, employee count), polling (`useCompany`'s `refetchInterval`) while `status === "pending"` so it updates itself once the background task finishes +- [x] Deliberately excluded: the "Similar People" endpoint (a role-anchored prospecting tool with no natural onboarding-time trigger), the Website Lookup endpoint (redundant - `official_website` already comes from Phase 11 discovery), and auto-merging NinjaPear's suggested competitors into the user-reviewed `Company.competitors` list (would silently mutate user-controlled data) +- [x] 261 backend tests (up from 240: 8 provider tests incl. mock honesty, 6 orchestration-service tests incl. the leadership cap and partial/failed status derivation, 3 Celery-task integration tests, 2 create-company enqueue-gating tests - the real regression guard, since every other test in the suite runs without a key configured and stays green throughout - 2 report-generation evidence-block tests), frontend `tsc`/eslint/vitest (19 tests) all clean +- [x] **Verified live** with the user's real `NINJAPEAR_API_KEY` and `USPTO_API_KEY` (both supplied after explicit go-ahead, since NinjaPear spends real credits per call). The initial schema (written from a JS-rendered docs page that couldn't be fully scraped) had real bugs the live pass caught: NinjaPear identifies companies by `website` only (no name-based lookup), `employee_count`/`industry`/funding amounts come back as raw numbers not strings, funding's `investors` are objects not plain strings, and most response field names differed from the initial guesses (`executives`, `total_funds_raised`, `x_profile_url`, etc. - full list in `KNOWN_LIMITATIONS.md`). All fixed and re-verified: a real company creation (Stripe) returned real leadership bios, work emails, X profiles, funding history, competitors-with-reasons, live blog updates, and customers for 34 real credits, landing on `status: partial` (one now-fixed bug, one legitimate 404 for a person not in NinjaPear's database) - and the credit-balance endpoint path/field was also wrong and fixed (`/api/v1/meta/credit-balance` → `credit_balance`, not `/company/credit-balance` → `balance`). USPTO's real branch (Phase 14, unverified until now) turned out to have its own live-only bugs too - a wrong sort-field path (`500` error), title/date fields nested one level deeper than assumed, and `404` "no matching records" being treated as a failure instead of an honest empty result - all fixed. 265 backend tests (up from 261: schema-corrected provider/service tests, a no-website-guard test, and a USPTO 404-as-empty-result test) +- [x] **USPTO company-name search follow-up**: confirmed (by inspecting a real response's full field list, for a query that *did* return 110k+ real results by inventor name) that USPTO's Patent Application Search has no queryable assignee/company field at all - not a bug to fix, a real constraint of that dataset. At the user's request, worked around it: `CompanyContext` gained `leadership_names` (`app/collectors/base.py`), threaded through from `CompanyEnrichment.data["leadership_team"]` in `collection_service.to_company_context` (new `_leadership_names` helper - DB-free collectors stay DB-free, the ORM→dataclass seam is the one place allowed to read it). `PatentSourceCollector.collect()` now searches USPTO by each of the company's leadership names (capped at 5, `_MAX_INVENTOR_SEARCHES`) instead of by company name, dedupes results by application number across names, and trust-scores every match at 0.5 with explicit "heuristic, not verified" labeling in the document content - a name match is a real signal, not proof of company ownership. Live-verified against Stripe: 19 real patent documents found via its executives' names, zero NinjaPear credits spent (USPTO is free). Self-heals over time: a company whose patents source was discovered before enrichment finished just returns empty until the next scheduled collection re-reads (now-populated) leadership names from the DB. 266 backend tests (up from 265: leadership-name search/dedup, empty-without-names no-network-call, and eager-load fixes to 3 pre-existing integration tests that constructed `Company` objects directly without loading the new `.enrichment` relationship) diff --git a/apps/api/alembic.ini b/apps/api/alembic.ini new file mode 100644 index 0000000..003e661 --- /dev/null +++ b/apps/api/alembic.ini @@ -0,0 +1,38 @@ +[alembic] +script_location = migrations +prepend_sys_path = . +version_path_separator = os + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/apps/api/app/__init__.py b/apps/api/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/app/analysis/__init__.py b/apps/api/app/analysis/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/app/analysis/llm/__init__.py b/apps/api/app/analysis/llm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/app/analysis/llm/anthropic_provider.py b/apps/api/app/analysis/llm/anthropic_provider.py new file mode 100644 index 0000000..eaa80bd --- /dev/null +++ b/apps/api/app/analysis/llm/anthropic_provider.py @@ -0,0 +1,82 @@ +"""Anthropic provider: structured output via forced tool-use (the response +schema becomes the tool's input_schema, so the model can only "call" it with +arguments matching the shape we asked for), with a bounded repair loop for +the rare malformed response. +""" + +from __future__ import annotations + +from anthropic import AsyncAnthropic +from pydantic import BaseModel, ValidationError + +from app.analysis.llm.base import LLMResponseError +from app.core.config import Settings + +_TOOL_NAME = "emit_result" + + +class AnthropicLLMProvider: + provider_name = "anthropic" + + def __init__(self, settings: Settings) -> None: + self._settings = settings + self._client = AsyncAnthropic(api_key=settings.anthropic_api_key) + + async def generate_structured[T: BaseModel]( + self, system_prompt: str, user_prompt: str, response_model: type[T] + ) -> T: + tools = [ + { + "name": _TOOL_NAME, + "description": f"Emit the result matching the {response_model.__name__} schema.", + "input_schema": response_model.model_json_schema(), + } + ] + + last_error: Exception | None = None + messages: list[dict] = [{"role": "user", "content": user_prompt}] + + for attempt in range(self._settings.llm_max_retries + 1): + if attempt > 0 and last_error is not None: + messages = [ + *messages, + { + "role": "user", + "content": ( + f"Your previous response did not match the required schema: " + f"{last_error}. Please try again." + ), + }, + ] + + response = await self._client.messages.create( + model=self._settings.anthropic_model, + max_tokens=self._settings.llm_max_tokens_per_request, + system=system_prompt, + tools=tools, + tool_choice={"type": "tool", "name": _TOOL_NAME}, + messages=messages, + ) + tool_use = next((b for b in response.content if b.type == "tool_use"), None) + if tool_use is None: + last_error = ValueError("No tool_use block in the model's response") + continue + try: + return response_model.model_validate(tool_use.input) + except ValidationError as exc: + last_error = exc + continue + + raise LLMResponseError( + f"Anthropic provider failed to produce a valid {response_model.__name__} after " + f"{self._settings.llm_max_retries + 1} attempt(s): {last_error}" + ) + + async def generate_text(self, system_prompt: str, user_prompt: str) -> str: + response = await self._client.messages.create( + model=self._settings.anthropic_model, + max_tokens=self._settings.llm_max_tokens_per_request, + system=system_prompt, + messages=[{"role": "user", "content": user_prompt}], + ) + return "\n".join(block.text for block in response.content if block.type == "text") diff --git a/apps/api/app/analysis/llm/base.py b/apps/api/app/analysis/llm/base.py new file mode 100644 index 0000000..2ddf9d5 --- /dev/null +++ b/apps/api/app/analysis/llm/base.py @@ -0,0 +1,33 @@ +"""LLM provider interface. Every analysis task (app/prompts/*.py) is written +against this Protocol, never against a specific vendor SDK - swapping +`LLM_PROVIDER` changes which class `get_llm_provider()` returns and nothing +else has to change. `generate_structured` is the primary method: it always +returns a validated instance of the caller's Pydantic response model, never +raw text, so a malformed model response can never propagate un-typed data +into the rest of the app (see `LLMResponseError` / the repair loop in +anthropic_provider.py). +""" + +from __future__ import annotations + +from typing import Protocol + +from pydantic import BaseModel + + +class LLMResponseError(Exception): + """Raised when a provider can't produce a response matching the + requested schema, even after any repair attempts.""" + + +class LLMProvider(Protocol): + provider_name: str + + async def generate_structured[T: BaseModel]( + self, + system_prompt: str, + user_prompt: str, + response_model: type[T], + ) -> T: ... + + async def generate_text(self, system_prompt: str, user_prompt: str) -> str: ... diff --git a/apps/api/app/analysis/llm/factory.py b/apps/api/app/analysis/llm/factory.py new file mode 100644 index 0000000..c3b1893 --- /dev/null +++ b/apps/api/app/analysis/llm/factory.py @@ -0,0 +1,31 @@ +"""Resolves `LLM_PROVIDER` to a concrete provider instance. Never imported +directly by prompt task modules or services - always go through +`get_llm_provider()` so swapping providers stays a one-line config change. +""" + +from __future__ import annotations + +from app.analysis.llm.base import LLMProvider +from app.analysis.llm.mock import MockLLMProvider +from app.core.config import Settings, get_settings + + +def get_llm_provider(settings: Settings | None = None) -> LLMProvider: + settings = settings or get_settings() + + if settings.llm_provider == "anthropic": + from app.analysis.llm.anthropic_provider import AnthropicLLMProvider + + return AnthropicLLMProvider(settings) + + if settings.llm_provider == "ollama": + from app.analysis.llm.ollama_provider import OllamaLLMProvider + + return OllamaLLMProvider(settings) + + if settings.llm_provider == "gemini": + from app.analysis.llm.gemini_provider import GeminiLLMProvider + + return GeminiLLMProvider(settings) + + return MockLLMProvider() diff --git a/apps/api/app/analysis/llm/gemini_provider.py b/apps/api/app/analysis/llm/gemini_provider.py new file mode 100644 index 0000000..8e93264 --- /dev/null +++ b/apps/api/app/analysis/llm/gemini_provider.py @@ -0,0 +1,73 @@ +"""Gemini provider: structured output via the SDK's native +`response_schema` support (the model is constrained to the schema and the +SDK parses the result into an instance of it directly), with the same +bounded repair loop on a malformed/unparsed response as +`anthropic_provider.py`. Chosen as the production LLM_PROVIDER option +alongside Anthropic because Gemini has an actual free rate-limited API +tier (gemini-2.0-flash), unlike OpenAI's expiring trial credits. +""" + +from __future__ import annotations + +from google import genai +from google.genai import types +from pydantic import BaseModel, ValidationError + +from app.analysis.llm.base import LLMResponseError +from app.core.config import Settings + + +class GeminiLLMProvider: + provider_name = "gemini" + + def __init__(self, settings: Settings) -> None: + self._settings = settings + self._client = genai.Client(api_key=settings.gemini_api_key) + + async def generate_structured[T: BaseModel]( + self, system_prompt: str, user_prompt: str, response_model: type[T] + ) -> T: + last_error: Exception | None = None + prompt = user_prompt + + for attempt in range(self._settings.llm_max_retries + 1): + if attempt > 0 and last_error is not None: + prompt = ( + f"{user_prompt}\n\nYour previous response did not match the required " + f"schema: {last_error}. Please try again." + ) + + response = await self._client.aio.models.generate_content( + model=self._settings.gemini_model, + contents=prompt, + config=types.GenerateContentConfig( + system_instruction=system_prompt, + response_mime_type="application/json", + response_schema=response_model, + max_output_tokens=self._settings.llm_max_tokens_per_request, + ), + ) + if response.parsed is None: + last_error = ValueError("Gemini did not return a parsed structured response") + continue + try: + return response_model.model_validate(response.parsed) + except ValidationError as exc: + last_error = exc + continue + + raise LLMResponseError( + f"Gemini provider failed to produce a valid {response_model.__name__} after " + f"{self._settings.llm_max_retries + 1} attempt(s): {last_error}" + ) + + async def generate_text(self, system_prompt: str, user_prompt: str) -> str: + response = await self._client.aio.models.generate_content( + model=self._settings.gemini_model, + contents=user_prompt, + config=types.GenerateContentConfig( + system_instruction=system_prompt, + max_output_tokens=self._settings.llm_max_tokens_per_request, + ), + ) + return response.text or "" diff --git a/apps/api/app/analysis/llm/mock.py b/apps/api/app/analysis/llm/mock.py new file mode 100644 index 0000000..b81cda5 --- /dev/null +++ b/apps/api/app/analysis/llm/mock.py @@ -0,0 +1,327 @@ +"""Deterministic mock provider - the default (`LLM_PROVIDER=mock`) and what +every automated test runs against. Never calls a network or costs money. + +Rather than a generic reflection-based filler, each of the six analysis +tasks gets a purpose-built, deterministic builder that reads the same +evidence block a real model would see (see app/prompts/base.py) and +produces genuinely useful output from it - real counts, real titles, real +severities - never fabricated facts. This is what makes the fixture demo +(Phase 9) work end-to-end without a paid API key. +""" + +from __future__ import annotations + +import re +from typing import Any + +from pydantic import BaseModel + +from app.prompts.alert_summarization import AlertSummary +from app.prompts.base import extract_evidence_block +from app.prompts.change_significance import ChangeSignificanceAssessment +from app.prompts.company_profile import CompanyProfileExtraction +from app.prompts.extraction import ExtractionResult +from app.prompts.relevance import RelevanceAssessment +from app.prompts.report_generation import ReportContent +from app.prompts.synthesis import SynthesisResult + + +def _confidence_label(score: float) -> str: + if score >= 0.85: + return "confirmed" + if score >= 0.65: + return "strongly_indicated" + if score >= 0.45: + return "likely" + if score >= 0.25: + return "possible" + if score > 0: + return "unconfirmed" + return "insufficient_evidence" + + +def _build_relevance(evidence: dict[str, Any]) -> dict[str, Any]: + text = (evidence.get("document_text") or "").lower() + focus = (evidence.get("monitoring_focus") or "").lower() + focus_words = [w for w in focus.split() if len(w) > 4] + matches_focus = any(w in text for w in focus_words) if focus_words else False + return { + "is_relevant": True, + "matches_focus": matches_focus, + "topic_categories": [], + "source_reliability": 0.7, + "reasoning": "Mock provider: keyword-based heuristic (no live LLM configured).", + } + + +def _build_extraction(evidence: dict[str, Any]) -> dict[str, Any]: + text = (evidence.get("document_text") or "").strip() + if not text: + return {"signals": []} + first_sentence = text.split(".")[0][:200].strip() + if not first_sentence: + return {"signals": []} + return { + "signals": [ + { + "signal_type": "event", + "description": first_sentence, + "supporting_passage": first_sentence, + "date": None, + "entities": [], + } + ] + } + + +def _build_synthesis(evidence: dict[str, Any]) -> dict[str, Any]: + signals = evidence.get("signals") or [] + if len(signals) < 2: + return {"conclusions": []} + return { + "conclusions": [ + { + "conclusion": ( + "Multiple related signals were detected together; the mock provider " + "does not attempt fine-grained synthesis - configure a live LLM provider " + "for a specific conclusion." + ), + "evidence_summary": [s.get("description", "") for s in signals[:5]], + "source_count": len(signals), + "confidence": 0.3, + "alternative_explanations": [ + "A live LLM provider would assess this more precisely." + ], + "missing_information": [], + } + ] + } + + +def _build_report(evidence: dict[str, Any]) -> dict[str, Any]: + profile = evidence.get("company_profile") or {} + company_name = profile.get("name") or "The company" + documents = evidence.get("source_documents") or [] + changes = evidence.get("detected_changes") or [] + failed = evidence.get("sources_that_failed_to_collect") or [] + + # The discovered profile (real, from onboarding) is genuine evidence even + # when no monitoring run has collected source_documents/detected_changes + # yet - build company_overview/market_positioning from it honestly + # rather than defaulting straight to "insufficient evidence". + overview_parts = [] + if profile.get("description"): + overview_parts.append(profile["description"]) + facts = [] + if profile.get("industry"): + facts.append(f"industry: {profile['industry']}") + if profile.get("headquarters"): + facts.append(f"headquartered in {profile['headquarters']}") + elif profile.get("country") or profile.get("region"): + facts.append( + f"based in {', '.join(f for f in (profile.get('country'), profile.get('region')) if f)}" + ) + if profile.get("aliases"): + facts.append(f"also known as {', '.join(profile['aliases'])}") + if facts: + overview_parts.append(f"{company_name} ({'; '.join(facts)}).") + company_overview = " ".join(overview_parts) or f"No description on file for {company_name}." + + if profile.get("competitors"): + market_positioning = ( + f"{company_name} operates in a space that includes " + f"{', '.join(profile['competitors'])} as named competitors, per the discovered " + "company profile. No comparative data (pricing, features, market share) is " + "available to assess relative positioning." + ) + else: + market_positioning = "Insufficient evidence to assess market positioning." + + executive_summary = ( + f"Mock analysis (LLM_PROVIDER=mock) based on {len(documents)} collected document(s) " + f"and {len(changes)} detected change(s) for {company_name}." + ) + if failed: + executive_summary += ( + f" {len(failed)} source(s) failed to collect this run and are excluded below." + ) + + recent_developments = [ + { + "headline": change.get("summary") or "Change detected", + "summary": ( + f"{(change.get('change_type') or 'change').replace('_', ' ')} detected " + f"with {change.get('severity') or 'unknown'} severity." + ), + "evidence": [ + { + "detected_change_id": change.get("id"), + "description": change.get("summary") or "", + } + ], + "confidence": _confidence_label(change.get("confidence_score") or 0.5), + "category": change.get("change_type"), + "date": change.get("created_at"), + } + for change in changes[:10] + ] + + hiring_signals = [ + { + "headline": doc.get("title") or doc.get("url") or "Job posting", + "summary": (doc.get("excerpt") or "")[:280], + "evidence": [ + { + "source_document_id": doc.get("id"), + "url": doc.get("url"), + "description": "Collected source document", + } + ], + "confidence": "confirmed", + "category": "job_posting", + "date": doc.get("retrieved_date"), + } + for doc in documents + if doc.get("source_type") == "job_posting" + ] + + unknowns = ( + [f"{len(failed)} source(s) failed to collect this run: {', '.join(failed[:5])}"] + if failed + else [] + ) + + return { + "executive_summary": executive_summary, + "company_overview": company_overview, + "products_and_services": [], + "market_positioning": market_positioning, + "recent_developments": recent_developments, + "strategic_initiatives": [], + "key_inferred_projects": [], + "leadership_changes": [ + d for d in recent_developments if d["category"] == "leadership_change" + ], + "hiring_signals": hiring_signals, + "technology_signals": [], + "patent_signals": [], + "manufacturing_and_expansion_signals": [], + "partnerships_and_acquisitions": [], + "financial_signals": [d for d in recent_developments if d["category"] == "filing_new"], + "regulatory_and_legal_signals": [], + "customer_sentiment": "Insufficient evidence to assess customer sentiment.", + "competitor_comparison": "Insufficient evidence to compare against competitors.", + "swot": {"strengths": [], "weaknesses": [], "opportunities": [], "threats": []}, + "risks": [], + "opportunities": [], + "unknowns_and_missing_data": unknowns, + "monitoring_recommendations": ["Continue monitoring configured sources on schedule."], + "methodology": ( + f"Generated by the mock LLM provider from {len(documents)} stored source " + f"document(s) and {len(changes)} deterministic change-detection result(s). No " + "external model was called." + ), + "limitations": ( + "Generated by the deterministic mock provider, not a live LLM. Set " + "LLM_PROVIDER=anthropic or LLM_PROVIDER=ollama for narrative synthesis." + ), + } + + +def _build_change_significance(evidence: dict[str, Any]) -> dict[str, Any]: + severity = evidence.get("deterministic_severity") or "low" + confidence = evidence.get("deterministic_confidence") or 0.5 + is_meaningful = severity in ("critical", "high", "medium") + change_type = (evidence.get("change_type") or "change").replace("_", " ") + return { + "is_real_change": True, + "is_meaningful": is_meaningful, + "why_it_matters": ( + f"Deterministic scoring classified this {change_type} as {severity} severity " + f"with {confidence:.0%} confidence." + ), + "confidence": confidence, + "should_notify": is_meaningful, + } + + +def _build_alert_summary(evidence: dict[str, Any]) -> dict[str, Any]: + company = evidence.get("company_name") or "The company" + change_type = (evidence.get("change_type") or "change").replace("_", " ") + severity = evidence.get("severity") or "medium" + confidence = evidence.get("confidence") or 0.5 + return { + "title": f"{company}: {change_type} detected"[:100], + "summary": evidence.get("change_summary") or f"A {change_type} was detected for {company}.", + "why_it_matters": f"Classified as {severity} severity with {confidence:.0%} confidence.", + } + + +_HQ_RE = re.compile(r"(?:headquartered|based) in ([A-Z][\w\s,]{2,60}?)(?:[.\n]|$)", re.IGNORECASE) +_FORMERLY_RE = re.compile( + r"formerly (?:known as|named) ([A-Z][\w&\s]{2,60}?)(?:[.,\n]|$)", re.IGNORECASE +) + + +def _build_company_profile(evidence: dict[str, Any]) -> dict[str, Any]: + """Mirrors app/change_detection/extractors.py's philosophy: cheap, + deterministic regex heuristics over real evidence text, never a + fabricated guess. Most fields (industry/country/region/public + identifiers) stay empty since a name-only mock search has no real + signal for them - see search/mock.py's docstring for why that's + intentional, not a gap.""" + homepage_text = evidence.get("homepage_text") or "" + search_results = evidence.get("search_results") or [] + combined_text = homepage_text + "\n" + "\n".join(r.get("snippet", "") for r in search_results) + + hq_match = _HQ_RE.search(combined_text) + headquarters = hq_match.group(1).strip() if hq_match else None + + alias_match = _FORMERLY_RE.search(combined_text) + aliases = [alias_match.group(1).strip()] if alias_match else [] + + # First real sentence of the fetched homepage, if any - an honest, + # evidence-derived summary rather than a fabricated one. + first_sentence = re.split(r"(?<=[.!?])\s", homepage_text.strip(), maxsplit=1)[0].strip() + description = first_sentence[:280] if first_sentence and len(first_sentence) > 15 else None + + return { + "description": description, + "industry": None, + "country": None, + "region": None, + "headquarters": headquarters, + "aliases": aliases, + "competitors": [], + "public_identifiers": [], + } + + +_BUILDERS = { + RelevanceAssessment: _build_relevance, + ExtractionResult: _build_extraction, + SynthesisResult: _build_synthesis, + ReportContent: _build_report, + ChangeSignificanceAssessment: _build_change_significance, + AlertSummary: _build_alert_summary, + CompanyProfileExtraction: _build_company_profile, +} + + +class MockLLMProvider: + provider_name = "mock" + + async def generate_structured[T: BaseModel]( + self, system_prompt: str, user_prompt: str, response_model: type[T] + ) -> T: + evidence = extract_evidence_block(user_prompt) + builder = _BUILDERS.get(response_model) + data = builder(evidence) if builder is not None else {} + return response_model.model_validate(data) + + async def generate_text(self, system_prompt: str, user_prompt: str) -> str: + evidence = extract_evidence_block(user_prompt) + return ( + "[mock provider] No live LLM configured. " + f"{len(evidence)} evidence field(s) were provided for this request." + ) diff --git a/apps/api/app/analysis/llm/ollama_provider.py b/apps/api/app/analysis/llm/ollama_provider.py new file mode 100644 index 0000000..2bffd79 --- /dev/null +++ b/apps/api/app/analysis/llm/ollama_provider.py @@ -0,0 +1,85 @@ +"""Ollama provider: local models via Ollama's HTTP API. Uses JSON mode +(`format: "json"`) plus a bounded repair loop, since not every locally-run +model supports strict schema-constrained decoding the way Anthropic's +tool-use does - the schema is instead embedded in the system prompt as an +instruction. +""" + +from __future__ import annotations + +import json + +import httpx +from pydantic import BaseModel, ValidationError + +from app.analysis.llm.base import LLMResponseError +from app.core.config import Settings + + +class OllamaLLMProvider: + provider_name = "ollama" + + def __init__(self, settings: Settings) -> None: + self._settings = settings + + async def generate_structured[T: BaseModel]( + self, system_prompt: str, user_prompt: str, response_model: type[T] + ) -> T: + schema_instructions = ( + f"{system_prompt}\n\nRespond with ONLY a single JSON object matching this JSON " + f"schema, no other text, no markdown fences:\n" + f"{json.dumps(response_model.model_json_schema())}" + ) + + last_error: Exception | None = None + prompt = user_prompt + + for attempt in range(self._settings.llm_max_retries + 1): + if attempt > 0 and last_error is not None: + prompt = ( + f"{user_prompt}\n\nYour previous response was invalid: {last_error}. " + "Try again, returning ONLY valid JSON matching the schema." + ) + + async with httpx.AsyncClient(timeout=120) as client: + response = await client.post( + f"{self._settings.ollama_base_url}/api/chat", + json={ + "model": self._settings.ollama_model, + "messages": [ + {"role": "system", "content": schema_instructions}, + {"role": "user", "content": prompt}, + ], + "format": "json", + "stream": False, + }, + ) + response.raise_for_status() + content = response.json().get("message", {}).get("content", "") + try: + data = json.loads(content) + return response_model.model_validate(data) + except (json.JSONDecodeError, ValidationError) as exc: + last_error = exc + continue + + raise LLMResponseError( + f"Ollama provider failed to produce a valid {response_model.__name__} after " + f"{self._settings.llm_max_retries + 1} attempt(s): {last_error}" + ) + + async def generate_text(self, system_prompt: str, user_prompt: str) -> str: + async with httpx.AsyncClient(timeout=120) as client: + response = await client.post( + f"{self._settings.ollama_base_url}/api/chat", + json={ + "model": self._settings.ollama_model, + "messages": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ], + "stream": False, + }, + ) + response.raise_for_status() + return response.json().get("message", {}).get("content", "") diff --git a/apps/api/app/api/__init__.py b/apps/api/app/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/app/api/v1/__init__.py b/apps/api/app/api/v1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/app/api/v1/admin.py b/apps/api/app/api/v1/admin.py new file mode 100644 index 0000000..88f0926 --- /dev/null +++ b/apps/api/app/api/v1/admin.py @@ -0,0 +1,91 @@ +"""Public unban-request intake plus admin-only IP-ban management. Thin per +ARCHITECTURE.md - business logic lives in app.services.unban_service.""" + +from __future__ import annotations + +import uuid + +from fastapi import APIRouter, Depends, Request, status +from sqlalchemy.ext.asyncio import AsyncSession + +from app.auth.dependencies import require_admin +from app.core.config import Settings, get_settings +from app.core.rate_limit import limiter +from app.core.security import get_client_ip +from app.db.session import get_db +from app.models.user import User +from app.repositories.unban_request_repository import UnbanRequestRepository +from app.schemas.unban import ( + BanIpRequest, + IpBanResponse, + UnbanRequestPayload, + UnbanRequestResponse, +) +from app.services import unban_service + +router = APIRouter(tags=["admin"]) + + +@router.post("/unban-requests", status_code=status.HTTP_204_NO_CONTENT) +@limiter.limit("3/minute") +async def submit_unban_request( + request: Request, + payload: UnbanRequestPayload, + settings: Settings = Depends(get_settings), + db: AsyncSession = Depends(get_db), +) -> None: + client_ip = get_client_ip(request, settings) + await unban_service.submit_unban_request(db, settings, client_ip, payload.message) + + +@router.get("/admin/ip-bans", response_model=list[IpBanResponse]) +async def list_ip_bans( + db: AsyncSession = Depends(get_db), _admin: User = Depends(require_admin) +) -> list[IpBanResponse]: + bans = await unban_service.list_ip_bans(db) + return [IpBanResponse.model_validate(b) for b in bans] + + +@router.delete("/admin/ip-bans/{ip_address}", status_code=status.HTTP_204_NO_CONTENT) +async def delete_ip_ban( + ip_address: str, + db: AsyncSession = Depends(get_db), + _admin: User = Depends(require_admin), +) -> None: + await unban_service.unban_ip(db, ip_address) + + +@router.post("/admin/ip-bans", response_model=IpBanResponse, status_code=status.HTTP_201_CREATED) +async def create_ip_ban( + payload: BanIpRequest, + db: AsyncSession = Depends(get_db), + _admin: User = Depends(require_admin), +) -> IpBanResponse: + ban = await unban_service.ban_ip(db, payload.ip_address) + return IpBanResponse.model_validate(ban) + + +@router.get("/admin/unban-requests", response_model=list[UnbanRequestResponse]) +async def list_unban_requests( + db: AsyncSession = Depends(get_db), _admin: User = Depends(require_admin) +) -> list[UnbanRequestResponse]: + requests = await UnbanRequestRepository(db).list_all() + return [UnbanRequestResponse.model_validate(r) for r in requests] + + +@router.post("/admin/unban-requests/{request_id}/accept", status_code=status.HTTP_204_NO_CONTENT) +async def accept_unban_request( + request_id: uuid.UUID, + db: AsyncSession = Depends(get_db), + _admin: User = Depends(require_admin), +) -> None: + await unban_service.accept_unban_request(db, request_id) + + +@router.delete("/admin/unban-requests/{request_id}", status_code=status.HTTP_204_NO_CONTENT) +async def reject_unban_request( + request_id: uuid.UUID, + db: AsyncSession = Depends(get_db), + _admin: User = Depends(require_admin), +) -> None: + await unban_service.reject_unban_request(db, request_id) diff --git a/apps/api/app/api/v1/alerts.py b/apps/api/app/api/v1/alerts.py new file mode 100644 index 0000000..7ce3222 --- /dev/null +++ b/apps/api/app/api/v1/alerts.py @@ -0,0 +1,78 @@ +"""Alert routes: list with filters, detail (with delivery status), and +read/resolved mutation.""" + +from __future__ import annotations + +import uuid + +from fastapi import APIRouter, Depends +from sqlalchemy.ext.asyncio import AsyncSession + +from app.auth.dependencies import get_current_user +from app.db.session import get_db +from app.models.enums import SeverityLevel +from app.models.user import User +from app.schemas.alert import AlertDetailResponse, AlertResponse, AlertUpdate +from app.services import alert_service + +router = APIRouter(prefix="/alerts", tags=["alerts"]) + + +@router.get("", response_model=list[AlertResponse]) +async def list_alerts( + company_id: uuid.UUID | None = None, + severity: SeverityLevel | None = None, + read: bool | None = None, + resolved: bool | None = None, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> list[AlertResponse]: + alerts = await alert_service.list_alerts( + db, user.id, company_id=company_id, severity=severity, read=read, resolved=resolved + ) + return [AlertResponse.model_validate(a) for a in alerts] + + +@router.get("/{alert_id}", response_model=AlertDetailResponse) +async def get_alert( + alert_id: uuid.UUID, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> AlertDetailResponse: + alert, deliveries = await alert_service.get_alert_with_deliveries(db, user.id, alert_id) + return AlertDetailResponse( + **AlertResponse.model_validate(alert).model_dump(), deliveries=deliveries + ) + + +@router.patch("/{alert_id}", response_model=AlertResponse) +async def update_alert( + alert_id: uuid.UUID, + payload: AlertUpdate, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> AlertResponse: + alert = await alert_service.update_alert( + db, user.id, alert_id, read=payload.read, resolved=payload.resolved + ) + return AlertResponse.model_validate(alert) + + +@router.post("/{alert_id}/read", response_model=AlertResponse) +async def mark_alert_read( + alert_id: uuid.UUID, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> AlertResponse: + alert = await alert_service.mark_read(db, user.id, alert_id) + return AlertResponse.model_validate(alert) + + +@router.post("/{alert_id}/resolve", response_model=AlertResponse) +async def resolve_alert( + alert_id: uuid.UUID, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> AlertResponse: + alert = await alert_service.mark_resolved(db, user.id, alert_id) + return AlertResponse.model_validate(alert) diff --git a/apps/api/app/api/v1/auth.py b/apps/api/app/api/v1/auth.py new file mode 100644 index 0000000..37842a1 --- /dev/null +++ b/apps/api/app/api/v1/auth.py @@ -0,0 +1,161 @@ +"""Auth routes. Thin per ARCHITECTURE.md: parse/validate, call one service +method, map the result. All rules live in app.services.auth_service.""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends, Request, status +from sqlalchemy.ext.asyncio import AsyncSession + +from app.auth.dependencies import get_current_user +from app.core.config import Settings, get_settings +from app.core.errors import ValidationAppError +from app.core.rate_limit import limiter +from app.core.security import get_client_ip, is_localhost +from app.db.session import get_db +from app.models.user import LOCAL_DEV_USER_ID, User +from app.schemas.auth import ( + ConfirmPasswordResetRequest, + LoginRequest, + LogoutRequest, + RefreshRequest, + RegisterRequest, + RequestPasswordResetRequest, + ResendVerificationRequest, + SecurityEventResponse, + TokenResponse, + VerifyEmailRequest, +) +from app.schemas.user import MeResponse, UserResponse +from app.services import auth_service, system_secret_service +from app.services.turnstile_service import turnstile_required, verify_turnstile + +router = APIRouter(prefix="/auth", tags=["auth"]) + + +async def _enforce_turnstile( + request: Request, settings: Settings, db: AsyncSession, token: str | None, client_ip: str +) -> None: + """Required for register/login/request-password-reset when the caller + isn't on loopback and a secret is configured - skipped entirely + otherwise (see turnstile_service.turnstile_required). The secret may be + admin-configured (system_secret_service) rather than only .env-set.""" + effective = await system_secret_service.get_effective_settings(db, settings) + if not turnstile_required(is_localhost(request, settings), effective): + return + if not token or not await verify_turnstile(token, client_ip, effective): + raise ValidationAppError("Captcha verification required.") + + +@router.post("/register", response_model=UserResponse, status_code=status.HTTP_201_CREATED) +@limiter.limit("5/minute") +async def register( + request: Request, + payload: RegisterRequest, + settings: Settings = Depends(get_settings), + db: AsyncSession = Depends(get_db), +) -> UserResponse: + client_ip = get_client_ip(request, settings) + await _enforce_turnstile(request, settings, db, payload.turnstile_token, client_ip) + user = await auth_service.register(db, settings, client_ip, payload) + return UserResponse.model_validate(user) + + +@router.post("/login", response_model=TokenResponse) +@limiter.limit("10/minute") +async def login( + request: Request, + payload: LoginRequest, + settings: Settings = Depends(get_settings), + db: AsyncSession = Depends(get_db), +) -> TokenResponse: + client_ip = get_client_ip(request, settings) + await _enforce_turnstile(request, settings, db, payload.turnstile_token, client_ip) + return await auth_service.login(db, settings, client_ip, payload) + + +@router.post("/refresh", response_model=TokenResponse) +@limiter.limit("20/minute") +async def refresh_token( + request: Request, + payload: RefreshRequest, + settings: Settings = Depends(get_settings), + db: AsyncSession = Depends(get_db), +) -> TokenResponse: + return await auth_service.refresh(db, settings, payload.refresh_token) + + +@router.post("/logout", status_code=status.HTTP_204_NO_CONTENT) +async def logout( + payload: LogoutRequest, + settings: Settings = Depends(get_settings), + db: AsyncSession = Depends(get_db), +) -> None: + await auth_service.logout(db, settings, payload.refresh_token) + + +@router.post("/verify-email", status_code=status.HTTP_204_NO_CONTENT) +async def verify_email( + request: Request, + payload: VerifyEmailRequest, + settings: Settings = Depends(get_settings), + db: AsyncSession = Depends(get_db), +) -> None: + client_ip = get_client_ip(request, settings) + await auth_service.verify_email(db, client_ip, payload) + + +@router.post("/resend-verification", status_code=status.HTTP_204_NO_CONTENT) +async def resend_verification( + request: Request, + payload: ResendVerificationRequest, + settings: Settings = Depends(get_settings), + db: AsyncSession = Depends(get_db), +) -> None: + client_ip = get_client_ip(request, settings) + await auth_service.resend_verification(db, settings, client_ip, payload) + + +@router.post("/request-password-reset", status_code=status.HTTP_204_NO_CONTENT) +async def request_password_reset( + request: Request, + payload: RequestPasswordResetRequest, + settings: Settings = Depends(get_settings), + db: AsyncSession = Depends(get_db), +) -> None: + client_ip = get_client_ip(request, settings) + await _enforce_turnstile(request, settings, db, payload.turnstile_token, client_ip) + await auth_service.request_password_reset(db, settings, client_ip, payload) + + +@router.post("/confirm-password-reset", status_code=status.HTTP_204_NO_CONTENT) +async def confirm_password_reset( + request: Request, + payload: ConfirmPasswordResetRequest, + settings: Settings = Depends(get_settings), + db: AsyncSession = Depends(get_db), +) -> None: + client_ip = get_client_ip(request, settings) + await auth_service.confirm_password_reset(db, client_ip, payload) + + +@router.get("/security-events", response_model=list[SecurityEventResponse]) +async def security_events( + db: AsyncSession = Depends(get_db), user: User = Depends(get_current_user) +) -> list[SecurityEventResponse]: + """The calling user's own security activity - not admin-gated, it's + their own data (see the app-wide admin log feed in app.api.v1.system for + the operational counterpart).""" + events = await auth_service.list_security_events(db, user.id) + return [SecurityEventResponse.model_validate(e) for e in events] + + +@router.get("/me", response_model=MeResponse) +async def me(user: User = Depends(get_current_user)) -> MeResponse: + # Reports what actually happened for *this* request, not the raw + # AUTH_MODE setting - the fixed local-dev user only ever comes back via + # the loopback bypass (see get_current_user), so its id is a reliable + # per-request signal even though the setting itself is almost always + # "local". + effective_auth_mode = "local" if user.id == LOCAL_DEV_USER_ID else "jwt" + base = UserResponse.model_validate(user).model_dump() + return MeResponse(**base, auth_mode=effective_auth_mode) diff --git a/apps/api/app/api/v1/companies.py b/apps/api/app/api/v1/companies.py new file mode 100644 index 0000000..d9bd7b5 --- /dev/null +++ b/apps/api/app/api/v1/companies.py @@ -0,0 +1,155 @@ +"""Company + monitor-configuration routes. Thin per ARCHITECTURE.md.""" + +from __future__ import annotations + +import uuid + +from fastapi import APIRouter, Depends, Request, status +from sqlalchemy.ext.asyncio import AsyncSession + +from app.analysis.llm.factory import get_llm_provider +from app.auth.dependencies import get_current_user +from app.core.config import Settings, get_settings +from app.core.errors import NotFoundError +from app.core.rate_limit import limiter +from app.db.session import get_db +from app.models.user import User +from app.schemas.company import ( + CompanyCreate, + CompanyResponse, + CompanyUpdate, + MonitorConfigurationResponse, + MonitorConfigurationUpdate, +) +from app.schemas.discovery import DiscoverCompanyRequest, DiscoveredCompanyProfile +from app.search.factory import get_search_provider +from app.services import company_service, discovery_service, user_api_key_service + +router = APIRouter(prefix="/companies", tags=["companies"]) + + +@router.get("", response_model=list[CompanyResponse]) +async def list_companies( + user: User = Depends(get_current_user), db: AsyncSession = Depends(get_db) +) -> list[CompanyResponse]: + companies = await company_service.list_companies(db, user.id) + return [CompanyResponse.from_company(c) for c in companies] + + +@router.post("", response_model=CompanyResponse, status_code=status.HTTP_201_CREATED) +@limiter.limit("20/minute") +async def create_company( + request: Request, + payload: CompanyCreate, + user: User = Depends(get_current_user), + settings: Settings = Depends(get_settings), + db: AsyncSession = Depends(get_db), +) -> CompanyResponse: + settings = await user_api_key_service.get_effective_settings(db, user.id, settings) + company = await company_service.create_company(db, settings, user.id, payload) + return CompanyResponse.from_company(company) + + +@router.post("/discover", response_model=DiscoveredCompanyProfile) +@limiter.limit("5/minute") +async def discover_company( + request: Request, + payload: DiscoverCompanyRequest, + user: User = Depends(get_current_user), + settings: Settings = Depends(get_settings), + db: AsyncSession = Depends(get_db), +) -> DiscoveredCompanyProfile: + """Proposes a company profile from just a name - persists nothing. The + wizard's "Discover" step calls this, then lets the user edit the result + before the existing POST /companies actually creates anything. Tightly + rate-limited: unlike everything else in this router, this costs a real + search + LLM call per invocation.""" + settings = await user_api_key_service.get_effective_settings(db, user.id, settings) + search = get_search_provider(settings) + llm = get_llm_provider(settings) + return await discovery_service.discover_company_profile( + search, + llm, + settings, + name=payload.name, + official_website=payload.official_website, + monitoring_focus=payload.monitoring_focus, + competitor_names=payload.competitor_names, + alias_names=payload.alias_names, + ) + + +@router.get("/{company_id}", response_model=CompanyResponse) +async def get_company( + company_id: uuid.UUID, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> CompanyResponse: + company = await company_service.get_company(db, user.id, company_id) + return CompanyResponse.from_company(company) + + +@router.patch("/{company_id}", response_model=CompanyResponse) +async def update_company( + company_id: uuid.UUID, + payload: CompanyUpdate, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> CompanyResponse: + company = await company_service.update_company(db, user.id, company_id, payload) + return CompanyResponse.from_company(company) + + +@router.delete("/{company_id}", status_code=status.HTTP_204_NO_CONTENT) +async def delete_company( + company_id: uuid.UUID, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> None: + await company_service.delete_company(db, user.id, company_id) + + +@router.post("/{company_id}/pause", response_model=CompanyResponse) +async def pause_company( + company_id: uuid.UUID, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> CompanyResponse: + company = await company_service.pause_company(db, user.id, company_id) + return CompanyResponse.from_company(company) + + +@router.post("/{company_id}/resume", response_model=CompanyResponse) +async def resume_company( + company_id: uuid.UUID, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> CompanyResponse: + company = await company_service.resume_company(db, user.id, company_id) + return CompanyResponse.from_company(company) + + +@router.get("/{company_id}/monitor", response_model=MonitorConfigurationResponse) +async def get_monitor_configuration( + company_id: uuid.UUID, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> MonitorConfigurationResponse: + company = await company_service.get_company(db, user.id, company_id) + if company.monitor_configuration is None: + raise NotFoundError("Monitor configuration not found") + return MonitorConfigurationResponse.model_validate(company.monitor_configuration) + + +@router.patch("/{company_id}/monitor", response_model=MonitorConfigurationResponse) +async def update_monitor_configuration( + company_id: uuid.UUID, + payload: MonitorConfigurationUpdate, + user: User = Depends(get_current_user), + settings: Settings = Depends(get_settings), + db: AsyncSession = Depends(get_db), +) -> MonitorConfigurationResponse: + config = await company_service.update_monitor_configuration( + db, settings, user.id, company_id, payload + ) + return MonitorConfigurationResponse.model_validate(config) diff --git a/apps/api/app/api/v1/dashboard.py b/apps/api/app/api/v1/dashboard.py new file mode 100644 index 0000000..0c7dc95 --- /dev/null +++ b/apps/api/app/api/v1/dashboard.py @@ -0,0 +1,23 @@ +"""Dashboard-level aggregate analytics, scoped to the current user across +every company they own - see app/services/analytics_service.py.""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends +from sqlalchemy.ext.asyncio import AsyncSession + +from app.auth.dependencies import get_current_user +from app.db.session import get_db +from app.models.user import User +from app.schemas.dashboard import DashboardAnalytics +from app.services import analytics_service + +router = APIRouter(prefix="/dashboard", tags=["dashboard"]) + + +@router.get("/analytics", response_model=DashboardAnalytics) +async def get_dashboard_analytics( + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> DashboardAnalytics: + return await analytics_service.get_dashboard_analytics(db, user.id) diff --git a/apps/api/app/api/v1/monitoring.py b/apps/api/app/api/v1/monitoring.py new file mode 100644 index 0000000..35ed28f --- /dev/null +++ b/apps/api/app/api/v1/monitoring.py @@ -0,0 +1,55 @@ +"""Monitoring run routes: run-now, run history, single-run status polling.""" + +from __future__ import annotations + +import uuid + +from fastapi import APIRouter, Depends, Request, status +from sqlalchemy.ext.asyncio import AsyncSession + +from app.auth.dependencies import get_current_user +from app.core.config import Settings, get_settings +from app.core.rate_limit import limiter +from app.db.session import get_db +from app.models.user import User +from app.schemas.monitoring import MonitoringRunResponse +from app.services import monitoring_service + +router = APIRouter(tags=["monitoring"]) + + +@router.post( + "/companies/{company_id}/run", + response_model=MonitoringRunResponse, + status_code=status.HTTP_202_ACCEPTED, +) +@limiter.limit("20/minute") +async def run_company_now( + request: Request, + company_id: uuid.UUID, + user: User = Depends(get_current_user), + settings: Settings = Depends(get_settings), + db: AsyncSession = Depends(get_db), +) -> MonitoringRunResponse: + run = await monitoring_service.enqueue_run_now(db, settings, user.id, company_id) + return MonitoringRunResponse.model_validate(run) + + +@router.get("/companies/{company_id}/runs", response_model=list[MonitoringRunResponse]) +async def list_company_runs( + company_id: uuid.UUID, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> list[MonitoringRunResponse]: + runs = await monitoring_service.list_runs(db, user.id, company_id) + return [MonitoringRunResponse.model_validate(r) for r in runs] + + +@router.get("/runs/{run_id}", response_model=MonitoringRunResponse) +async def get_run( + run_id: uuid.UUID, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> MonitoringRunResponse: + run = await monitoring_service.get_run(db, user.id, run_id) + return MonitoringRunResponse.model_validate(run) diff --git a/apps/api/app/api/v1/notification_destinations.py b/apps/api/app/api/v1/notification_destinations.py new file mode 100644 index 0000000..61b6726 --- /dev/null +++ b/apps/api/app/api/v1/notification_destinations.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import uuid + +from fastapi import APIRouter, Depends, Request, status +from sqlalchemy.ext.asyncio import AsyncSession + +from app.auth.dependencies import get_current_user +from app.core.config import Settings, get_settings +from app.core.rate_limit import limiter +from app.db.session import get_db +from app.models.user import User +from app.schemas.notification_destination import ( + NotificationDestinationCreate, + NotificationDestinationResponse, + NotificationDestinationUpdate, + NotificationTestResult, +) +from app.services import alert_service +from app.services import notification_destination_service as service + +router = APIRouter(prefix="/notification-destinations", tags=["notifications"]) + + +@router.get("", response_model=list[NotificationDestinationResponse]) +async def list_destinations( + user: User = Depends(get_current_user), db: AsyncSession = Depends(get_db) +) -> list[NotificationDestinationResponse]: + destinations = await service.list_destinations(db, user.id) + return [NotificationDestinationResponse.from_destination(d) for d in destinations] + + +@router.post( + "", response_model=NotificationDestinationResponse, status_code=status.HTTP_201_CREATED +) +@limiter.limit("20/minute") +async def create_destination( + request: Request, + payload: NotificationDestinationCreate, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> NotificationDestinationResponse: + destination = await service.create_destination(db, user.id, payload) + return NotificationDestinationResponse.from_destination(destination) + + +@router.patch("/{destination_id}", response_model=NotificationDestinationResponse) +async def update_destination( + destination_id: uuid.UUID, + payload: NotificationDestinationUpdate, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> NotificationDestinationResponse: + destination = await service.update_destination(db, user.id, destination_id, payload) + return NotificationDestinationResponse.from_destination(destination) + + +@router.delete("/{destination_id}", status_code=status.HTTP_204_NO_CONTENT) +async def delete_destination( + destination_id: uuid.UUID, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> None: + await service.delete_destination(db, user.id, destination_id) + + +@router.delete("/{destination_id}/companies/{company_id}", status_code=status.HTTP_204_NO_CONTENT) +async def unlink_company( + destination_id: uuid.UUID, + company_id: uuid.UUID, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> None: + """Unlinks one company from a destination without deleting it outright + - a destination shared across several companies must survive losing + just one of them. If this was its last link, it's garbage-collected + the same way a company deletion already orphans-and-removes one.""" + await service.unlink_company(db, user.id, destination_id, company_id) + + +@router.post("/{destination_id}/test", response_model=NotificationTestResult) +@limiter.limit("10/minute") +async def test_destination( + request: Request, + destination_id: uuid.UUID, + user: User = Depends(get_current_user), + settings: Settings = Depends(get_settings), + db: AsyncSession = Depends(get_db), +) -> NotificationTestResult: + result = await alert_service.send_test_notification(db, settings, user.id, destination_id) + return NotificationTestResult(success=result.success, error=result.error) diff --git a/apps/api/app/api/v1/reports.py b/apps/api/app/api/v1/reports.py new file mode 100644 index 0000000..9d0128e --- /dev/null +++ b/apps/api/app/api/v1/reports.py @@ -0,0 +1,80 @@ +"""Report routes: list/detail, manual generation, and raw markdown/json +export - mixes `/companies/{company_id}/reports...` and `/reports/{id}...` +paths per the spec, same pattern as sources.py/monitoring.py.""" + +from __future__ import annotations + +import uuid + +from fastapi import APIRouter, Depends, Request, Response, status +from sqlalchemy.ext.asyncio import AsyncSession + +from app.analysis.llm.factory import get_llm_provider +from app.auth.dependencies import get_current_user +from app.core.config import Settings, get_settings +from app.core.rate_limit import limiter +from app.db.session import get_db +from app.models.user import User +from app.schemas.report import ReportListItem, ReportResponse +from app.services import report_service, user_api_key_service + +router = APIRouter(tags=["reports"]) + + +@router.get("/companies/{company_id}/reports", response_model=list[ReportListItem]) +async def list_reports( + company_id: uuid.UUID, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> list[ReportListItem]: + reports = await report_service.list_reports(db, user.id, company_id) + return [ReportListItem.model_validate(r) for r in reports] + + +@router.post( + "/companies/{company_id}/reports/generate", + response_model=ReportResponse, + status_code=status.HTTP_201_CREATED, +) +@limiter.limit("10/minute") +async def generate_report( + request: Request, + company_id: uuid.UUID, + user: User = Depends(get_current_user), + settings: Settings = Depends(get_settings), + db: AsyncSession = Depends(get_db), +) -> ReportResponse: + settings = await user_api_key_service.get_effective_settings(db, user.id, settings) + llm = get_llm_provider(settings) + report = await report_service.generate_report_now(db, settings, llm, user.id, company_id) + return ReportResponse.model_validate(report) + + +@router.get("/reports/{report_id}", response_model=ReportResponse) +async def get_report( + report_id: uuid.UUID, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> ReportResponse: + report = await report_service.get_report(db, user.id, report_id) + return ReportResponse.model_validate(report) + + +@router.get("/reports/{report_id}/markdown", response_class=Response) +async def get_report_markdown( + report_id: uuid.UUID, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> Response: + report = await report_service.get_report(db, user.id, report_id) + return Response(content=report.markdown_content, media_type="text/markdown") + + +@router.get("/reports/{report_id}/json") +async def get_report_json( + report_id: uuid.UUID, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> dict: + report = await report_service.get_report(db, user.id, report_id) + return report.structured_report diff --git a/apps/api/app/api/v1/router.py b/apps/api/app/api/v1/router.py new file mode 100644 index 0000000..73930d3 --- /dev/null +++ b/apps/api/app/api/v1/router.py @@ -0,0 +1,35 @@ +"""Aggregates all /api/v1 routers. Individual routers are added here as each +phase implements them - keeps main.py stable.""" + +from __future__ import annotations + +from fastapi import APIRouter + +from app.api.v1 import ( + admin, + alerts, + auth, + companies, + dashboard, + monitoring, + notification_destinations, + reports, + snapshots, + sources, + system, + user_api_keys, +) + +api_v1_router = APIRouter(prefix="/api/v1") +api_v1_router.include_router(system.router) +api_v1_router.include_router(auth.router) +api_v1_router.include_router(admin.router) +api_v1_router.include_router(user_api_keys.router) +api_v1_router.include_router(companies.router) +api_v1_router.include_router(notification_destinations.router) +api_v1_router.include_router(sources.router) +api_v1_router.include_router(snapshots.router) +api_v1_router.include_router(monitoring.router) +api_v1_router.include_router(reports.router) +api_v1_router.include_router(alerts.router) +api_v1_router.include_router(dashboard.router) diff --git a/apps/api/app/api/v1/snapshots.py b/apps/api/app/api/v1/snapshots.py new file mode 100644 index 0000000..7caee67 --- /dev/null +++ b/apps/api/app/api/v1/snapshots.py @@ -0,0 +1,27 @@ +"""Read-only snapshot-history route. Snapshots are written internally by +collection_service.py during monitoring runs - nothing here creates one.""" + +from __future__ import annotations + +import uuid + +from fastapi import APIRouter, Depends +from sqlalchemy.ext.asyncio import AsyncSession + +from app.auth.dependencies import get_current_user +from app.db.session import get_db +from app.models.user import User +from app.schemas.snapshot import SnapshotResponse +from app.services import snapshot_service + +router = APIRouter(tags=["snapshots"]) + + +@router.get("/companies/{company_id}/snapshots", response_model=list[SnapshotResponse]) +async def list_snapshots( + company_id: uuid.UUID, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> list[SnapshotResponse]: + snapshots = await snapshot_service.list_snapshots(db, user.id, company_id) + return [SnapshotResponse.model_validate(s) for s in snapshots] diff --git a/apps/api/app/api/v1/sources.py b/apps/api/app/api/v1/sources.py new file mode 100644 index 0000000..efea444 --- /dev/null +++ b/apps/api/app/api/v1/sources.py @@ -0,0 +1,83 @@ +"""Source routes. Mixes `/companies/{company_id}/sources` and +`/sources/{source_id}` paths per the spec - kept in one router since both +share the same schemas/service.""" + +from __future__ import annotations + +import uuid + +from fastapi import APIRouter, Depends, Request, status +from sqlalchemy.ext.asyncio import AsyncSession + +from app.auth.dependencies import get_current_user +from app.core.config import Settings, get_settings +from app.core.rate_limit import limiter +from app.db.session import get_db +from app.models.user import User +from app.schemas.source import SourceCreate, SourceResponse, SourceTestResult, SourceUpdate +from app.services import source_service + +router = APIRouter(tags=["sources"]) + + +@router.get("/companies/{company_id}/sources", response_model=list[SourceResponse]) +async def list_sources( + company_id: uuid.UUID, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> list[SourceResponse]: + sources = await source_service.list_sources(db, user.id, company_id) + return [SourceResponse.model_validate(s) for s in sources] + + +@router.post( + "/companies/{company_id}/sources", + response_model=SourceResponse, + status_code=status.HTTP_201_CREATED, +) +@limiter.limit("30/minute") +async def create_source( + request: Request, + company_id: uuid.UUID, + payload: SourceCreate, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> SourceResponse: + source = await source_service.create_source(db, user.id, company_id, payload) + return SourceResponse.model_validate(source) + + +@router.patch("/sources/{source_id}", response_model=SourceResponse) +async def update_source( + source_id: uuid.UUID, + payload: SourceUpdate, + user: User = Depends(get_current_user), + settings: Settings = Depends(get_settings), + db: AsyncSession = Depends(get_db), +) -> SourceResponse: + source = await source_service.update_source(db, settings, user.id, source_id, payload) + return SourceResponse.model_validate(source) + + +@router.delete("/sources/{source_id}", status_code=status.HTTP_204_NO_CONTENT) +async def delete_source( + source_id: uuid.UUID, + user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> None: + await source_service.delete_source(db, user.id, source_id) + + +@router.post("/sources/{source_id}/test", response_model=SourceTestResult) +@limiter.limit("20/minute") +async def test_source( + request: Request, + source_id: uuid.UUID, + user: User = Depends(get_current_user), + settings: Settings = Depends(get_settings), + db: AsyncSession = Depends(get_db), +) -> SourceTestResult: + result = await source_service.test_source(db, settings, user.id, source_id) + return SourceTestResult( + status=result.status, documents_found=len(result.documents), error=result.error + ) diff --git a/apps/api/app/api/v1/system.py b/apps/api/app/api/v1/system.py new file mode 100644 index 0000000..edf18b8 --- /dev/null +++ b/apps/api/app/api/v1/system.py @@ -0,0 +1,196 @@ +"""Health/readiness/system-status endpoints. + +Kept dependency-light on purpose: `/health` must answer even if the database +or Redis is down, so infra can tell "process is up" apart from "process is +ready to serve traffic". +""" + +from __future__ import annotations + +from typing import Literal + +import httpx +import redis.asyncio as redis_asyncio +from fastapi import APIRouter, Depends, Request +from pydantic import BaseModel +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +from app.auth.dependencies import require_admin +from app.core.config import Settings, get_settings +from app.core.logging import get_logger, get_recent_logs +from app.core.security import get_client_ip, is_localhost +from app.db.session import get_db +from app.models.enums import SystemSecretKey +from app.models.user import User +from app.schemas.system_secret import SetSystemSecretRequest, SystemSecretStatus +from app.services import system_secret_service +from app.services.enrichment_service import estimate_max_credits_per_company + +logger = get_logger(__name__) + +router = APIRouter(tags=["system"]) + + +class HealthResponse(BaseModel): + status: Literal["ok"] = "ok" + app_name: str + + +class ComponentStatus(BaseModel): + name: str + status: Literal["ok", "error"] + detail: str | None = None + + +class ReadyResponse(BaseModel): + status: Literal["ready", "not_ready"] + components: list[ComponentStatus] + + +class SystemStatusResponse(BaseModel): + app_env: str + auth_mode: str + llm_provider: str + search_provider: str + sms_enabled: bool + sms_provider: str + ninjapear_configured: bool + ninjapear_credit_balance: int | None + ninjapear_estimated_credits_per_company: int | None + is_localhost: bool + # Public by design (meant to be embedded in the frontend bundle/page) - + # resolved through system_secret_service so an admin-updated value here + # takes effect immediately, without a frontend rebuild. None when + # Turnstile isn't configured at all (neither .env nor admin-set). + turnstile_site_key: str | None + components: list[ComponentStatus] + + +class LogEntryResponse(BaseModel): + ts: str + level: str + category: str + logger: str + event: str + context: dict + + +@router.get("/health", response_model=HealthResponse) +async def health(settings: Settings = Depends(get_settings)) -> HealthResponse: + return HealthResponse(app_name=settings.app_name) + + +async def _check_database(db: AsyncSession) -> ComponentStatus: + try: + await db.execute(text("SELECT 1")) + return ComponentStatus(name="database", status="ok") + except Exception as exc: # pragma: no cover - defensive + return ComponentStatus(name="database", status="error", detail=str(exc)) + + +async def _check_redis(settings: Settings) -> ComponentStatus: + try: + client = redis_asyncio.from_url(settings.redis_url, socket_connect_timeout=2) + await client.ping() + await client.aclose() + return ComponentStatus(name="redis", status="ok") + except Exception as exc: # pragma: no cover - defensive + return ComponentStatus(name="redis", status="error", detail=str(exc)) + + +@router.get("/ready", response_model=ReadyResponse) +async def ready( + settings: Settings = Depends(get_settings), db: AsyncSession = Depends(get_db) +) -> ReadyResponse: + components = [await _check_database(db), await _check_redis(settings)] + overall = "ready" if all(c.status == "ok" for c in components) else "not_ready" + return ReadyResponse(status=overall, components=components) + + +async def _get_ninjapear_credit_balance(settings: Settings) -> int | None: + """Free endpoint, safe to call on every status check - never lets a + failure here break the rest of /system/status.""" + if not settings.ninjapear_api_key: + return None + try: + async with httpx.AsyncClient(timeout=10) as client: + response = await client.get( + "https://nubela.co/api/v1/meta/credit-balance", + headers={"Authorization": f"Bearer {settings.ninjapear_api_key}"}, + ) + response.raise_for_status() + data = response.json() + return data.get("credit_balance") or data.get("balance") + except Exception as exc: # pragma: no cover - defensive, status must not 500 on this + logger.warning("ninjapear_credit_balance_check_failed", error=str(exc)) + return None + + +@router.get("/system/status", response_model=SystemStatusResponse) +async def system_status( + request: Request, settings: Settings = Depends(get_settings), db: AsyncSession = Depends(get_db) +) -> SystemStatusResponse: + components = [await _check_database(db), await _check_redis(settings)] + effective = await system_secret_service.get_effective_settings(db, settings) + return SystemStatusResponse( + app_env=settings.app_env, + auth_mode=settings.auth_mode, + llm_provider=settings.llm_provider, + search_provider=settings.search_provider, + sms_enabled=settings.notification_sms_enabled, + sms_provider=settings.sms_provider, + ninjapear_configured=bool(settings.ninjapear_api_key), + ninjapear_credit_balance=await _get_ninjapear_credit_balance(settings), + ninjapear_estimated_credits_per_company=( + estimate_max_credits_per_company(settings.ninjapear_max_leadership_lookups) + if settings.ninjapear_api_key + else None + ), + is_localhost=is_localhost(request, settings), + turnstile_site_key=effective.turnstile_site_key or None, + components=components, + ) + + +@router.get("/system/secrets", response_model=list[SystemSecretStatus]) +async def list_system_secrets( + settings: Settings = Depends(get_settings), + db: AsyncSession = Depends(get_db), + _admin: User = Depends(require_admin), +) -> list[SystemSecretStatus]: + """Admin-only visibility into server-wide secrets (Turnstile site + key/secret) - see app.services.system_secret_service.""" + statuses = await system_secret_service.list_status(db, settings) + return [SystemSecretStatus(**s) for s in statuses] + + +@router.put("/system/secrets/{key}", response_model=SystemSecretStatus) +async def set_system_secret( + request: Request, + key: SystemSecretKey, + payload: SetSystemSecretRequest, + settings: Settings = Depends(get_settings), + db: AsyncSession = Depends(get_db), + admin: User = Depends(require_admin), +) -> SystemSecretStatus: + await system_secret_service.set_secret( + db, + key, + payload.value, + settings, + admin_user_id=admin.id, + client_ip=get_client_ip(request, settings), + ) + statuses = await system_secret_service.list_status(db, settings) + match = next(s for s in statuses if s["key"] == key.value) + return SystemSecretStatus(**match) + + +@router.get("/system/logs", response_model=list[LogEntryResponse]) +async def system_logs( + settings: Settings = Depends(get_settings), _admin: User = Depends(require_admin) +) -> list[LogEntryResponse]: + """Most-recent-first view into the application's live log stream (capped + at the last 500 entries app-wide, see `core/logging.py`).""" + return [LogEntryResponse(**entry) for entry in await get_recent_logs(settings, limit=100)] diff --git a/apps/api/app/api/v1/user_api_keys.py b/apps/api/app/api/v1/user_api_keys.py new file mode 100644 index 0000000..69fa101 --- /dev/null +++ b/apps/api/app/api/v1/user_api_keys.py @@ -0,0 +1,46 @@ +"""Per-user API key management - each user's own keys, visible and +editable only by themselves. Thin per ARCHITECTURE.md - logic lives in +app.services.user_api_key_service.""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends, Request +from sqlalchemy.ext.asyncio import AsyncSession + +from app.auth.dependencies import get_current_user +from app.core.config import Settings, get_settings +from app.core.security import get_client_ip +from app.db.session import get_db +from app.models.enums import ApiKeyProvider +from app.models.user import User +from app.schemas.user_api_key import SetUserApiKeyRequest, UserApiKeyStatus +from app.services import user_api_key_service + +router = APIRouter(prefix="/user-api-keys", tags=["user-api-keys"]) + + +@router.get("", response_model=list[UserApiKeyStatus]) +async def list_user_api_keys( + db: AsyncSession = Depends(get_db), + settings: Settings = Depends(get_settings), + user: User = Depends(get_current_user), +) -> list[UserApiKeyStatus]: + statuses = await user_api_key_service.list_status(db, user.id, settings) + return [UserApiKeyStatus(**s) for s in statuses] + + +@router.put("/{provider}", response_model=UserApiKeyStatus) +async def set_user_api_key( + request: Request, + provider: ApiKeyProvider, + payload: SetUserApiKeyRequest, + db: AsyncSession = Depends(get_db), + settings: Settings = Depends(get_settings), + user: User = Depends(get_current_user), +) -> UserApiKeyStatus: + await user_api_key_service.set_key( + db, user.id, provider, payload.key, settings, client_ip=get_client_ip(request, settings) + ) + statuses = await user_api_key_service.list_status(db, user.id, settings) + match = next(s for s in statuses if s["provider"] == provider.value) + return UserApiKeyStatus(**match) diff --git a/apps/api/app/auth/__init__.py b/apps/api/app/auth/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/app/auth/dependencies.py b/apps/api/app/auth/dependencies.py new file mode 100644 index 0000000..fdebb33 --- /dev/null +++ b/apps/api/app/auth/dependencies.py @@ -0,0 +1,69 @@ +"""FastAPI dependency implementing the `AuthProvider` contract described in +ARCHITECTURE.md: `get_current_user` always returns a `User` row or raises +401, regardless of caller. This is the seam a future Firebase Auth +integration would replace. + +When AUTH_MODE=local (the default), the fixed local-dev user is only +returned to a request that's actually from loopback (see +`app.core.security.is_localhost`) - anyone reaching the API from a LAN or +WAN connection still needs a real bearer token, even with that setting. +AUTH_MODE=jwt disables the loopback convenience entirely (required in +production, see `Settings._forbid_local_auth_in_production`). +""" + +from __future__ import annotations + +from fastapi import Depends, Header, HTTPException, Request, status +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.config import Settings, get_settings +from app.core.security import ( + InvalidTokenError, + TokenType, + decode_token, + get_client_ip, + is_localhost, +) +from app.db.session import get_db +from app.models.user import User +from app.repositories.user_repository import UserRepository +from app.services.auth_service import get_or_create_local_user + + +async def get_current_user( + request: Request, + authorization: str | None = Header(default=None), + settings: Settings = Depends(get_settings), + db: AsyncSession = Depends(get_db), +) -> User: + if settings.auth_mode == "local" and is_localhost(request, settings): + return await get_or_create_local_user(db, get_client_ip(request, settings)) + + if authorization is None or not authorization.lower().startswith("bearer "): + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated") + + token = authorization.split(" ", 1)[1] + try: + decoded = decode_token(token, settings, TokenType.ACCESS) + except InvalidTokenError as exc: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or expired access token", + ) from exc + + repo = UserRepository(db) + user = await repo.get_by_id(decoded.user_id) + if user is None or not user.is_active: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or expired access token", + ) + return user + + +async def require_admin(user: User = Depends(get_current_user)) -> User: + if not user.is_admin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, detail="Admin privileges required" + ) + return user diff --git a/apps/api/app/change_detection/__init__.py b/apps/api/app/change_detection/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/app/change_detection/extractors.py b/apps/api/app/change_detection/extractors.py new file mode 100644 index 0000000..fb19f48 --- /dev/null +++ b/apps/api/app/change_detection/extractors.py @@ -0,0 +1,25 @@ +"""Lightweight, best-effort regex extractors for specific signal types the +severity model treats specially (pricing, leadership). These are heuristics, +not NLP - they exist to catch the common "$X/month" and "named a new CEO" +phrasings, not to parse arbitrary text reliably. Phase 7's LLM extraction +task is the higher-fidelity version of this; these run cheaply and +deterministically as part of scoring, without a model call. +""" + +from __future__ import annotations + +import re + +_PRICE_RE = re.compile(r"\$\s?\d[\d,]*(?:\.\d{2})?\s*(?:/\s*(?:month|mo|year|yr))?") +_LEADERSHIP_TITLE_RE = re.compile( + r"(?i)\b(Chief Executive Officer|CEO|Chief Financial Officer|CFO|Chief Technology Officer|" + r"CTO|President|Chairman|Chairwoman|Chairperson)\b" +) + + +def extract_prices(text: str) -> set[str]: + return set(_PRICE_RE.findall(text or "")) + + +def mentions_leadership_title(text: str) -> bool: + return bool(_LEADERSHIP_TITLE_RE.search(text or "")) diff --git a/apps/api/app/change_detection/noise_filters.py b/apps/api/app/change_detection/noise_filters.py new file mode 100644 index 0000000..e1c4e4d --- /dev/null +++ b/apps/api/app/change_detection/noise_filters.py @@ -0,0 +1,32 @@ +"""Strips content that changes on every fetch but carries no meaning, so it +never counts toward a text diff. Applied before Layer 3 (text diff) - see +ARCHITECTURE.md and spec section 18. +""" + +from __future__ import annotations + +import re + +_NOISE_PATTERNS = [ + # Dynamic timestamps: "Updated: 2026-01-01", "Last modified 01/02/2026 14:30" + re.compile(r"(?i)\b(updated|last modified|generated|retrieved)\s*:?\s*[\d/:\-\sTZ]+"), + # Session/CSRF-style tokens embedded in visible text (rare, but happens on thin pages) + re.compile(r"\b[A-Za-z0-9_-]{24,}\b"), + # Cookie/consent banner boilerplate + re.compile(r"(?i)we use cookies[^.]*\.?"), + re.compile(r"(?i)by (continuing|using this site)[^.]*\.?"), + # Copyright year lines, which change every January with no real signal + re.compile(r"(?i)copyright\s*(?:©|\(c\))?\s*\d{4}[–\-]?\d{0,4}"), + # View/like/share counters + re.compile(r"(?i)\b\d[\d,]*\s*(views|likes|shares)\b"), +] + +_WHITESPACE_RUN = re.compile(r"[ \t]{2,}") +_BLANK_LINES = re.compile(r"\n{3,}") + + +def strip_noise(text: str) -> str: + for pattern in _NOISE_PATTERNS: + text = pattern.sub(" ", text) + text = _WHITESPACE_RUN.sub(" ", text) + return _BLANK_LINES.sub("\n\n", text).strip() diff --git a/apps/api/app/change_detection/scoring.py b/apps/api/app/change_detection/scoring.py new file mode 100644 index 0000000..4dd19ca --- /dev/null +++ b/apps/api/app/change_detection/scoring.py @@ -0,0 +1,103 @@ +"""Layer 5: significance scoring + severity classification. + +This is the documented, unit-tested formula referenced in ARCHITECTURE.md. +Deterministic on purpose - severity must be explainable and reproducible +without a model call. Phase 7's LLM analysis narrates *why* a change +matters; it does not decide *how much* it matters. + + significance = base_weight + * source_trust_score (0.3 - 1.0) + * min(1.0, independent_sources / 2) # corroboration, caps at 2 sources + * focus_match_multiplier (1.3 if it matches the user's stated focus, else 1.0) + * recency_multiplier (1.0 if new, 0.5 if a repeat of a recent change) + + confidence = clamp( + 0.5 * extraction_confidence + 0.4 * source_trust_score + 0.1 + + (0.15 if independent_sources >= 2 else 0.0), + 0.0, 1.0 + ) + + severity = bucket(significance * confidence), with a hard floor: + CRITICAL requires confidence >= CRITICAL_MIN_CONFIDENCE regardless of score - + an uncorroborated single-source signal can never be labeled Critical. +""" + +from __future__ import annotations + +from app.models.enums import ChangeType, SeverityLevel + +BASE_WEIGHTS: dict[ChangeType, float] = { + ChangeType.LEADERSHIP_CHANGE: 0.9, + ChangeType.FILING_NEW: 0.85, + ChangeType.PRICE_CHANGE: 0.6, + ChangeType.NEW_DOCUMENT: 0.5, + ChangeType.CONTENT_MODIFIED: 0.3, # further scaled by diff_ratio - see compute_significance + ChangeType.REMOVED_DOCUMENT: 0.3, +} + +# (score_threshold, severity) - first match wins, checked highest first. +# Calibrated against compute_significance/compute_confidence's actual output +# range (a single-source signal is already discounted ~2x by the +# corroboration multiplier) so CRITICAL_MIN_CONFIDENCE below is reachable: +# with significance capped at 1.0, a score of 0.6 needs confidence >= 0.6, +# which leaves room for the 0.7 confidence floor to actually bite and +# downgrade a subset of would-be-CRITICAL cases to HIGH. +SEVERITY_THRESHOLDS: list[tuple[float, SeverityLevel]] = [ + (0.6, SeverityLevel.CRITICAL), + (0.4, SeverityLevel.HIGH), + (0.2, SeverityLevel.MEDIUM), + (0.0, SeverityLevel.LOW), +] + +CRITICAL_MIN_CONFIDENCE = 0.7 + + +def compute_significance( + *, + change_type: ChangeType, + source_trust_score: float, + independent_source_count: int = 1, + focus_match: bool = False, + is_repeat: bool = False, + diff_ratio: float | None = None, +) -> float: + base = BASE_WEIGHTS[change_type] + if change_type is ChangeType.CONTENT_MODIFIED and diff_ratio is not None: + # A one-line wording tweak and a full page rewrite are both + # "content_modified" but shouldn't score the same. + base = base + diff_ratio * 0.5 + + trust_multiplier = _clamp(source_trust_score, 0.3, 1.0) + corroboration_multiplier = min(1.0, independent_source_count / 2) + focus_multiplier = 1.3 if focus_match else 1.0 + recency_multiplier = 0.5 if is_repeat else 1.0 + + significance = ( + base * trust_multiplier * corroboration_multiplier * focus_multiplier * recency_multiplier + ) + return round(_clamp(significance, 0.0, 1.0), 4) + + +def compute_confidence( + *, + extraction_confidence: float, + source_trust_score: float, + independent_source_count: int = 1, +) -> float: + corroboration_bonus = 0.15 if independent_source_count >= 2 else 0.0 + confidence = 0.5 * extraction_confidence + 0.4 * source_trust_score + 0.1 + corroboration_bonus + return round(_clamp(confidence, 0.0, 1.0), 4) + + +def classify_severity(significance: float, confidence: float) -> SeverityLevel: + score = significance * confidence + for threshold, severity in SEVERITY_THRESHOLDS: + if score >= threshold: + if severity is SeverityLevel.CRITICAL and confidence < CRITICAL_MIN_CONFIDENCE: + return SeverityLevel.HIGH + return severity + return SeverityLevel.LOW # pragma: no cover - thresholds bottom out at 0.0 + + +def _clamp(value: float, lo: float, hi: float) -> float: + return max(lo, min(hi, value)) diff --git a/apps/api/app/change_detection/structured_diff.py b/apps/api/app/change_detection/structured_diff.py new file mode 100644 index 0000000..a4b46e3 --- /dev/null +++ b/apps/api/app/change_detection/structured_diff.py @@ -0,0 +1,33 @@ +"""Layer 2: structured field comparison. + +Compares the *set* of items (job postings, press releases, filings, +products - whatever the source's documents represent) between two +snapshots' `structured_summary["urls"]`/`["titles"]`. This is what catches +"a new job posting appeared" or "a press release was removed" without +needing a bespoke parser per source type - collection_service already +records the full current item set on every run, so this is a plain set +diff between two runs. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass(frozen=True) +class StructuredDiff: + added: list[str] = field(default_factory=list) + removed: list[str] = field(default_factory=list) + + @property + def has_changes(self) -> bool: + return bool(self.added or self.removed) + + +def diff_item_sets(previous_urls: list[str], current_urls: list[str]) -> StructuredDiff: + previous_set = set(previous_urls) + current_set = set(current_urls) + return StructuredDiff( + added=sorted(current_set - previous_set), + removed=sorted(previous_set - current_set), + ) diff --git a/apps/api/app/change_detection/text_diff.py b/apps/api/app/change_detection/text_diff.py new file mode 100644 index 0000000..d65243c --- /dev/null +++ b/apps/api/app/change_detection/text_diff.py @@ -0,0 +1,56 @@ +"""Layer 3: bounded text diff. + +Runs on noise-stripped text (see noise_filters.py) so navigation/cookie/ +timestamp churn doesn't register as a change. Bounded: only a capped number +of added/removed lines are kept, so a full page rewrite doesn't produce an +unbounded diff blob for storage or LLM consumption later. +""" + +from __future__ import annotations + +import difflib +from dataclasses import dataclass, field + +from app.change_detection.noise_filters import strip_noise + +_MAX_DIFF_LINES = 40 + + +@dataclass(frozen=True) +class TextDiffResult: + diff_ratio: float # 0.0 = identical, 1.0 = completely different + added_lines: list[str] = field(default_factory=list) + removed_lines: list[str] = field(default_factory=list) + + @property + def is_identical(self) -> bool: + return self.diff_ratio == 0.0 + + +def bounded_text_diff(previous_text: str, current_text: str) -> TextDiffResult: + previous_clean = strip_noise(previous_text or "") + current_clean = strip_noise(current_text or "") + + if previous_clean == current_clean: + return TextDiffResult(diff_ratio=0.0) + + previous_lines = [line for line in previous_clean.splitlines() if line.strip()] + current_lines = [line for line in current_clean.splitlines() if line.strip()] + + matcher = difflib.SequenceMatcher(a=previous_lines, b=current_lines, autojunk=False) + similarity = matcher.ratio() + diff_ratio = round(1.0 - similarity, 4) + + added: list[str] = [] + removed: list[str] = [] + for tag, i1, i2, j1, j2 in matcher.get_opcodes(): + if tag in ("replace", "delete"): + removed.extend(previous_lines[i1:i2]) + if tag in ("replace", "insert"): + added.extend(current_lines[j1:j2]) + + return TextDiffResult( + diff_ratio=diff_ratio, + added_lines=added[:_MAX_DIFF_LINES], + removed_lines=removed[:_MAX_DIFF_LINES], + ) diff --git a/apps/api/app/collectors/__init__.py b/apps/api/app/collectors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/app/collectors/base.py b/apps/api/app/collectors/base.py new file mode 100644 index 0000000..26c2076 --- /dev/null +++ b/apps/api/app/collectors/base.py @@ -0,0 +1,96 @@ +"""Collector interface. Every source type (website, RSS, SEC EDGAR, GitHub, +custom URL, job postings, and the fixture-backed patent/review adapters) +implements this same `SourceCollector` protocol, so `tasks/collection.py` +(Phase 5) can treat them uniformly. + +Collectors never talk to the database - they take plain dataclasses in and +return plain dataclasses out. Persisting `CollectedDocument`s into +`SourceDocument` rows is the caller's job (a service function, not the +collector), which keeps collectors trivially unit-testable against fixtures. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Protocol + +from app.models.enums import SourceStatus, SourceType + + +@dataclass(frozen=True) +class CompanyContext: + """Read-only view of a Company, passed into collectors instead of the ORM object.""" + + id: str + name: str + official_website: str | None + monitoring_focus: str | None + aliases: list[str] = field(default_factory=list) + competitors: list[str] = field(default_factory=list) + # From NinjaPear enrichment (app/models/company_enrichment.py), when it + # ran and found a leadership team - empty otherwise (no key, still + # pending, or no leadership data). Used by PatentSourceCollector to + # search USPTO by inventor name, since that endpoint has no queryable + # company/assignee field at all - see collectors/patents.py. + leadership_names: list[str] = field(default_factory=list) + # The owning user's effective USPTO key (their own, or the server's + # global one) - None means "use the server's global settings.uspto_api_key + # directly", for call sites that never resolved a per-user override + # (e.g. discovery preview paths outside a monitoring run). + uspto_api_key: str | None = None + + +@dataclass(frozen=True) +class DiscoveredSource: + source_type: SourceType + name: str + base_url: str | None + configuration_metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class SourceConfig: + id: str + source_type: SourceType + name: str + base_url: str | None + configuration_metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class CollectedDocument: + url: str + canonical_url: str + title: str | None + author: str | None + publication_date: datetime | None + retrieved_date: datetime + content_text: str + content_hash: str + metadata: dict[str, Any] = field(default_factory=dict) + language: str | None = None + http_status: int | None = None + extraction_method: str = "unknown" + trust_score: float = 0.7 + + +@dataclass +class CollectionResult: + status: SourceStatus + documents: list[CollectedDocument] = field(default_factory=list) + error: str | None = None + pages_attempted: int = 0 + + +class SourceCollector(Protocol): + source_type: SourceType + + async def discover(self, company: CompanyContext) -> list[DiscoveredSource]: + """Suggest sources for a newly added company. May return an empty + list if this collector type can't be auto-discovered (e.g. patents).""" + ... + + async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult: + """Fetch and extract current content for a configured source.""" + ... diff --git a/apps/api/app/collectors/custom_url.py b/apps/api/app/collectors/custom_url.py new file mode 100644 index 0000000..8e56ae6 --- /dev/null +++ b/apps/api/app/collectors/custom_url.py @@ -0,0 +1,87 @@ +"""User-supplied custom URL collector - fetches, extracts, and monitors a +single public URL the user explicitly added.""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import httpx + +from app.collectors.base import ( + CollectedDocument, + CollectionResult, + CompanyContext, + DiscoveredSource, + SourceConfig, +) +from app.collectors.extraction import ( + canonicalize_url, + compute_content_hash, + extract_readable_text, + extract_title, +) +from app.collectors.robots import is_allowed +from app.core.config import get_settings +from app.core.http import FetchError, SsrfBlockedError, fetch_with_retries +from app.models.enums import SourceStatus, SourceType + + +class CustomUrlCollector: + source_type = SourceType.CUSTOM_URL + + async def discover(self, company: CompanyContext) -> list[DiscoveredSource]: + return [] # Custom URLs are always user-supplied, never auto-discovered. + + async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult: + if not source.base_url: + return CollectionResult(status=SourceStatus.FAILED, error="No URL configured") + + settings = get_settings() + try: + if not await is_allowed(source.base_url, settings=settings): + return CollectionResult( + status=SourceStatus.BLOCKED_BY_POLICY, + error="Disallowed by robots.txt", + pages_attempted=1, + ) + + result = await fetch_with_retries(source.base_url, settings=settings) + except SsrfBlockedError as exc: + return CollectionResult( + status=SourceStatus.BLOCKED_BY_POLICY, error=str(exc), pages_attempted=1 + ) + except (FetchError, httpx.HTTPError) as exc: + return CollectionResult(status=SourceStatus.FAILED, error=str(exc), pages_attempted=1) + + if result.status_code in (401, 403): + return CollectionResult( + status=SourceStatus.AUTH_REQUIRED, + error=f"HTTP {result.status_code}", + pages_attempted=1, + ) + if result.status_code >= 400: + return CollectionResult( + status=SourceStatus.FAILED, error=f"HTTP {result.status_code}", pages_attempted=1 + ) + + text, method = extract_readable_text(result.text, source.base_url) + if not text: + return CollectionResult( + status=SourceStatus.FAILED, error="No extractable content", pages_attempted=1 + ) + + document = CollectedDocument( + url=source.base_url, + canonical_url=canonicalize_url(result.final_url), + title=extract_title(result.text), + author=None, + publication_date=None, + retrieved_date=datetime.now(UTC), + content_text=text, + content_hash=compute_content_hash(text), + metadata={"http_status": result.status_code}, + extraction_method=method, + http_status=result.status_code, + trust_score=0.7, + ) + return CollectionResult(status=SourceStatus.ACTIVE, documents=[document], pages_attempted=1) diff --git a/apps/api/app/collectors/extraction.py b/apps/api/app/collectors/extraction.py new file mode 100644 index 0000000..fa75ed7 --- /dev/null +++ b/apps/api/app/collectors/extraction.py @@ -0,0 +1,76 @@ +"""Shared content extraction/normalization helpers used by every collector. + +Centralizing this (rather than letting each collector roll its own) is what +makes cross-collector dedup and hashing behave consistently. +""" + +from __future__ import annotations + +import hashlib +import re +from urllib.parse import urlsplit, urlunsplit + +import trafilatura +from bs4 import BeautifulSoup + +_WHITESPACE_RE = re.compile(r"[ \t\f\v]+") +_BLANK_LINES_RE = re.compile(r"\n{3,}") + +# Query params that vary per-request/session but don't change page meaning - +# stripped so the same logical page always canonicalizes identically. +_NOISE_QUERY_PREFIXES = ("utm_", "fbclid", "gclid", "mc_", "_hs") + + +def normalize_whitespace(text: str) -> str: + text = text.replace("\r\n", "\n").replace("\r", "\n") + text = _WHITESPACE_RE.sub(" ", text) + lines = [line.strip() for line in text.split("\n")] + text = "\n".join(lines) + return _BLANK_LINES_RE.sub("\n\n", text).strip() + + +def compute_content_hash(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def canonicalize_url(url: str) -> str: + parts = urlsplit(url) + query_pairs = [ + pair + for pair in parts.query.split("&") + if pair and not pair.split("=")[0].startswith(_NOISE_QUERY_PREFIXES) + ] + path = parts.path.rstrip("/") or "/" + return urlunsplit((parts.scheme.lower(), parts.netloc.lower(), path, "&".join(query_pairs), "")) + + +def extract_readable_text(html: str, url: str) -> tuple[str, str]: + """Returns (text, extraction_method). Prefers trafilatura (boilerplate + removal tuned for articles/press releases); falls back to a plain + BeautifulSoup text extraction for pages trafilatura can't parse (e.g. + thin job listing pages).""" + extracted = trafilatura.extract( + html, + url=url, + include_comments=False, + include_tables=True, + favor_precision=True, + ) + if extracted and extracted.strip(): + return normalize_whitespace(extracted), "trafilatura" + + soup = BeautifulSoup(html, "lxml") + for tag in soup(["script", "style", "nav", "footer", "header", "noscript"]): + tag.decompose() + text = soup.get_text(separator="\n") + return normalize_whitespace(text), "beautifulsoup_fallback" + + +def extract_title(html: str) -> str | None: + soup = BeautifulSoup(html, "lxml") + if soup.title and soup.title.string: + return soup.title.string.strip() + h1 = soup.find("h1") + if h1: + return h1.get_text(strip=True) + return None diff --git a/apps/api/app/collectors/github.py b/apps/api/app/collectors/github.py new file mode 100644 index 0000000..2e001c2 --- /dev/null +++ b/apps/api/app/collectors/github.py @@ -0,0 +1,147 @@ +"""GitHub collector - public organization/repository metadata via the +public REST API. `GITHUB_TOKEN` is optional and only raises the rate limit; +nothing here requires authentication. +""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime + +import httpx + +from app.collectors.base import ( + CollectedDocument, + CollectionResult, + CompanyContext, + DiscoveredSource, + SourceConfig, +) +from app.collectors.extraction import compute_content_hash, normalize_whitespace +from app.core.config import Settings, get_settings +from app.core.http import FetchError, SsrfBlockedError, fetch_with_retries +from app.core.logging import get_logger +from app.models.enums import SourceStatus, SourceType + +logger = get_logger(__name__) + +_API_BASE = "https://api.github.com" + + +def _auth_headers(settings: Settings) -> dict[str, str]: + headers = {"Accept": "application/vnd.github+json"} + if settings.github_token: + headers["Authorization"] = f"Bearer {settings.github_token}" + return headers + + +class GithubCollector: + source_type = SourceType.GITHUB + + async def discover(self, company: CompanyContext) -> list[DiscoveredSource]: + settings = get_settings() + org_login = await self._find_org(company.name, settings) + if org_login is None: + return [] + return [ + DiscoveredSource( + source_type=SourceType.GITHUB, + name=f"{company.name} — GitHub", + base_url=f"https://github.com/{org_login}", + configuration_metadata={"org": org_login}, + ) + ] + + async def _find_org(self, company_name: str, settings: Settings) -> str | None: + url = str( + httpx.URL( + f"{_API_BASE}/search/users", + params={"q": f"{company_name} type:org", "per_page": 1}, + ) + ) + try: + result = await fetch_with_retries( + url, settings=settings, max_attempts=2, extra_headers=_auth_headers(settings) + ) + except (SsrfBlockedError, FetchError, httpx.HTTPError) as exc: + logger.warning("github_org_search_failed", company=company_name, error=str(exc)) + return None + if result.status_code != 200: + return None + try: + payload = json.loads(result.text) + except json.JSONDecodeError: + return None + items = payload.get("items", []) + return items[0]["login"] if items else None + + async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult: + org = source.configuration_metadata.get("org") + if not org: + return CollectionResult(status=SourceStatus.FAILED, error="No GitHub org configured") + + settings = get_settings() + url = str( + httpx.URL(f"{_API_BASE}/orgs/{org}/repos", params={"sort": "pushed", "per_page": 15}) + ) + try: + result = await fetch_with_retries( + url, settings=settings, extra_headers=_auth_headers(settings) + ) + except SsrfBlockedError as exc: + return CollectionResult(status=SourceStatus.BLOCKED_BY_POLICY, error=str(exc)) + except (FetchError, httpx.HTTPError) as exc: + return CollectionResult(status=SourceStatus.FAILED, error=str(exc)) + + if result.status_code == 404: + return CollectionResult( + status=SourceStatus.FAILED, error=f"GitHub org not found: {org}" + ) + if result.status_code == 403: + return CollectionResult( + status=SourceStatus.RATE_LIMITED, error="GitHub API rate limited" + ) + if result.status_code >= 400: + return CollectionResult(status=SourceStatus.FAILED, error=f"HTTP {result.status_code}") + + try: + repos = json.loads(result.text) + except json.JSONDecodeError: + return CollectionResult(status=SourceStatus.FAILED, error="Malformed GitHub response") + + documents: list[CollectedDocument] = [] + for repo in repos: + text = normalize_whitespace( + f"{repo.get('full_name')}\n" + f"{repo.get('description') or ''}\n" + f"Language: {repo.get('language') or 'unknown'}\n" + f"Stars: {repo.get('stargazers_count', 0)}\n" + f"Last pushed: {repo.get('pushed_at')}" + ) + documents.append( + CollectedDocument( + url=repo.get("html_url"), + canonical_url=repo.get("html_url"), + title=repo.get("full_name"), + author=org, + publication_date=( + datetime.fromisoformat(repo["pushed_at"].replace("Z", "+00:00")) + if repo.get("pushed_at") + else None + ), + retrieved_date=datetime.now(UTC), + content_text=text, + content_hash=compute_content_hash(text), + metadata={ + "stars": repo.get("stargazers_count"), + "language": repo.get("language"), + "archived": repo.get("archived"), + }, + extraction_method="github_api", + http_status=result.status_code, + trust_score=0.75, + ) + ) + + status = SourceStatus.ACTIVE + return CollectionResult(status=status, documents=documents, pages_attempted=1) diff --git a/apps/api/app/collectors/gov_contracts.py b/apps/api/app/collectors/gov_contracts.py new file mode 100644 index 0000000..5089705 --- /dev/null +++ b/apps/api/app/collectors/gov_contracts.py @@ -0,0 +1,131 @@ +"""Federal contracts collector via USASpending.gov's public Award Search +API - free, keyless, no registration (a fixed, trusted, first-party +integration endpoint like Brave/Twilio, so this calls httpx directly rather +than through `safe_fetch`, which exists to guard arbitrary/user-supplied +collector targets, not our own known API integrations). + +Offered for every company regardless of type, same as SecEdgarCollector - +a private company simply returns zero awards, which is a normal empty +result, not a failure. +""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import httpx + +from app.collectors.base import ( + CollectedDocument, + CollectionResult, + CompanyContext, + DiscoveredSource, + SourceConfig, +) +from app.collectors.extraction import compute_content_hash, normalize_whitespace +from app.core.logging import get_logger +from app.models.enums import SourceStatus, SourceType + +logger = get_logger(__name__) + +_SEARCH_URL = "https://api.usaspending.gov/api/v2/search/spending_by_award/" +_AWARD_TYPE_CODES = ["A", "B", "C", "D"] # contracts (definitive/BPA/purchase order/delivery order) + + +class GovContractCollector: + source_type = SourceType.GOV_CONTRACT + + async def discover(self, company: CompanyContext) -> list[DiscoveredSource]: + return [ + DiscoveredSource( + source_type=SourceType.GOV_CONTRACT, + name=f"{company.name} — Federal Contracts", + base_url=None, + configuration_metadata={"recipient_search_text": company.name}, + ) + ] + + async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult: + recipient = source.configuration_metadata.get("recipient_search_text") or company.name + + body = { + "filters": { + "recipient_search_text": [recipient], + "award_type_codes": _AWARD_TYPE_CODES, + }, + "fields": [ + "Award ID", + "Recipient Name", + "Award Amount", + "Start Date", + "Awarding Agency", + "Description", + ], + "sort": "Award Amount", + "order": "desc", + "page": 1, + "limit": 25, + } + try: + async with httpx.AsyncClient(timeout=20) as client: + response = await client.post(_SEARCH_URL, json=body) + except httpx.HTTPError as exc: + return CollectionResult(status=SourceStatus.FAILED, error=str(exc)) + + if response.status_code >= 400: + return CollectionResult( + status=SourceStatus.FAILED, + error=f"USASpending API error {response.status_code}: {response.text[:200]}", + pages_attempted=1, + ) + + try: + payload = response.json() + except ValueError: + return CollectionResult( + status=SourceStatus.FAILED, + error="Malformed USASpending response", + pages_attempted=1, + ) + + results = payload.get("results", []) + documents: list[CollectedDocument] = [] + for award in results: + award_id = award.get("Award ID", "unknown") + agency = award.get("Awarding Agency", "Unknown agency") + amount = award.get("Award Amount") + amount_display = ( + f"${amount:,.0f}" if isinstance(amount, (int, float)) else "unknown amount" + ) + start_date = award.get("Start Date", "") + description = award.get("Description") or "" + + text = normalize_whitespace( + f"{award.get('Recipient Name', recipient)} was awarded federal contract " + f"{award_id} by {agency} for {amount_display}, starting {start_date}. " + f"{description}" + ) + documents.append( + CollectedDocument( + url=_SEARCH_URL, + canonical_url=_SEARCH_URL, + title=f"{agency}: {award_id} — {amount_display}", + author="USASpending.gov", + publication_date=( + datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=UTC) + if start_date + else None + ), + retrieved_date=datetime.now(UTC), + content_text=text, + content_hash=compute_content_hash(text), + metadata={"award_id": award_id, "awarding_agency": agency}, + extraction_method="usaspending_api", + http_status=response.status_code, + trust_score=0.8, + ) + ) + + # Zero awards isn't a failure - most companies never win a federal + # contract, same non-error empty-result handling as SecEdgarCollector. + return CollectionResult(status=SourceStatus.ACTIVE, documents=documents, pages_attempted=1) diff --git a/apps/api/app/collectors/jobs.py b/apps/api/app/collectors/jobs.py new file mode 100644 index 0000000..c86ce45 --- /dev/null +++ b/apps/api/app/collectors/jobs.py @@ -0,0 +1,156 @@ +"""Job posting collector: generic heuristic extraction from a company's own +careers page. Board-specific APIs (LinkedIn, Indeed, etc.) are not +implemented - most require paid access or prohibit automated collection in +their terms; see KNOWN_LIMITATIONS.md. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from urllib.parse import urljoin + +import httpx +from bs4 import BeautifulSoup + +from app.collectors.base import ( + CollectedDocument, + CollectionResult, + CompanyContext, + DiscoveredSource, + SourceConfig, +) +from app.collectors.extraction import ( + canonicalize_url, + compute_content_hash, + extract_readable_text, + extract_title, + normalize_whitespace, +) +from app.collectors.robots import is_allowed +from app.core.config import get_settings +from app.core.http import FetchError, SsrfBlockedError, fetch_with_retries +from app.models.enums import SourceStatus, SourceType + +_JOB_LINK_KEYWORDS = ("job", "career", "position", "opening", "role", "vacan") +_MAX_LISTINGS = 50 + + +class JobPostingCollector: + source_type = SourceType.JOB_POSTING + + async def discover(self, company: CompanyContext) -> list[DiscoveredSource]: + if not company.official_website: + return [] + careers_url = urljoin(company.official_website, "/careers") + return [ + DiscoveredSource( + source_type=SourceType.JOB_POSTING, + name=f"{company.name} — Careers", + base_url=careers_url, + ) + ] + + async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult: + if not source.base_url: + return CollectionResult(status=SourceStatus.FAILED, error="No careers URL configured") + + settings = get_settings() + try: + if not await is_allowed(source.base_url, settings=settings): + return CollectionResult( + status=SourceStatus.BLOCKED_BY_POLICY, + error="Disallowed by robots.txt", + pages_attempted=1, + ) + result = await fetch_with_retries(source.base_url, settings=settings) + except SsrfBlockedError as exc: + return CollectionResult( + status=SourceStatus.BLOCKED_BY_POLICY, error=str(exc), pages_attempted=1 + ) + except (FetchError, httpx.HTTPError) as exc: + return CollectionResult(status=SourceStatus.FAILED, error=str(exc), pages_attempted=1) + + if result.status_code in (401, 403): + return CollectionResult( + status=SourceStatus.AUTH_REQUIRED, + error=f"HTTP {result.status_code}", + pages_attempted=1, + ) + if result.status_code >= 400: + return CollectionResult( + status=SourceStatus.FAILED, error=f"HTTP {result.status_code}", pages_attempted=1 + ) + + listings = self._extract_job_listings(result.text, result.final_url) + + if not listings: + # Non-standard careers page (e.g. a third-party ATS iframe) - + # fall back to the whole page as one document rather than + # reporting a failure for a page that did load successfully. + text, method = extract_readable_text(result.text, source.base_url) + if not text: + return CollectionResult( + status=SourceStatus.FAILED, error="No extractable content", pages_attempted=1 + ) + document = CollectedDocument( + url=source.base_url, + canonical_url=canonicalize_url(result.final_url), + title=extract_title(result.text), + author=None, + publication_date=None, + retrieved_date=datetime.now(UTC), + content_text=text, + content_hash=compute_content_hash(text), + metadata={"extraction": "fallback_whole_page"}, + extraction_method=method, + http_status=result.status_code, + trust_score=0.55, + ) + return CollectionResult( + status=SourceStatus.ACTIVE, documents=[document], pages_attempted=1 + ) + + documents = [ + CollectedDocument( + url=link, + canonical_url=canonicalize_url(link), + title=title, + author=None, + publication_date=None, + retrieved_date=datetime.now(UTC), + content_text=normalize_whitespace(f"{title}\n{snippet}"), + content_hash=compute_content_hash(normalize_whitespace(f"{title}\n{snippet}")), + metadata={"source_page": source.base_url}, + extraction_method="job_link_heuristic", + http_status=result.status_code, + trust_score=0.65, + ) + for title, link, snippet in listings + ] + return CollectionResult(status=SourceStatus.ACTIVE, documents=documents, pages_attempted=1) + + def _extract_job_listings(self, html: str, base_url: str) -> list[tuple[str, str, str]]: + soup = BeautifulSoup(html, "lxml") + listings: list[tuple[str, str, str]] = [] + seen_links: set[str] = set() + + for anchor in soup.find_all("a", href=True): + href = anchor["href"] + text = anchor.get_text(strip=True) + if not text or len(text) < 4 or len(text) > 150: + continue + if not any(keyword in href.lower() for keyword in _JOB_LINK_KEYWORDS): + continue + + link = urljoin(base_url, href) + if link in seen_links: + continue + seen_links.add(link) + + parent = anchor.find_parent() + snippet = parent.get_text(" ", strip=True)[:300] if parent else "" + listings.append((text, link, snippet)) + if len(listings) >= _MAX_LISTINGS: + break + + return listings diff --git a/apps/api/app/collectors/patents.py b/apps/api/app/collectors/patents.py new file mode 100644 index 0000000..14b8ab1 --- /dev/null +++ b/apps/api/app/collectors/patents.py @@ -0,0 +1,245 @@ +"""Patent source collector. + +USPTO's PatentsView data migrated into the Open Data Portal (ODP) in March +2026; a free API key is available via account registration at +data.uspto.gov/apis/getting-started (see `Settings.uspto_api_key`). Without +a key configured (the default), this collector never fabricates patent +data - it truthfully reports `DISABLED` with an explanation, same as +before this integration existed, and a fixture adapter remains available +for local development/testing. + +With a key configured, `collect()` calls the real ODP Patent Application +Search API - by INVENTOR NAME, not company name. Confirmed live (2026-08) +by inspecting a real response's full field list, including one from a +query that returned 110k+ real results: there is no assignee/company field +anywhere in this endpoint's data model. Company-name search here always +returns "no matching records," even for assignees with thousands of real +patents - it isn't a wrong-field-name bug, the field doesn't exist on this +dataset. USPTO's Patent Application Search reliably supports inventor-name +and application-number lookups only. + +So `collect()` instead searches by each of the company's known leadership +names (from NinjaPear enrichment, see `CompanyContext.leadership_names` / +`enrichment_service.py`) and treats a match as a heuristic company signal, +not a verified one - there is still no way to confirm a given patent +actually belongs to the monitored company rather than, say, a same-named +person, or work the person did at a prior employer. Every resulting +document is trust-scored lower (0.5, vs. a hypothetical verified-assignee +match) and its content explicitly says which leadership name it matched +on, so the report LLM's confidence labeling reflects this rather than +treating it as confirmed fact. With no leadership names available (no +NinjaPear key, enrichment still pending, or it returned no leadership +data), this reports an honest empty result without making a network call +- there's nothing meaningful to search USPTO for. +""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from pathlib import Path + +import httpx + +from app.collectors.base import ( + CollectedDocument, + CollectionResult, + CompanyContext, + DiscoveredSource, + SourceConfig, +) +from app.collectors.extraction import compute_content_hash, normalize_whitespace +from app.core.config import get_settings +from app.core.logging import get_logger +from app.models.enums import SourceStatus, SourceType + +logger = get_logger(__name__) + +_FIXTURES_DIR = Path(__file__).resolve().parents[2] / "tests" / "fixtures" / "patents" +_SEARCH_URL = "https://api.uspto.gov/api/v1/patent/applications/search" +_MAX_INVENTOR_SEARCHES = 5 +_MAX_DOCUMENTS = 25 + + +class PatentSourceCollector: + source_type = SourceType.PATENT + + async def discover(self, company: CompanyContext) -> list[DiscoveredSource]: + # company.uspto_api_key, when set by the caller (see + # collection_service.to_company_context), is the resolved effective + # key for whichever user owns this company - their own if they've + # set one, else the server's global default. None means no + # per-user resolution happened for this call path, so fall back to + # the global settings directly. + api_key = company.uspto_api_key or get_settings().uspto_api_key + if not api_key: + return [] # No live discovery without a configured provider. + return [ + DiscoveredSource( + source_type=SourceType.PATENT, + name=f"{company.name} — Patent Filings", + base_url=None, + configuration_metadata={"assignee": company.name}, + ) + ] + + async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult: + api_key = company.uspto_api_key or get_settings().uspto_api_key + if api_key: + return await self._collect_live(company.name, company.leadership_names, api_key) + + fixture_key = source.configuration_metadata.get("fixture_key") + if not fixture_key: + return CollectionResult( + status=SourceStatus.DISABLED, + error=( + "No patent data provider is configured. This collector implements the " + "SourceCollector interface for a live integration (USPTO Open Data Portal) " + "but does not fabricate results without a configured USPTO_API_KEY." + ), + ) + + fixture_path = _FIXTURES_DIR / f"{fixture_key}.json" + if not fixture_path.exists(): + return CollectionResult( + status=SourceStatus.DISABLED, + error=f"No fixture found for {fixture_key!r} and no live provider is configured.", + ) + + payload = json.loads(fixture_path.read_text(encoding="utf-8")) + documents: list[CollectedDocument] = [] + for entry in payload.get("patents", []): + text = normalize_whitespace(f"{entry['title']}\n\n{entry.get('abstract', '')}") + documents.append( + CollectedDocument( + url=entry.get("url", fixture_path.as_uri()), + canonical_url=entry.get("url", fixture_path.as_uri()), + title=entry["title"], + author=entry.get("assignee"), + publication_date=( + datetime.fromisoformat(entry["filed_date"]).replace(tzinfo=UTC) + if entry.get("filed_date") + else None + ), + retrieved_date=datetime.now(UTC), + content_text=text, + content_hash=compute_content_hash(text), + metadata={ + "is_fixture": True, + "data_source": "fixture", + "fixture_key": fixture_key, + }, + extraction_method="fixture", + trust_score=0.5, + ) + ) + return CollectionResult(status=SourceStatus.ACTIVE, documents=documents, pages_attempted=1) + + async def _collect_live( + self, company_name: str, leadership_names: list[str], api_key: str + ) -> CollectionResult: + if not leadership_names: + # No names to search USPTO's inventor index with - see the + # module docstring for why company-name search doesn't work on + # this endpoint at all. Honest empty result, no network call. + return CollectionResult(status=SourceStatus.ACTIVE, documents=[], pages_attempted=0) + + documents: list[CollectedDocument] = [] + seen_app_numbers: set[str] = set() + pages_attempted = 0 + errors: list[str] = [] + + async with httpx.AsyncClient(timeout=20) as client: + for inventor_name in leadership_names[:_MAX_INVENTOR_SEARCHES]: + pages_attempted += 1 + body = { + "q": f'applicationMetaData.inventorBag.inventorNameText:"{inventor_name}"', + "pagination": {"limit": _MAX_DOCUMENTS}, + "sort": [{"field": "applicationMetaData.filingDate", "order": "desc"}], + } + try: + response = await client.post( + _SEARCH_URL, json=body, headers={"x-api-key": api_key} + ) + except httpx.HTTPError as exc: + errors.append(f"{inventor_name}: {exc}") + continue + + if response.status_code == 404: + # USPTO returns 404 for "no matching records" rather + # than 200 with an empty array - a real, expected + # outcome for most names, not a failure. + continue + if response.status_code >= 400: + errors.append(f"{inventor_name}: USPTO API error {response.status_code}") + continue + + try: + payload = response.json() + except ValueError: + errors.append(f"{inventor_name}: malformed USPTO response") + continue + + entries = payload.get("patentFileWrapperDataBag") or payload.get("results") or [] + for entry in entries: + metadata = entry.get("applicationMetaData") or {} + app_number = entry.get("applicationNumberText") or entry.get( + "applicationNumber" + ) + if not app_number or app_number in seen_app_numbers: + continue + seen_app_numbers.add(app_number) + + title = metadata.get("inventionTitle") or "Untitled patent filing" + filing_date = metadata.get("filingDate") + abstract = metadata.get("abstractText") or "" + text = normalize_whitespace( + f"{title}\n\nInventor match: {inventor_name} (leadership-name " + f"heuristic, not a verified {company_name} assignee - USPTO's " + "application search has no queryable assignee/company field).\n\n" + f"{abstract}" + ) + documents.append( + CollectedDocument( + url=( + f"{_SEARCH_URL}?applicationNumber={app_number}" + if app_number + else _SEARCH_URL + ), + canonical_url=_SEARCH_URL, + title=title, + author=inventor_name, + publication_date=( + datetime.fromisoformat(filing_date).replace(tzinfo=UTC) + if filing_date + else None + ), + retrieved_date=datetime.now(UTC), + content_text=text, + content_hash=compute_content_hash(text), + metadata={ + "application_number": app_number, + "data_source": "uspto_odp", + "matched_inventor_name": inventor_name, + "match_type": "leadership_name_heuristic", + }, + extraction_method="uspto_odp_api", + http_status=response.status_code, + # Lower than a verified-assignee match would be + # (was 0.9) - this is a heuristic name match, + # not confirmed company ownership. + trust_score=0.5, + ) + ) + if len(documents) >= _MAX_DOCUMENTS: + break + + if errors and not documents: + return CollectionResult( + status=SourceStatus.FAILED, + error="; ".join(errors[:3]), + pages_attempted=pages_attempted, + ) + return CollectionResult( + status=SourceStatus.ACTIVE, documents=documents, pages_attempted=pages_attempted + ) diff --git a/apps/api/app/collectors/registry.py b/apps/api/app/collectors/registry.py new file mode 100644 index 0000000..014f01a --- /dev/null +++ b/apps/api/app/collectors/registry.py @@ -0,0 +1,36 @@ +"""Maps SourceType -> collector instance. The single place Phase 5's Celery +task (and this phase's tests) resolve a collector from a Source row.""" + +from __future__ import annotations + +from app.collectors.base import SourceCollector +from app.collectors.custom_url import CustomUrlCollector +from app.collectors.github import GithubCollector +from app.collectors.gov_contracts import GovContractCollector +from app.collectors.jobs import JobPostingCollector +from app.collectors.patents import PatentSourceCollector +from app.collectors.reviews import ReviewSourceCollector +from app.collectors.rss import RssCollector +from app.collectors.sec_edgar import SecEdgarCollector +from app.collectors.website import WebsiteCollector +from app.models.enums import SourceType + +_COLLECTORS: dict[SourceType, SourceCollector] = { + SourceType.WEBSITE: WebsiteCollector(), + SourceType.RSS: RssCollector(), + SourceType.CUSTOM_URL: CustomUrlCollector(), + SourceType.SEC_EDGAR: SecEdgarCollector(), + SourceType.GITHUB: GithubCollector(), + SourceType.JOB_POSTING: JobPostingCollector(), + SourceType.PATENT: PatentSourceCollector(), + SourceType.REVIEW: ReviewSourceCollector(), + SourceType.GOV_CONTRACT: GovContractCollector(), +} + + +def get_collector(source_type: SourceType) -> SourceCollector: + return _COLLECTORS[source_type] + + +def all_collectors() -> list[SourceCollector]: + return list(_COLLECTORS.values()) diff --git a/apps/api/app/collectors/reviews.py b/apps/api/app/collectors/reviews.py new file mode 100644 index 0000000..9227851 --- /dev/null +++ b/apps/api/app/collectors/reviews.py @@ -0,0 +1,85 @@ +"""Customer review source collector. + +Most review platforms (G2, Trustpilot, Glassdoor, etc.) either prohibit +automated scraping in their terms or require a paid API. This collector +implements the `SourceCollector` interface and a documented fixture adapter +for local development/testing; it never scrapes a review site directly and +never fabricates review data when no permitted live provider is configured. +""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from pathlib import Path + +from app.collectors.base import ( + CollectedDocument, + CollectionResult, + CompanyContext, + DiscoveredSource, + SourceConfig, +) +from app.collectors.extraction import compute_content_hash, normalize_whitespace +from app.models.enums import SourceStatus, SourceType + +_FIXTURES_DIR = Path(__file__).resolve().parents[2] / "tests" / "fixtures" / "reviews" + + +class ReviewSourceCollector: + source_type = SourceType.REVIEW + + async def discover(self, company: CompanyContext) -> list[DiscoveredSource]: + return [] + + async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult: + fixture_key = source.configuration_metadata.get("fixture_key") + if not fixture_key: + return CollectionResult( + status=SourceStatus.DISABLED, + error=( + "No review data provider is configured. Most review platforms prohibit " + "automated scraping in their terms; this collector implements the " + "SourceCollector interface for a future permitted API integration but " + "does not fabricate results without one." + ), + ) + + fixture_path = _FIXTURES_DIR / f"{fixture_key}.json" + if not fixture_path.exists(): + return CollectionResult( + status=SourceStatus.DISABLED, + error=f"No fixture found for {fixture_key!r} and no live provider is configured.", + ) + + payload = json.loads(fixture_path.read_text(encoding="utf-8")) + documents: list[CollectedDocument] = [] + for entry in payload.get("reviews", []): + text = normalize_whitespace( + f"Rating: {entry.get('rating', 'n/a')}/5\n\n{entry.get('body', '')}" + ) + documents.append( + CollectedDocument( + url=entry.get("url", fixture_path.as_uri()), + canonical_url=entry.get("url", fixture_path.as_uri()), + title=entry.get("title") or "Customer review", + author=entry.get("author"), + publication_date=( + datetime.fromisoformat(entry["date"]).replace(tzinfo=UTC) + if entry.get("date") + else None + ), + retrieved_date=datetime.now(UTC), + content_text=text, + content_hash=compute_content_hash(text), + metadata={ + "is_fixture": True, + "data_source": "fixture", + "fixture_key": fixture_key, + "rating": entry.get("rating"), + }, + extraction_method="fixture", + trust_score=0.4, + ) + ) + return CollectionResult(status=SourceStatus.ACTIVE, documents=documents, pages_attempted=1) diff --git a/apps/api/app/collectors/robots.py b/apps/api/app/collectors/robots.py new file mode 100644 index 0000000..0661f7c --- /dev/null +++ b/apps/api/app/collectors/robots.py @@ -0,0 +1,54 @@ +"""robots.txt compliance check - see SECURITY.md rule 1.""" + +from __future__ import annotations + +import time +from urllib.parse import urljoin, urlparse +from urllib.robotparser import RobotFileParser + +from app.core.config import Settings, get_settings +from app.core.http import SsrfBlockedError, safe_fetch +from app.core.logging import get_logger + +logger = get_logger(__name__) + +_CACHE_TTL_SECONDS = 3600 +_cache: dict[str, tuple[float, RobotFileParser]] = {} + + +async def _get_parser(base_url: str, settings: Settings) -> RobotFileParser: + parsed = urlparse(base_url) + origin = f"{parsed.scheme}://{parsed.netloc}" + cached = _cache.get(origin) + now = time.monotonic() + if cached and now - cached[0] < _CACHE_TTL_SECONDS: + return cached[1] + + parser = RobotFileParser() + robots_url = urljoin(origin, "/robots.txt") + try: + result = await safe_fetch(robots_url, settings=settings) + if result.status_code == 200: + parser.parse(result.text.splitlines()) + else: + # No robots.txt or inaccessible -> "allow all" per convention. + parser.parse([]) + except SsrfBlockedError: + parser.parse([]) + except Exception as exc: # pragma: no cover - defensive + logger.warning("robots_txt_fetch_failed", url=robots_url, error=str(exc)) + parser.parse([]) + + _cache[origin] = (now, parser) + return parser + + +async def is_allowed(url: str, *, settings: Settings | None = None) -> bool: + settings = settings or get_settings() + parser = await _get_parser(url, settings) + return parser.can_fetch(settings.scraper_user_agent, url) + + +def clear_cache() -> None: + """Test helper - the module-level cache would otherwise leak between tests.""" + _cache.clear() diff --git a/apps/api/app/collectors/rss.py b/apps/api/app/collectors/rss.py new file mode 100644 index 0000000..57e5f18 --- /dev/null +++ b/apps/api/app/collectors/rss.py @@ -0,0 +1,111 @@ +"""RSS/Atom feed collector.""" + +from __future__ import annotations + +import time as time_module +from datetime import UTC, datetime +from urllib.parse import quote + +import feedparser +import httpx + +from app.collectors.base import ( + CollectedDocument, + CollectionResult, + CompanyContext, + DiscoveredSource, + SourceConfig, +) +from app.collectors.extraction import canonicalize_url, compute_content_hash, normalize_whitespace +from app.core.config import get_settings +from app.core.http import FetchError, SsrfBlockedError, fetch_with_retries +from app.core.logging import get_logger +from app.models.enums import SourceStatus, SourceType + +logger = get_logger(__name__) + + +class RssCollector: + source_type = SourceType.RSS + + async def discover(self, company: CompanyContext) -> list[DiscoveredSource]: + # Google News' search RSS endpoint needs no API key and reliably + # exists for any query - unlike a company's own press-room feed + # (which would need a search provider to locate), this one URL + # formula works for every company and already aggregates wire- + # service releases (PRNewswire/BusinessWire/GlobeNewswire) as + # they're published, so a dedicated wire-specific collector isn't + # needed on top of it. Users can still add any other feed manually + # (see custom_url.py's sibling "add any public URL" path). + query_url = ( + f"https://news.google.com/rss/search?q={quote(company.name)}&hl=en-US&gl=US&ceid=US:en" + ) + return [ + DiscoveredSource( + source_type=SourceType.RSS, + name=f"{company.name} — Google News", + base_url=query_url, + ) + ] + + async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult: + if not source.base_url: + return CollectionResult(status=SourceStatus.FAILED, error="No feed URL configured") + + settings = get_settings() + try: + result = await fetch_with_retries(source.base_url, settings=settings) + except SsrfBlockedError as exc: + return CollectionResult(status=SourceStatus.BLOCKED_BY_POLICY, error=str(exc)) + except (FetchError, httpx.HTTPError) as exc: + return CollectionResult(status=SourceStatus.FAILED, error=str(exc)) + + if result.status_code >= 400: + return CollectionResult( + status=SourceStatus.FAILED, error=f"HTTP {result.status_code}", pages_attempted=1 + ) + + parsed = feedparser.parse(result.content) + if parsed.bozo and not parsed.entries: + return CollectionResult( + status=SourceStatus.FAILED, + error=str(parsed.get("bozo_exception", "Unparseable feed")), + pages_attempted=1, + ) + + max_items = source.configuration_metadata.get("max_items", 20) + documents: list[CollectedDocument] = [] + for entry in parsed.entries[:max_items]: + link = entry.get("link") + if not link: + continue + summary = entry.get("summary", "") or entry.get("description", "") + text = normalize_whitespace(f"{entry.get('title', '')}\n\n{summary}") + if not text: + continue + + pub_date = None + if entry.get("published_parsed"): + pub_date = datetime.fromtimestamp( + time_module.mktime(entry.published_parsed), tz=UTC + ) + + documents.append( + CollectedDocument( + url=link, + canonical_url=canonicalize_url(link), + title=entry.get("title"), + author=entry.get("author"), + publication_date=pub_date, + retrieved_date=datetime.now(UTC), + content_text=text, + content_hash=compute_content_hash(text), + metadata={"feed_url": source.base_url}, + extraction_method="feedparser", + http_status=result.status_code, + trust_score=0.6, + ) + ) + + status = SourceStatus.ACTIVE if documents else SourceStatus.FAILED + return CollectionResult(status=status, documents=documents, pages_attempted=1) diff --git a/apps/api/app/collectors/sec_edgar.py b/apps/api/app/collectors/sec_edgar.py new file mode 100644 index 0000000..fde7072 --- /dev/null +++ b/apps/api/app/collectors/sec_edgar.py @@ -0,0 +1,167 @@ +"""SEC EDGAR collector for US public companies. + +No API key required, but SEC asks that callers identify themselves with a +descriptive User-Agent (see `SCRAPER_USER_AGENT` in .env.example) and stay +within its rate limits - the shared `safe_fetch` per-domain delay covers +that. We store filing *metadata* (form type, date, accession number, link) +rather than parsing full filing bodies, which is out of scope for this pass. +""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from xml.etree import ElementTree + +import httpx + +from app.collectors.base import ( + CollectedDocument, + CollectionResult, + CompanyContext, + DiscoveredSource, + SourceConfig, +) +from app.collectors.extraction import compute_content_hash, normalize_whitespace +from app.core.config import get_settings +from app.core.http import FetchError, SsrfBlockedError, fetch_with_retries +from app.core.logging import get_logger +from app.models.enums import SourceStatus, SourceType + +logger = get_logger(__name__) + +_RELEVANT_FORMS = {"10-K", "10-Q", "8-K"} +_SEARCH_URL = "https://www.sec.gov/cgi-bin/browse-edgar" +_SUBMISSIONS_URL = "https://data.sec.gov/submissions/CIK{cik}.json" + + +class SecEdgarCollector: + source_type = SourceType.SEC_EDGAR + + async def discover(self, company: CompanyContext) -> list[DiscoveredSource]: + settings = get_settings() + cik = await self._lookup_cik(company.name, settings) + if cik is None: + return [] + return [ + DiscoveredSource( + source_type=SourceType.SEC_EDGAR, + name=f"{company.name} — SEC EDGAR filings", + base_url=f"https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={cik}", + configuration_metadata={"cik": cik}, + ) + ] + + async def _lookup_cik(self, company_name: str, settings) -> str | None: + params = { + "action": "getcompany", + "company": company_name, + "type": "10-K", + "dateb": "", + "owner": "include", + "count": "5", + "output": "atom", + } + url = str(httpx.URL(_SEARCH_URL, params=params)) + try: + result = await fetch_with_retries(url, settings=settings, max_attempts=2) + except (SsrfBlockedError, FetchError, httpx.HTTPError) as exc: + logger.warning("sec_edgar_lookup_failed", company=company_name, error=str(exc)) + return None + if result.status_code != 200: + return None + try: + root = ElementTree.fromstring(result.content) + except ElementTree.ParseError: + return None + + ns = {"a": "http://www.w3.org/2005/Atom"} + for entry in root.findall(".//a:entry", ns): + cik_elem = entry.find("a:content", ns) + title_elem = entry.find("a:title", ns) + if cik_elem is None or title_elem is None: + continue + # The atom feed embeds "CIK=0000320193" style text in . + text = "".join(cik_elem.itertext()) + if "CIK=" in text: + cik = text.split("CIK=")[1].split("&")[0].strip() + return cik.zfill(10) + return None + + async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult: + cik = source.configuration_metadata.get("cik") + if not cik: + return CollectionResult(status=SourceStatus.FAILED, error="No CIK configured") + + settings = get_settings() + url = _SUBMISSIONS_URL.format(cik=str(cik).zfill(10)) + try: + result = await fetch_with_retries(url, settings=settings) + except SsrfBlockedError as exc: + return CollectionResult(status=SourceStatus.BLOCKED_BY_POLICY, error=str(exc)) + except (FetchError, httpx.HTTPError) as exc: + return CollectionResult(status=SourceStatus.FAILED, error=str(exc)) + + if result.status_code == 404: + return CollectionResult(status=SourceStatus.FAILED, error="CIK not found on EDGAR") + if result.status_code >= 400: + return CollectionResult(status=SourceStatus.FAILED, error=f"HTTP {result.status_code}") + + try: + payload = json.loads(result.text) + except json.JSONDecodeError: + return CollectionResult(status=SourceStatus.FAILED, error="Malformed EDGAR response") + + recent = payload.get("filings", {}).get("recent", {}) + forms = recent.get("form", []) + dates = recent.get("filingDate", []) + accessions = recent.get("accessionNumber", []) + primary_docs = recent.get("primaryDocument", []) + company_name = payload.get("name", company.name) + + documents: list[CollectedDocument] = [] + for i, form in enumerate(forms): + if form not in _RELEVANT_FORMS: + continue + if len(documents) >= 10: + break + accession = accessions[i].replace("-", "") if i < len(accessions) else "" + primary_doc = primary_docs[i] if i < len(primary_docs) else "" + filing_date = dates[i] if i < len(dates) else "" + filing_url = ( + f"https://www.sec.gov/Archives/edgar/data/{int(cik)}/{accession}/{primary_doc}" + if accession and primary_doc + else url + ) + text = normalize_whitespace( + f"{company_name} filed a {form} with the SEC on {filing_date}.\n" + f"Accession number: {accessions[i] if i < len(accessions) else 'unknown'}.\n" + f"Filing document: {filing_url}" + ) + documents.append( + CollectedDocument( + url=filing_url, + canonical_url=filing_url, + title=f"{company_name} {form} ({filing_date})", + author="SEC EDGAR", + publication_date=( + datetime.strptime(filing_date, "%Y-%m-%d").replace(tzinfo=UTC) + if filing_date + else None + ), + retrieved_date=datetime.now(UTC), + content_text=text, + content_hash=compute_content_hash(text), + metadata={ + "form": form, + "accession_number": accessions[i] if i < len(accessions) else None, + }, + extraction_method="sec_edgar_metadata", + http_status=result.status_code, + trust_score=0.95, + ) + ) + + # Zero relevant filings isn't a failure - the company may simply have + # none in its recent filing history. + return CollectionResult(status=SourceStatus.ACTIVE, documents=documents, pages_attempted=1) diff --git a/apps/api/app/collectors/website.py b/apps/api/app/collectors/website.py new file mode 100644 index 0000000..97cd098 --- /dev/null +++ b/apps/api/app/collectors/website.py @@ -0,0 +1,176 @@ +"""Official website collector: sitemap.xml + heuristic page discovery, +robots.txt-respecting, capped crawl depth/page count. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from urllib.parse import urljoin +from xml.etree import ElementTree + +import httpx + +from app.collectors.base import ( + CollectedDocument, + CollectionResult, + CompanyContext, + DiscoveredSource, + SourceConfig, +) +from app.collectors.extraction import ( + canonicalize_url, + compute_content_hash, + extract_readable_text, + extract_title, +) +from app.collectors.robots import is_allowed +from app.core.config import Settings, get_settings +from app.core.http import FetchError, SsrfBlockedError, fetch_with_retries +from app.core.logging import get_logger +from app.models.enums import SourceStatus, SourceType + +logger = get_logger(__name__) + +HEURISTIC_PATHS = [ + "", + "/about", + "/about-us", + "/products", + "/services", + "/news", + "/press", + "/press-releases", + "/careers", + "/jobs", + "/leadership", + "/team", + "/investors", + "/investor-relations", + "/sustainability", + "/contact", +] + + +class WebsiteCollector: + source_type = SourceType.WEBSITE + + async def discover(self, company: CompanyContext) -> list[DiscoveredSource]: + if not company.official_website: + return [] + settings = get_settings() + pages = await self._discover_pages(company.official_website, settings) + return [ + DiscoveredSource( + source_type=SourceType.WEBSITE, + name=f"{company.name} — Official Website", + base_url=company.official_website, + configuration_metadata={"pages": pages}, + ) + ] + + async def _discover_pages(self, base_url: str, settings: Settings) -> list[str]: + pages: list[str] = [] + + sitemap_urls = await self._read_sitemap(base_url, settings) + pages.extend(sitemap_urls[: settings.max_pages_per_domain]) + + if len(pages) < settings.max_pages_per_domain: + for path in HEURISTIC_PATHS: + candidate = urljoin(base_url, path) + if candidate not in pages: + pages.append(candidate) + if len(pages) >= settings.max_pages_per_domain: + break + + return pages[: settings.max_pages_per_domain] + + async def _read_sitemap(self, base_url: str, settings: Settings) -> list[str]: + sitemap_url = urljoin(base_url, "/sitemap.xml") + try: + result = await fetch_with_retries(sitemap_url, settings=settings, max_attempts=1) + except (SsrfBlockedError, FetchError, httpx.HTTPError): + return [] + if result.status_code != 200: + return [] + try: + root = ElementTree.fromstring(result.content) + except ElementTree.ParseError: + return [] + + ns = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"} + urls = [loc.text.strip() for loc in root.findall(".//sm:url/sm:loc", ns) if loc.text] + return urls + + async def collect(self, source: SourceConfig, company: CompanyContext) -> CollectionResult: + settings = get_settings() + pages: list[str] = source.configuration_metadata.get("pages") or ( + [source.base_url] if source.base_url else [] + ) + pages = pages[: settings.max_pages_per_domain] + + documents: list[CollectedDocument] = [] + seen_hashes: set[str] = set() + attempted = 0 + any_success = False + last_error: str | None = None + + for page_url in pages: + attempted += 1 + try: + if not await is_allowed(page_url, settings=settings): + logger.info("website_collector_robots_disallowed", url=page_url) + continue + + result = await fetch_with_retries(page_url, settings=settings) + if result.status_code == 401 or result.status_code == 403: + last_error = f"HTTP {result.status_code} (auth required) for {page_url}" + continue + if result.status_code >= 400: + last_error = f"HTTP {result.status_code} for {page_url}" + continue + + text, method = extract_readable_text(result.text, page_url) + if not text: + continue + content_hash = compute_content_hash(text) + if content_hash in seen_hashes: + continue + seen_hashes.add(content_hash) + + documents.append( + CollectedDocument( + url=page_url, + canonical_url=canonicalize_url(result.final_url), + title=extract_title(result.text), + author=None, + publication_date=None, + retrieved_date=datetime.now(UTC), + content_text=text, + content_hash=content_hash, + metadata={"http_status": result.status_code}, + extraction_method=method, + http_status=result.status_code, + trust_score=0.85, + ) + ) + any_success = True + except SsrfBlockedError as exc: + last_error = str(exc) + logger.warning("website_collector_ssrf_blocked", url=page_url, error=str(exc)) + except (FetchError, httpx.HTTPError) as exc: + last_error = str(exc) + logger.warning("website_collector_fetch_failed", url=page_url, error=str(exc)) + + if not documents: + status = SourceStatus.FAILED if attempted > 0 else SourceStatus.ACTIVE + return CollectionResult( + status=status, documents=[], error=last_error, pages_attempted=attempted + ) + + status = SourceStatus.ACTIVE if any_success else SourceStatus.FAILED + return CollectionResult( + status=status, + documents=documents, + error=last_error if not any_success else None, + pages_attempted=attempted, + ) diff --git a/apps/api/app/core/__init__.py b/apps/api/app/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/app/core/config.py b/apps/api/app/core/config.py new file mode 100644 index 0000000..6f78b77 --- /dev/null +++ b/apps/api/app/core/config.py @@ -0,0 +1,178 @@ +"""Centralized application configuration. + +Every environment-dependent value is read here, once, via pydantic-settings. +Application code should depend on `get_settings()`, never on `os.environ` +directly - that's what keeps provider selection (LLM/search/notifications/auth) +swappable from a single place. +""" + +from __future__ import annotations + +from functools import lru_cache +from typing import Literal + +from pydantic import model_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + model_config = SettingsConfigDict(env_file=".env", extra="ignore") + + # --- App --- + app_env: Literal["development", "test", "production"] = "development" + app_name: str = "CI Agent" + frontend_url: str = "http://localhost:3000" + backend_url: str = "http://localhost:8000" + + # --- Reverse proxy --- + # Empty (default) = trust only the direct TCP connection for client-IP + # resolution (app.core.security.get_client_ip) - correct today, no proxy + # exists. Set to "CF-Connecting-IP" once deployed behind Cloudflare's + # proxy so IP-based localhost detection and the ban/throttle system read + # the real visitor IP instead of the proxy's own address. Only ever set + # this when it's actually known a trusted proxy sits in front and strips + # this header from untrusted clients - see KNOWN_LIMITATIONS.md. + trusted_proxy_ip_header: str = "" + + # --- Local-dev convenience --- + # Comma-separated extra IPs that `is_localhost` treats as equivalent to + # real loopback, on top of 127.0.0.1/::1. Needed because Docker + # Desktop's bridge networking means even traffic that originates on the + # host machine itself (dev tooling driving a browser against + # localhost:3000/8000) arrives at the container from the bridge + # gateway address, not literal loopback - so without this, + # is_localhost is always false for that traffic, which both re-exposes + # the admin-only API-keys/logs boxes' loopback gate and forces the + # Turnstile widget to render (a real crash risk for automated browser + # tooling - see KNOWN_LIMITATIONS.md). Empty by default (no bypass); + # only ever populate this with IPs you know are your own dev host, + # never in a real deployment. + additional_trusted_local_ips: str = "" + + # --- Auth --- + auth_mode: Literal["local", "jwt"] = "local" + jwt_secret: str = "dev-only-change-me-32-characters-minimum" + jwt_access_token_minutes: int = 15 + jwt_refresh_token_days: int = 7 + # Encrypts each user's own stored API keys at rest (app/core/crypto.py) + # - a Fernet key (44-char urlsafe-base64). This dev-only default is + # fixed/insecure by design (same precedent as jwt_secret above); a real + # deployment must set its own via `Fernet.generate_key()`. Rotating + # this value makes every already-stored user key undecryptable, so + # treat it like any other production secret - never regenerate it + # casually once real keys exist. + api_key_encryption_secret: str = "_wYtsm3nJ070987snBFp2eWVI5pyC0H9gGFUb6Cy4cQ=" + + # --- Database --- + database_url: str = "sqlite+aiosqlite:///./ciagent_dev.db" + + # --- Redis / Celery --- + redis_url: str = "redis://localhost:6379/0" + celery_task_always_eager: bool = False + + # --- LLM --- + llm_provider: Literal["mock", "anthropic", "ollama", "gemini"] = "mock" + anthropic_api_key: str = "" + anthropic_model: str = "claude-sonnet-5" + ollama_base_url: str = "http://localhost:11434" + ollama_model: str = "llama3.1" + # Gemini has an actual free rate-limited tier (unlike OpenAI's expiring + # trial credits), so it's the production option this app ships wired up. + gemini_api_key: str = "" + gemini_model: str = "gemini-2.0-flash" + llm_max_tokens_per_request: int = 4000 + llm_max_retries: int = 2 + + # --- Search --- + search_provider: Literal["mock", "brave"] = "mock" + brave_search_api_key: str = "" + serpapi_api_key: str = "" + bing_search_api_key: str = "" + + # --- Patents --- + # Free key via account registration at data.uspto.gov/apis/getting-started. + # Unset by default - PatentSourceCollector falls back to its existing + # honest disabled/fixture behavior when this is empty. + uspto_api_key: str = "" + + # --- Company enrichment (NinjaPear / nubela.co) --- + # Paid, per-credit API - unset by default. Enrichment only ever fires + # once, at company-creation time (never on a recurring schedule), and + # the enqueue itself is skipped entirely when this is empty - see + # company_service.create_company. + ninjapear_api_key: str = "" + ninjapear_max_leadership_lookups: int = 5 + + # --- Email --- + smtp_host: str = "localhost" + smtp_port: int = 1025 + smtp_username: str = "" + smtp_password: str = "" + smtp_from_email: str = "alerts@ci-agent.local" + smtp_use_tls: bool = False + + # --- Resend (transactional security email: verification/reset/lockout) --- + # Unset by default - security_email_service falls back to the SMTP + # provider above (Mailpit locally) when this is empty, so the whole + # verification/reset flow is testable with zero Resend account needed. + # Deliberately separate from the alert-notification path (smtp_from_email + # above) - a different sender identity for account-security mail. + resend_api_key: str = "" + resend_security_from_email: str = "security@ciagent.org" + + # --- Cloudflare Turnstile (CAPTCHA on register/login/password-reset) --- + # Unset by default - skipped entirely for register/login/password-reset + # when either the caller is on loopback (see is_localhost) or no secret + # is configured, matching this app's usual optional-provider convention. + turnstile_site_key: str = "" + turnstile_secret: str = "" + + # --- SMS --- + notification_sms_enabled: bool = False + sms_provider: Literal["twilio", "telnyx"] = "twilio" + twilio_account_sid: str = "" + twilio_auth_token: str = "" + twilio_from_number: str = "" + telnyx_api_key: str = "" + telnyx_from_number: str = "" + sms_monthly_cap: int = 50 + + # --- GitHub --- + github_token: str = "" + + # --- Scheduling --- + default_timezone: str = "America/New_York" + default_monitoring_frequency: str = "weekly" + minimum_monitoring_interval_minutes: int = 60 + + # --- Scraper --- + scraper_user_agent: str = "CIAgentBot/1.0 (+https://ci-agent.local/bot)" + max_pages_per_domain: int = 25 + scraper_request_timeout_seconds: int = 30 + scraper_domain_delay_seconds: float = 2.0 + + # --- Cost controls --- + max_companies_per_user: int = 25 + max_manual_runs_per_day: int = 10 + + # --- Retention / logging --- + data_retention_days: int = 365 + log_level: str = "INFO" + + @property + def is_production(self) -> bool: + return self.app_env == "production" + + @model_validator(mode="after") + def _forbid_local_auth_in_production(self) -> Settings: + if self.app_env == "production" and self.auth_mode == "local": + raise ValueError( + "AUTH_MODE=local is a development convenience and must not be used " + "when APP_ENV=production. Set AUTH_MODE=jwt." + ) + return self + + +@lru_cache +def get_settings() -> Settings: + return Settings() diff --git a/apps/api/app/core/crypto.py b/apps/api/app/core/crypto.py new file mode 100644 index 0000000..41e0097 --- /dev/null +++ b/apps/api/app/core/crypto.py @@ -0,0 +1,25 @@ +"""Symmetric encryption for secrets stored at rest - currently just +per-user API keys (app/models/user_api_key.py). Not used for passwords +(those are one-way hashed via app.core.security, never decrypted) - this +is specifically for secrets the app must later read back out in plaintext +to actually call a third-party API on the user's behalf. +""" + +from __future__ import annotations + +from cryptography.fernet import Fernet, InvalidToken + +from app.core.config import Settings + + +def encrypt_secret(plaintext: str, settings: Settings) -> str: + return Fernet(settings.api_key_encryption_secret).encrypt(plaintext.encode()).decode() + + +def decrypt_secret(ciphertext: str, settings: Settings) -> str: + try: + return Fernet(settings.api_key_encryption_secret).decrypt(ciphertext.encode()).decode() + except InvalidToken as exc: + # Only real cause in practice: api_key_encryption_secret was + # rotated after this value was encrypted under the old one. + raise ValueError("Stored value cannot be decrypted with the current key") from exc diff --git a/apps/api/app/core/errors.py b/apps/api/app/core/errors.py new file mode 100644 index 0000000..b658cfa --- /dev/null +++ b/apps/api/app/core/errors.py @@ -0,0 +1,46 @@ +"""App-level exceptions, mapped to HTTP responses in one place (main.py). + +Services raise these instead of `fastapi.HTTPException` so business logic +stays importable/testable from Celery tasks, which don't have an HTTP +response to raise into. +""" + +from __future__ import annotations + + +class AppError(Exception): + """Base class for all app-level errors.""" + + +class NotFoundError(AppError): + pass + + +class ConflictError(AppError): + pass + + +class AuthenticationError(AppError): + pass + + +class ForbiddenError(AppError): + pass + + +class ValidationAppError(AppError): + pass + + +class RateLimitedError(AppError): + pass + + +class ThrottledError(RateLimitedError): + """Raised by the IP throttle/ban engine (app/services/ip_throttle_service.py) + - carries a machine-readable retry_after_seconds so the frontend can + drive a live countdown instead of just showing a generic message.""" + + def __init__(self, message: str, retry_after_seconds: int | None = None) -> None: + super().__init__(message) + self.retry_after_seconds = retry_after_seconds diff --git a/apps/api/app/core/http.py b/apps/api/app/core/http.py new file mode 100644 index 0000000..23ee726 --- /dev/null +++ b/apps/api/app/core/http.py @@ -0,0 +1,176 @@ +"""SSRF-safe HTTP fetching. Every collector and the custom-URL feature must +route network requests through `safe_fetch` / `fetch_with_retries` - see +SECURITY.md for the full threat model this defends against. +""" + +from __future__ import annotations + +import asyncio +import ipaddress +import socket +import time +from dataclasses import dataclass +from urllib.parse import urljoin, urlparse + +import httpx +import tenacity + +from app.core.config import Settings, get_settings +from app.core.logging import get_logger + +logger = get_logger(__name__) + +_ALLOWED_SCHEMES = {"http", "https"} +_MAX_REDIRECTS = 5 +_METADATA_IPS = {"169.254.169.254", "fd00:ec2::254"} + +# Per-hostname request spacing. In-process only - a multi-worker Celery +# deployment would need a shared store (e.g. Redis) for this to be a true +# global rate limit across workers; see KNOWN_LIMITATIONS.md. +_last_request_at: dict[str, float] = {} +_domain_locks: dict[str, asyncio.Lock] = {} + + +class SsrfBlockedError(Exception): + """Raised when a URL resolves to, or points at, a disallowed network target.""" + + +class FetchError(Exception): + """Raised for network-level failures after retries are exhausted.""" + + +def _is_blocked_ip(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool: + return ( + ip.is_private + or ip.is_loopback + or ip.is_link_local + or ip.is_multicast + or ip.is_reserved + or ip.is_unspecified + or str(ip) in _METADATA_IPS + ) + + +def _resolve_and_validate(hostname: str) -> None: + try: + infos = socket.getaddrinfo(hostname, None) + except socket.gaierror as exc: + raise SsrfBlockedError(f"Could not resolve host: {hostname}") from exc + + if not infos: + raise SsrfBlockedError(f"Could not resolve host: {hostname}") + + for info in infos: + raw_ip = info[4][0] + ip = ipaddress.ip_address(raw_ip.split("%")[0]) + if _is_blocked_ip(ip): + raise SsrfBlockedError(f"Resolved address for {hostname} is not a public address: {ip}") + + +def validate_url(url: str) -> str: + """Raises SsrfBlockedError if `url` is unsafe to fetch. Returns the + hostname.""" + parsed = urlparse(url) + if parsed.scheme not in _ALLOWED_SCHEMES: + raise SsrfBlockedError(f"Unsupported URL scheme: {parsed.scheme!r}") + if not parsed.hostname: + raise SsrfBlockedError("URL has no hostname") + _resolve_and_validate(parsed.hostname) + return parsed.hostname + + +async def _respect_domain_delay(hostname: str, delay_seconds: float) -> None: + if delay_seconds <= 0: + return + lock = _domain_locks.setdefault(hostname, asyncio.Lock()) + async with lock: + now = time.monotonic() + last = _last_request_at.get(hostname) + if last is not None: + elapsed = now - last + if elapsed < delay_seconds: + await asyncio.sleep(delay_seconds - elapsed) + _last_request_at[hostname] = time.monotonic() + + +@dataclass +class SafeFetchResult: + status_code: int + text: str + content: bytes + headers: dict[str, str] + final_url: str + + +async def safe_fetch( + url: str, + *, + settings: Settings | None = None, + method: str = "GET", + max_redirects: int = _MAX_REDIRECTS, + extra_headers: dict[str, str] | None = None, +) -> SafeFetchResult: + """Fetch `url` with SSRF validation applied to the initial URL and every + redirect hop. Never follows a redirect without re-validating it.""" + settings = settings or get_settings() + current_url = url + + for _ in range(max_redirects + 1): + hostname = validate_url(current_url) + await _respect_domain_delay(hostname, settings.scraper_domain_delay_seconds) + + headers = {"User-Agent": settings.scraper_user_agent, **(extra_headers or {})} + async with httpx.AsyncClient( + follow_redirects=False, + timeout=settings.scraper_request_timeout_seconds, + headers=headers, + ) as client: + response = await client.request(method, current_url) + + if response.status_code in (301, 302, 303, 307, 308) and "location" in response.headers: + current_url = urljoin(current_url, response.headers["location"]) + continue + + return SafeFetchResult( + status_code=response.status_code, + text=response.text, + content=response.content, + headers=dict(response.headers), + final_url=current_url, + ) + + raise SsrfBlockedError(f"Too many redirects starting from {url}") + + +def _is_retryable(exc: BaseException) -> bool: + if isinstance(exc, SsrfBlockedError): + return False + if isinstance(exc, httpx.HTTPError): + return True + return False + + +async def fetch_with_retries( + url: str, + *, + settings: Settings | None = None, + max_attempts: int = 3, + **kwargs, +) -> SafeFetchResult: + """`safe_fetch` wrapped with exponential-backoff retry for transient + network errors only - SSRF blocks and 4xx responses are not retried.""" + settings = settings or get_settings() + + async for attempt in tenacity.AsyncRetrying( + stop=tenacity.stop_after_attempt(max_attempts), + wait=tenacity.wait_exponential(multiplier=1, min=1, max=10), + retry=tenacity.retry_if_exception(_is_retryable), + reraise=True, + ): + with attempt: + result = await safe_fetch(url, settings=settings, **kwargs) + if result.status_code >= 500: + raise FetchError(f"Server error {result.status_code} fetching {url}") + return result + + raise FetchError(f"Exhausted retries fetching {url}") # pragma: no cover diff --git a/apps/api/app/core/logging.py b/apps/api/app/core/logging.py new file mode 100644 index 0000000..5f0ea9e --- /dev/null +++ b/apps/api/app/core/logging.py @@ -0,0 +1,149 @@ +"""Structured logging setup with secret redaction, plus a capped live-log +feed for the Settings UI. + +Never log raw credentials. Any log event field whose key looks secret-shaped +gets its value replaced before it leaves the process. + +Logs also get pushed to a capped Redis list (`app:logs`) so the Settings +page can show a live feed of what the application is doing - stdout/Docker +logs alone aren't visible from the UI, and this app runs as several +separate processes (API, Celery worker, beat), so Redis (already shared +infra across all of them) is the simplest common sink. A sink failure here +must never break the actual log call or crash the app - every Redis +operation is wrapped and swallowed. +""" + +from __future__ import annotations + +import json +import logging +import re +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any + +import structlog + +if TYPE_CHECKING: + from app.core.config import Settings + +_SECRET_KEY_PATTERN = re.compile( + r"(key|secret|token|password|authorization|credential)", re.IGNORECASE +) +_REDACTED = "***REDACTED***" + +LOG_STREAM_KEY = "app:logs" +LOG_STREAM_MAXLEN = 500 + +LOG_CATEGORY_LABELS = { + "internal_error": "Internal error", + "api_error": "API error", + "important": "Important", + "normal": "Normal", +} + +_NON_CONTEXT_KEYS = {"level", "timestamp", "logger", "event", "important"} + +_sync_redis_client: Any = None + + +def _redact_secrets(_logger: Any, _method_name: str, event_dict: dict) -> dict: + for key in list(event_dict.keys()): + if _SECRET_KEY_PATTERN.search(key): + event_dict[key] = _REDACTED + return event_dict + + +def _category_for(level: str, event_dict: dict) -> str: + if level in ("error", "critical"): + return "internal_error" + if level == "warning": + return "api_error" + if level == "info" and event_dict.get("important") is True: + return "important" + return "normal" + + +def _get_sync_redis_client(redis_url: str): + global _sync_redis_client + if _sync_redis_client is None: + import redis as redis_sync + + _sync_redis_client = redis_sync.Redis.from_url(redis_url) + return _sync_redis_client + + +def _make_capture_processor(redis_url: str): + def _capture_for_ui(_logger: Any, method_name: str, event_dict: dict) -> dict: + level = event_dict.get("level", method_name) + if level == "debug": + return event_dict + try: + record = { + "ts": event_dict.get("timestamp") or datetime.now(UTC).isoformat(), + "level": level, + "category": _category_for(level, event_dict), + "logger": event_dict.get("logger", "app"), + "event": str(event_dict.get("event", "")), + "context": {k: v for k, v in event_dict.items() if k not in _NON_CONTEXT_KEYS}, + } + client = _get_sync_redis_client(redis_url) + pipe = client.pipeline() + pipe.lpush(LOG_STREAM_KEY, json.dumps(record, default=str)) + pipe.ltrim(LOG_STREAM_KEY, 0, LOG_STREAM_MAXLEN - 1) + pipe.execute() + except Exception: # noqa: BLE001 - a broken log sink must never break the app + pass + return event_dict + + return _capture_for_ui + + +def configure_logging(settings: Settings) -> None: + logging.basicConfig(level=settings.log_level, format="%(message)s") + + structlog.configure( + processors=[ + structlog.contextvars.merge_contextvars, + structlog.stdlib.add_log_level, + structlog.processors.TimeStamper(fmt="iso"), + _redact_secrets, + _make_capture_processor(settings.redis_url), + structlog.processors.StackInfoRenderer(), + structlog.processors.format_exc_info, + structlog.processors.JSONRenderer(), + ], + wrapper_class=structlog.make_filtering_bound_logger( + logging.getLevelName(settings.log_level) + ), + context_class=dict, + logger_factory=structlog.PrintLoggerFactory(), + cache_logger_on_first_use=True, + ) + + +def get_logger(name: str) -> structlog.stdlib.BoundLogger: + # `.bind(logger=name)` (rather than `structlog.stdlib.add_logger_name`, + # which only works with `structlog.stdlib.LoggerFactory`) is what makes + # the module name available to `_capture_for_ui` below - this app uses + # `PrintLoggerFactory`, whose loggers have no `.name` attribute. + return structlog.get_logger().bind(logger=name) + + +async def get_recent_logs(settings: Settings, limit: int = 100) -> list[dict]: + """Most-recent-first log entries from the capped Redis feed, for the + Settings page's Logging box.""" + import redis.asyncio as redis_asyncio + + client = redis_asyncio.from_url(settings.redis_url) + try: + raw_entries = await client.lrange(LOG_STREAM_KEY, 0, limit - 1) + finally: + await client.aclose() + + entries: list[dict] = [] + for raw in raw_entries: + try: + entries.append(json.loads(raw)) + except (json.JSONDecodeError, TypeError): + continue + return entries diff --git a/apps/api/app/core/rate_limit.py b/apps/api/app/core/rate_limit.py new file mode 100644 index 0000000..eb954f4 --- /dev/null +++ b/apps/api/app/core/rate_limit.py @@ -0,0 +1,28 @@ +"""Shared rate limiter (slowapi / limits, in-memory by default). + +Applied per-route via `@limiter.limit(...)`. Auth endpoints get the +tightest limits since they're the classic credential-stuffing target. +""" + +from __future__ import annotations + +from fastapi import Request +from slowapi import Limiter + +from app.core.config import get_settings +from app.core.security import get_client_ip + + +def _client_ip_key(request: Request) -> str: + """Same IP resolution as everything else in the app (is_localhost, the + ban/throttle engine) - slowapi's own get_remote_address reads + request.client.host directly, which would be Nginx/Cloudflare's own + address for every visitor once deployed behind a reverse proxy, + collapsing all rate limits into one shared bucket. See + app.core.security.get_client_ip / Settings.trusted_proxy_ip_header.""" + return get_client_ip(request, get_settings()) + + +# Disabled under APP_ENV=test so the many auth calls a test suite makes don't +# trip real limits (real limiter behavior is covered by its own test). +limiter = Limiter(key_func=_client_ip_key, enabled=get_settings().app_env != "test") diff --git a/apps/api/app/core/security.py b/apps/api/app/core/security.py new file mode 100644 index 0000000..2590114 --- /dev/null +++ b/apps/api/app/core/security.py @@ -0,0 +1,171 @@ +"""Password hashing and JWT helpers. + +Password hashing uses Argon2 (via `argon2-cffi`) directly - it's the +currently recommended default and needs no extra abstraction layer. +JWTs are signed with `JWT_SECRET` (HS256); access tokens are short-lived, +refresh tokens are long-lived but stored server-side only as a hash so a +leaked DB row can't be replayed as a valid token by itself. +""" + +from __future__ import annotations + +import secrets +import uuid +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from enum import StrEnum + +import jwt +from argon2 import PasswordHasher +from argon2.exceptions import VerifyMismatchError +from fastapi import Request + +from app.core.config import Settings + +_hasher = PasswordHasher() + + +def get_client_ip(request: Request, settings: Settings) -> str: + """The single source of truth for "what IP is this request from" - used + by `is_localhost`, the IP throttle/ban engine, and anywhere else that + needs to identify a caller. Today there's no reverse proxy in front of + uvicorn in this stack, so the direct TCP peer (`request.client.host`) is + the real originating address. + + Once deployed behind Cloudflare (or Nginx), the direct peer becomes the + proxy itself, not the visitor - `settings.trusted_proxy_ip_header` (e.g. + "CF-Connecting-IP") switches this to read the real address from that + header instead. Only ever set this when it's actually known the proxy is + in front and stripping/overwriting that header from untrusted clients - + otherwise a client could simply forge it to spoof any IP. Left empty by + default (trust the direct connection only) - see KNOWN_LIMITATIONS.md.""" + header_name = settings.trusted_proxy_ip_header + if header_name: + forwarded = request.headers.get(header_name) + if forwarded: + return forwarded.strip() + return request.client.host if request.client is not None else "unknown" + + +def is_localhost(request: Request, settings: Settings) -> bool: + """True when the request's resolved client IP (see `get_client_ip`) is + this machine's loopback address - not merely "someone on the LAN" - or + is explicitly listed in `settings.additional_trusted_local_ips` (empty + by default; a narrow, opt-in escape hatch for Docker Desktop's bridge + networking, where even host-originated traffic doesn't arrive as + literal loopback - see KNOWN_LIMITATIONS.md).""" + client_ip = get_client_ip(request, settings) + if client_ip in ("127.0.0.1", "::1"): + return True + extra = {ip.strip() for ip in settings.additional_trusted_local_ips.split(",") if ip.strip()} + return client_ip in extra + + +def hash_password(raw_password: str) -> str: + return _hasher.hash(raw_password) + + +def verify_password(raw_password: str, password_hash: str) -> bool: + try: + return _hasher.verify(password_hash, raw_password) + except VerifyMismatchError: + return False + + +class TokenType(StrEnum): + ACCESS = "access" + REFRESH = "refresh" + + +@dataclass(frozen=True) +class DecodedToken: + user_id: uuid.UUID + token_type: TokenType + jti: str + expires_at: datetime + + +def create_access_token(user_id: uuid.UUID, settings: Settings) -> str: + return _encode_token( + user_id, TokenType.ACCESS, timedelta(minutes=settings.jwt_access_token_minutes), settings + ) + + +def create_refresh_token(user_id: uuid.UUID, settings: Settings) -> tuple[str, str, datetime]: + """Returns (raw_jwt, jti, expires_at). Caller stores a hash of `jti`, not the JWT itself.""" + expires_at = datetime.now(UTC) + timedelta(days=settings.jwt_refresh_token_days) + jti = secrets.token_urlsafe(32) + token = _encode_token( + user_id, + TokenType.REFRESH, + timedelta(days=settings.jwt_refresh_token_days), + settings, + jti=jti, + ) + return token, jti, expires_at + + +def _encode_token( + user_id: uuid.UUID, + token_type: TokenType, + expires_in: timedelta, + settings: Settings, + jti: str | None = None, +) -> str: + now = datetime.now(UTC) + payload = { + "sub": str(user_id), + "type": token_type.value, + "iat": now, + "exp": now + expires_in, + "jti": jti or secrets.token_urlsafe(16), + } + return jwt.encode(payload, settings.jwt_secret, algorithm="HS256") + + +class InvalidTokenError(Exception): + pass + + +def decode_token(token: str, settings: Settings, expected_type: TokenType) -> DecodedToken: + try: + payload = jwt.decode(token, settings.jwt_secret, algorithms=["HS256"]) + except jwt.PyJWTError as exc: + raise InvalidTokenError(str(exc)) from exc + + if payload.get("type") != expected_type.value: + raise InvalidTokenError(f"Expected a {expected_type.value} token") + + try: + user_id = uuid.UUID(payload["sub"]) + except (KeyError, ValueError) as exc: + raise InvalidTokenError("Malformed token subject") from exc + + return DecodedToken( + user_id=user_id, + token_type=TokenType(payload["type"]), + jti=payload["jti"], + expires_at=datetime.fromtimestamp(payload["exp"], tz=UTC), + ) + + +def hash_token_identifier(jti: str) -> str: + """One-way hash of a refresh token's `jti` for storage/comparison (not the JWT itself).""" + import hashlib + + return hashlib.sha256(jti.encode("utf-8")).hexdigest() + + +def generate_email_code() -> str: + """A 6-digit numeric code for email verification / password reset - + short-lived and IP-throttled (app/services/ip_throttle_service.py), so + it doesn't need Argon2's cost the way a password does.""" + return f"{secrets.randbelow(1_000_000):06d}" + + +def hash_email_code(code: str) -> str: + """One-way hash of an email code for storage/comparison (never the raw + code) - same precedent as hash_token_identifier.""" + import hashlib + + return hashlib.sha256(code.encode("utf-8")).hexdigest() diff --git a/apps/api/app/core/text.py b/apps/api/app/core/text.py new file mode 100644 index 0000000..503763c --- /dev/null +++ b/apps/api/app/core/text.py @@ -0,0 +1,13 @@ +"""Small text utilities with no natural home elsewhere.""" + +from __future__ import annotations + +import re + +_SLUG_STRIP_RE = re.compile(r"[^a-z0-9]+") + + +def slugify(value: str) -> str: + value = value.strip().lower() + value = _SLUG_STRIP_RE.sub("-", value).strip("-") + return value or "company" diff --git a/apps/api/app/db/__init__.py b/apps/api/app/db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/app/db/base.py b/apps/api/app/db/base.py new file mode 100644 index 0000000..9e5b756 --- /dev/null +++ b/apps/api/app/db/base.py @@ -0,0 +1,38 @@ +"""Declarative base + shared mixins for all ORM models.""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime + +from sqlalchemy import DateTime +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column + + +def utcnow() -> datetime: + return datetime.now(UTC) + + +def ensure_aware_utc(value: datetime) -> datetime: + """SQLite's `DateTime(timezone=True)` silently drops tzinfo on read back + (Postgres does not). Anything read from the DB and compared against an + aware `datetime.now(UTC)` must go through this first so the app behaves + identically on both backends.""" + if value.tzinfo is None: + return value.replace(tzinfo=UTC) + return value + + +class Base(DeclarativeBase): + pass + + +class UUIDPrimaryKeyMixin: + id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4, unique=True) + + +class TimestampMixin: + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, onupdate=utcnow + ) diff --git a/apps/api/app/db/session.py b/apps/api/app/db/session.py new file mode 100644 index 0000000..c2203ba --- /dev/null +++ b/apps/api/app/db/session.py @@ -0,0 +1,50 @@ +"""Async SQLAlchemy engine/session setup. + +Works against Postgres (`postgresql+psycopg://...`) or SQLite +(`sqlite+aiosqlite://...`) depending on `DATABASE_URL` - the same models and +repositories run against either, which is what makes the no-Docker local dev +path possible. +""" + +from __future__ import annotations + +from collections.abc import AsyncGenerator +from functools import lru_cache + +from sqlalchemy.ext.asyncio import ( + AsyncEngine, + AsyncSession, + async_sessionmaker, + create_async_engine, +) + +from app.core.config import get_settings + + +@lru_cache +def get_engine() -> AsyncEngine: + settings = get_settings() + connect_args = {} + if settings.database_url.startswith("sqlite"): + connect_args = {"check_same_thread": False} + return create_async_engine( + settings.database_url, + echo=False, + pool_pre_ping=True, + connect_args=connect_args, + ) + + +@lru_cache +def get_sessionmaker() -> async_sessionmaker[AsyncSession]: + return async_sessionmaker(bind=get_engine(), expire_on_commit=False) + + +async def get_db() -> AsyncGenerator[AsyncSession, None]: + session_factory = get_sessionmaker() + async with session_factory() as session: + try: + yield session + except Exception: + await session.rollback() + raise diff --git a/apps/api/app/enrichment/__init__.py b/apps/api/app/enrichment/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/app/enrichment/base.py b/apps/api/app/enrichment/base.py new file mode 100644 index 0000000..af6d0dc --- /dev/null +++ b/apps/api/app/enrichment/base.py @@ -0,0 +1,93 @@ +"""Company-enrichment provider interface. Answers "what does a paid, +first-party data vendor already know about this company?" - a richer, +metered complement to the free SearchProvider/LLM discovery pipeline +(app/services/discovery_service.py), never a replacement for it. Every +method here maps to one NinjaPear (nubela.co) API endpoint; orchestration +(call order, the leadership-lookup cap, partial-failure handling) lives in +app/services/enrichment_service.py, not here - this Protocol is a dumb I/O +layer, matching app/search/base.py's shape. +""" + +from __future__ import annotations + +from typing import Protocol + +from pydantic import BaseModel, Field + + +class LeadershipMember(BaseModel): + name: str + title: str | None = None + work_email: str | None = None + profile_url: str | None = None + bio: str | None = None + + +class CompanyDetails(BaseModel): + description: str | None = None + industry: str | None = None + founded_year: int | None = None + specialties: list[str] = Field(default_factory=list) + leadership_team: list[LeadershipMember] = Field(default_factory=list) + employee_count_range: str | None = None + + +class FundingRound(BaseModel): + round_name: str | None = None + amount: str | None = None + date: str | None = None + investors: list[str] = Field(default_factory=list) + + +class CompanyFunding(BaseModel): + total_raised: str | None = None + rounds: list[FundingRound] = Field(default_factory=list) + + +class CompetitorWithReason(BaseModel): + name: str + reason: str | None = None + + +class Product(BaseModel): + name: str + description: str | None = None + category: str | None = None + + +class RecentUpdate(BaseModel): + type: str + text: str + url: str | None = None + date: str | None = None + + +class Customer(BaseModel): + name: str + relationship: str | None = None + + +class EnrichmentProvider(Protocol): + provider_name: str + + async def get_company_details(self, name: str, website: str | None) -> CompanyDetails: ... + + async def get_funding(self, name: str, website: str | None) -> CompanyFunding: ... + + async def get_updates(self, name: str, website: str | None) -> list[RecentUpdate]: ... + + async def get_competitors( + self, name: str, website: str | None + ) -> list[CompetitorWithReason]: ... + + async def get_products(self, name: str, website: str | None) -> list[Product]: ... + + async def get_customers(self, name: str, website: str | None) -> list[Customer]: ... + + async def get_work_email(self, person_name: str, company_website: str) -> str | None: ... + + async def get_person_profile( + self, person_name: str, company_website: str + ) -> tuple[str | None, str | None]: + """Returns (profile_url, bio).""" + ... diff --git a/apps/api/app/enrichment/factory.py b/apps/api/app/enrichment/factory.py new file mode 100644 index 0000000..51694fd --- /dev/null +++ b/apps/api/app/enrichment/factory.py @@ -0,0 +1,21 @@ +"""Resolves NINJAPEAR_API_KEY to a concrete enrichment provider instance. +Never imported directly by enrichment_service - always go through +`get_enrichment_provider()` so a future second vendor stays a one-line +config change, matching app/search/factory.py's shape.""" + +from __future__ import annotations + +from app.core.config import Settings, get_settings +from app.enrichment.base import EnrichmentProvider +from app.enrichment.mock import MockEnrichmentProvider + + +def get_enrichment_provider(settings: Settings | None = None) -> EnrichmentProvider: + settings = settings or get_settings() + + if settings.ninjapear_api_key: + from app.enrichment.ninjapear import NinjaPearProvider + + return NinjaPearProvider(settings) + + return MockEnrichmentProvider() diff --git a/apps/api/app/enrichment/mock.py b/apps/api/app/enrichment/mock.py new file mode 100644 index 0000000..7c3ac06 --- /dev/null +++ b/apps/api/app/enrichment/mock.py @@ -0,0 +1,50 @@ +"""Deterministic mock enrichment provider - the default (no +`NINJAPEAR_API_KEY` configured) and what every automated test runs against. +Never calls a network, never fabricates data it has no basis for: every +field comes back empty/`None`, same honesty philosophy as +`app/search/mock.py` and `app/collectors/patents.py`'s no-key path. In +practice the enrichment task is never even enqueued without a real key +(see `company_service.create_company`), so this mostly exists for tests +and for direct calls to `enrichment_service.enrich_company`. +""" + +from __future__ import annotations + +from app.enrichment.base import ( + CompanyDetails, + CompanyFunding, + CompetitorWithReason, + Customer, + Product, + RecentUpdate, +) + + +class MockEnrichmentProvider: + provider_name = "mock" + + async def get_company_details(self, name: str, website: str | None) -> CompanyDetails: + return CompanyDetails() + + async def get_funding(self, name: str, website: str | None) -> CompanyFunding: + return CompanyFunding() + + async def get_updates(self, name: str, website: str | None) -> list[RecentUpdate]: + return [] + + async def get_competitors(self, name: str, website: str | None) -> list[CompetitorWithReason]: + return [] + + async def get_products(self, name: str, website: str | None) -> list[Product]: + return [] + + async def get_customers(self, name: str, website: str | None) -> list[Customer]: + return [] + + async def get_work_email(self, person_name: str, company_website: str) -> str | None: + return None + + async def get_person_profile( + self, person_name: str, company_website: str + ) -> tuple[str | None, str | None]: + return None, None diff --git a/apps/api/app/enrichment/ninjapear.py b/apps/api/app/enrichment/ninjapear.py new file mode 100644 index 0000000..4a2459f --- /dev/null +++ b/apps/api/app/enrichment/ninjapear.py @@ -0,0 +1,213 @@ +"""NinjaPear (nubela.co) company-enrichment provider. A fixed, trusted, +first-party integration endpoint (like Brave/Twilio) - calls httpx +directly rather than through `safe_fetch`, which exists specifically to +guard arbitrary/user-supplied collector targets, not our own known-safe +API integrations (see app/search/brave.py for the same reasoning). + +Endpoint paths, parameters, and response field names below are taken from +nubela.co/llms-full.txt (a plain-text API reference, unlike the JS-rendered +docs site) and verified live against a real account. Every company-level +endpoint identifies the company by `website` only (NinjaPear has no +name-based lookup for these calls) - a company with no `official_website` +on file cannot be enriched at all, see `_require_website`. Response +parsing stays defensive (`.get()` throughout) since a live vendor API can +still change shape without notice. +""" + +from __future__ import annotations + +import httpx + +from app.core.config import Settings +from app.enrichment.base import ( + CompanyDetails, + CompanyFunding, + CompetitorWithReason, + Customer, + FundingRound, + LeadershipMember, + Product, + RecentUpdate, +) + +_BASE_URL = "https://nubela.co/api/v1" +_DEFAULT_TIMEOUT = 100 +_FUNDING_TIMEOUT = 300 # documented by NinjaPear as long-running (up to 5 min) + + +def _require_website(website: str | None) -> str: + if not website: + raise ValueError( + "NinjaPear identifies a company by website only - this company has none on file" + ) + return website + + +def _domain_from_website(website: str) -> str: + domain = website.split("//", 1)[-1].split("/", 1)[0] + return domain[4:] if domain.startswith("www.") else domain + + +def _split_name(person_name: str) -> tuple[str, str | None]: + parts = person_name.split(maxsplit=1) + return (parts[0], parts[1] if len(parts) > 1 else None) + + +class NinjaPearProvider: + provider_name = "ninjapear" + + def __init__(self, settings: Settings) -> None: + self._api_key = settings.ninjapear_api_key + + def _headers(self) -> dict[str, str]: + return {"Authorization": f"Bearer {self._api_key}"} + + async def _get(self, url: str, params: dict[str, str], *, timeout: float) -> dict: + async with httpx.AsyncClient(timeout=timeout) as client: + response = await client.get(url, params=params, headers=self._headers()) + response.raise_for_status() + return response.json() + + async def get_company_details(self, name: str, website: str | None) -> CompanyDetails: + data = await self._get( + f"{_BASE_URL}/company/details", + {"website": _require_website(website)}, + timeout=_DEFAULT_TIMEOUT, + ) + leadership = [ + LeadershipMember(name=exec_["name"], title=exec_.get("title") or exec_.get("role")) + for exec_ in data.get("executives") or [] + if exec_.get("name") + ] + # employee_count comes back as a raw int (e.g. 9030) and industry as + # a numeric taxonomy code, not human-readable strings - stringified + # defensively rather than left as-is, since our internal shape + # types both as `str | None`. + employee_count = data.get("employee_count") + industry = data.get("industry") + return CompanyDetails( + description=data.get("description"), + industry=str(industry) if industry is not None else None, + founded_year=data.get("founded_year"), + specialties=data.get("specialties") or [], + leadership_team=leadership, + employee_count_range=str(employee_count) if employee_count is not None else None, + ) + + async def get_funding(self, name: str, website: str | None) -> CompanyFunding: + data = await self._get( + f"{_BASE_URL}/company/funding", + {"website": _require_website(website)}, + timeout=_FUNDING_TIMEOUT, + ) + # amount/amount_usd/total_funds_raised are raw numbers, and each + # investor is an object (name/type/website/...), not a plain string + # - stringified/extracted defensively, same reasoning as + # get_company_details's employee_count/industry coercion. + rounds = [] + for r in data.get("funding_rounds") or []: + amount = r.get("amount_usd") or r.get("amount") + investors = [inv.get("name") for inv in (r.get("investors") or []) if inv.get("name")] + rounds.append( + FundingRound( + round_name=r.get("round_type"), + amount=str(amount) if amount is not None else None, + date=r.get("date"), + investors=investors, + ) + ) + total_raised = data.get("total_funds_raised_usd") or data.get("total_funds_raised") + return CompanyFunding( + total_raised=str(total_raised) if total_raised is not None else None, rounds=rounds + ) + + async def get_updates(self, name: str, website: str | None) -> list[RecentUpdate]: + data = await self._get( + f"{_BASE_URL}/company/updates", + {"website": _require_website(website)}, + timeout=_DEFAULT_TIMEOUT, + ) + return [ + RecentUpdate( + type=u.get("source", "update"), + text=u.get("title") or u.get("description") or "", + url=u.get("url"), + date=u.get("timestamp"), + ) + for u in data.get("updates") or [] + if u.get("title") or u.get("description") + ] + + async def get_competitors(self, name: str, website: str | None) -> list[CompetitorWithReason]: + data = await self._get( + f"{_BASE_URL}/competitor/listing", + {"website": _require_website(website)}, + timeout=_DEFAULT_TIMEOUT, + ) + return [ + CompetitorWithReason( + name=c.get("name") or c.get("website", ""), reason=c.get("competition_reason") + ) + for c in data.get("competitors") or [] + if c.get("website") or c.get("name") + ] + + async def get_products(self, name: str, website: str | None) -> list[Product]: + data = await self._get( + f"{_BASE_URL}/product/listing", + {"website": _require_website(website)}, + timeout=_DEFAULT_TIMEOUT, + ) + products = [] + for p in data.get("products") or []: + if not p.get("name"): + continue + categories = p.get("categories") or [] + products.append( + Product( + name=p["name"], + description=p.get("description"), + category=", ".join(categories) if categories else None, + ) + ) + return products + + async def get_customers(self, name: str, website: str | None) -> list[Customer]: + data = await self._get( + f"{_BASE_URL}/customer/listing", + {"website": _require_website(website)}, + timeout=_DEFAULT_TIMEOUT, + ) + # NinjaPear returns three separately-categorized arrays rather than + # one flat list - merged here with a relationship tag per group. + customers = [] + for relationship, key in ( + ("customer", "customers"), + ("investor", "investors"), + ("partner", "partner_platforms"), + ): + for entry in data.get(key) or []: + if entry.get("name"): + customers.append(Customer(name=entry["name"], relationship=relationship)) + return customers + + async def get_work_email(self, person_name: str, company_website: str) -> str | None: + first_name, last_name = _split_name(person_name) + params = {"first_name": first_name, "domain": _domain_from_website(company_website)} + if last_name: + params["last_name"] = last_name + data = await self._get(f"{_BASE_URL}/employee/work-email", params, timeout=_DEFAULT_TIMEOUT) + return data.get("work_email") + + async def get_person_profile( + self, person_name: str, company_website: str + ) -> tuple[str | None, str | None]: + first_name, _ = _split_name(person_name) + # v2 endpoint per NinjaPear's docs - the only one of the endpoints + # used here that isn't under /api/v1. + data = await self._get( + "https://nubela.co/api/v2/employee/profile", + {"first_name": first_name, "employer_website": company_website}, + timeout=_DEFAULT_TIMEOUT, + ) + return data.get("x_profile_url"), data.get("bio") diff --git a/apps/api/app/main.py b/apps/api/app/main.py new file mode 100644 index 0000000..c366659 --- /dev/null +++ b/apps/api/app/main.py @@ -0,0 +1,136 @@ +"""FastAPI application entrypoint.""" + +from __future__ import annotations + +import uuid +from contextlib import asynccontextmanager + +from fastapi import FastAPI, Request, status +from fastapi.encoders import jsonable_encoder +from fastapi.exceptions import RequestValidationError +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse, Response +from slowapi import _rate_limit_exceeded_handler +from slowapi.errors import RateLimitExceeded +from slowapi.middleware import SlowAPIMiddleware +from structlog.contextvars import bound_contextvars + +from app.api.v1.router import api_v1_router +from app.core.config import get_settings +from app.core.errors import ( + AppError, + AuthenticationError, + ConflictError, + ForbiddenError, + NotFoundError, + RateLimitedError, + ThrottledError, + ValidationAppError, +) +from app.core.logging import configure_logging, get_logger +from app.core.rate_limit import limiter + +settings = get_settings() +configure_logging(settings) +logger = get_logger(__name__) + + +@asynccontextmanager +async def lifespan(_app: FastAPI): + logger.info( + "startup", + app_env=settings.app_env, + auth_mode=settings.auth_mode, + llm_provider=settings.llm_provider, + ) + yield + logger.info("shutdown") + + +app = FastAPI( + title=settings.app_name, + version="0.1.0", + lifespan=lifespan, + docs_url="/docs", + redoc_url="/redoc", + openapi_url="/openapi.json", +) + +app.state.limiter = limiter +app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) +app.add_middleware(SlowAPIMiddleware) + +app.add_middleware( + CORSMiddleware, + # FRONTEND_URL may be a comma-separated list (e.g. localhost plus a LAN + # address) so the same API can serve a browser on this machine and one + # elsewhere on the network at once. + allow_origins=[origin.strip() for origin in settings.frontend_url.split(",") if origin.strip()], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.middleware("http") +async def correlation_id_middleware(request: Request, call_next) -> Response: + """Every log line emitted while handling this request carries the same + request_id (structlog contextvars, merged in automatically - see + core/logging.py), and the id is echoed back so a client/proxy log can be + cross-referenced with ours. Reuses an inbound X-Request-ID if a gateway + already set one, rather than always minting a fresh id.""" + request_id = request.headers.get("X-Request-ID") or str(uuid.uuid4()) + with bound_contextvars(request_id=request_id): + response = await call_next(request) + response.headers["X-Request-ID"] = request_id + return response + + +_APP_ERROR_STATUS_CODES: dict[type[AppError], int] = { + NotFoundError: status.HTTP_404_NOT_FOUND, + ConflictError: status.HTTP_409_CONFLICT, + AuthenticationError: status.HTTP_401_UNAUTHORIZED, + ForbiddenError: status.HTTP_403_FORBIDDEN, + ValidationAppError: status.HTTP_400_BAD_REQUEST, + RateLimitedError: status.HTTP_429_TOO_MANY_REQUESTS, + ThrottledError: status.HTTP_429_TOO_MANY_REQUESTS, +} + + +@app.exception_handler(AppError) +async def app_error_handler(_request: Request, exc: AppError) -> JSONResponse: + status_code = _APP_ERROR_STATUS_CODES.get(type(exc), status.HTTP_400_BAD_REQUEST) + content: dict = {"detail": str(exc)} + headers: dict[str, str] = {} + if isinstance(exc, ThrottledError) and exc.retry_after_seconds is not None: + content["retry_after_seconds"] = exc.retry_after_seconds + headers["Retry-After"] = str(exc.retry_after_seconds) + return JSONResponse(status_code=status_code, content=content, headers=headers) + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler( + _request: Request, exc: RequestValidationError +) -> JSONResponse: + return JSONResponse( + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, + content={"detail": "Invalid request", "errors": jsonable_encoder(exc.errors())}, + ) + + +@app.exception_handler(Exception) +async def unhandled_exception_handler(_request: Request, exc: Exception) -> JSONResponse: + logger.error("unhandled_exception", error=str(exc), error_type=type(exc).__name__) + return JSONResponse( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + content={"detail": "Internal server error"}, + ) + + +@app.get("/health", include_in_schema=False) +async def root_health() -> dict[str, str]: + """Plain root-level liveness probe for infra tooling (Docker, etc.).""" + return {"status": "ok"} + + +app.include_router(api_v1_router) diff --git a/apps/api/app/models/__init__.py b/apps/api/app/models/__init__.py new file mode 100644 index 0000000..76ba0dd --- /dev/null +++ b/apps/api/app/models/__init__.py @@ -0,0 +1,35 @@ +"""SQLAlchemy ORM models. + +Every model module is imported here so that `Base.metadata` (used by Alembic +autogenerate) sees the full schema. Add new model modules to this list as +they're created. +""" + +from __future__ import annotations + +from app.models.alert import Alert # noqa: F401 +from app.models.company import Company, CompanyAlias, Competitor # noqa: F401 +from app.models.company_enrichment import CompanyEnrichment # noqa: F401 +from app.models.detected_change import DetectedChange # noqa: F401 +from app.models.email_code import EmailCode # noqa: F401 +from app.models.ip_ban import IpBan # noqa: F401 +from app.models.ip_throttle_state import IpThrottleState # noqa: F401 +from app.models.monitor_configuration import MonitorConfiguration # noqa: F401 +from app.models.monitoring_run import MonitoringRun # noqa: F401 +from app.models.notification_delivery import NotificationDelivery # noqa: F401 +from app.models.notification_destination import ( # noqa: F401 + NotificationDestination, + NotificationDestinationCompany, +) +from app.models.password_history import PasswordHistoryEntry # noqa: F401 +from app.models.refresh_token import RefreshToken # noqa: F401 +from app.models.report import Report # noqa: F401 +from app.models.snapshot import Snapshot # noqa: F401 +from app.models.source import Source # noqa: F401 +from app.models.source_document import SourceDocument # noqa: F401 +from app.models.system_secret import SystemSecret # noqa: F401 +from app.models.unban_request import UnbanRequest # noqa: F401 +from app.models.user import User # noqa: F401 +from app.models.user_api_key import UserApiKey # noqa: F401 +from app.models.user_known_ip import UserKnownIp # noqa: F401 +from app.models.user_security_event import UserSecurityEvent # noqa: F401 diff --git a/apps/api/app/models/alert.py b/apps/api/app/models/alert.py new file mode 100644 index 0000000..ed041b0 --- /dev/null +++ b/apps/api/app/models/alert.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import uuid + +from sqlalchemy import Boolean, Enum, Float, ForeignKey, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base, TimestampMixin, UUIDPrimaryKeyMixin +from app.models.enums import SeverityLevel + + +class Alert(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "alerts" + + company_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("companies.id", ondelete="CASCADE"), index=True + ) + detected_change_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("detected_changes.id", ondelete="CASCADE"), index=True + ) + user_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), index=True + ) + title: Mapped[str] = mapped_column(String(200)) + summary: Mapped[str] = mapped_column(Text) + why_it_matters: Mapped[str] = mapped_column(Text) + severity: Mapped[SeverityLevel] = mapped_column( + Enum(SeverityLevel, native_enum=False, length=20) + ) + confidence: Mapped[float] = mapped_column(Float) + read: Mapped[bool] = mapped_column(Boolean, default=False) + resolved: Mapped[bool] = mapped_column(Boolean, default=False) diff --git a/apps/api/app/models/company.py b/apps/api/app/models/company.py new file mode 100644 index 0000000..52e1238 --- /dev/null +++ b/apps/api/app/models/company.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import uuid +from typing import TYPE_CHECKING + +from sqlalchemy import JSON, Enum, ForeignKey, String, Text +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.db.base import Base, TimestampMixin, UUIDPrimaryKeyMixin +from app.models.enums import CompanyStatus + +if TYPE_CHECKING: + from app.models.company_enrichment import CompanyEnrichment + from app.models.monitor_configuration import MonitorConfiguration + from app.models.notification_destination import NotificationDestinationCompany + + +class Company(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "companies" + + user_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), index=True + ) + name: Mapped[str] = mapped_column(String(200)) + slug: Mapped[str] = mapped_column(String(220), index=True) + official_website: Mapped[str | None] = mapped_column(String(500), nullable=True) + description: Mapped[str | None] = mapped_column(Text, nullable=True) + monitoring_focus: Mapped[str | None] = mapped_column(Text, nullable=True) + industry: Mapped[str | None] = mapped_column(String(120), nullable=True) + country: Mapped[str | None] = mapped_column(String(120), nullable=True) + region: Mapped[str | None] = mapped_column(String(120), nullable=True) + headquarters: Mapped[str | None] = mapped_column(String(200), nullable=True) + # Best-effort discovered identifiers, e.g. {"ticker": "ACME", "linkedin_url": "..."} + # - see app/prompts/company_profile.py. Empty dict, never fabricated. + public_identifiers: Mapped[dict[str, str]] = mapped_column(JSON, default=dict) + status: Mapped[CompanyStatus] = mapped_column( + Enum(CompanyStatus, native_enum=False, length=20), default=CompanyStatus.ACTIVE + ) + + aliases: Mapped[list[CompanyAlias]] = relationship( + back_populates="company", cascade="all, delete-orphan" + ) + competitors: Mapped[list[Competitor]] = relationship( + back_populates="company", cascade="all, delete-orphan" + ) + monitor_configuration: Mapped[MonitorConfiguration | None] = relationship( + back_populates="company", cascade="all, delete-orphan", uselist=False + ) + enrichment: Mapped[CompanyEnrichment | None] = relationship( + back_populates="company", cascade="all, delete-orphan", uselist=False + ) + notification_links: Mapped[list[NotificationDestinationCompany]] = relationship( + cascade="all, delete-orphan" + ) + + +class CompanyAlias(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "company_aliases" + + company_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("companies.id", ondelete="CASCADE"), index=True + ) + alias: Mapped[str] = mapped_column(String(200)) + + company: Mapped[Company] = relationship(back_populates="aliases") + + +class Competitor(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "competitors" + + company_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("companies.id", ondelete="CASCADE"), index=True + ) + name: Mapped[str] = mapped_column(String(200)) + + company: Mapped[Company] = relationship(back_populates="competitors") diff --git a/apps/api/app/models/company_enrichment.py b/apps/api/app/models/company_enrichment.py new file mode 100644 index 0000000..85328ea --- /dev/null +++ b/apps/api/app/models/company_enrichment.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import TYPE_CHECKING, Any + +from sqlalchemy import JSON, DateTime, Enum, ForeignKey, Integer +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.db.base import Base, TimestampMixin, UUIDPrimaryKeyMixin +from app.models.enums import EnrichmentStatus + +if TYPE_CHECKING: + from app.models.company import Company + + +class CompanyEnrichment(Base, UUIDPrimaryKeyMixin, TimestampMixin): + """One-shot, onboarding-time company enrichment from a paid third-party + provider (NinjaPear/nubela.co) - never re-fetched on a schedule, see + app/services/enrichment_service.py. `data` holds a documented (not + DB-enforced) shape: employee_count, description, specialties, + leadership_team (each optionally carrying work_email/profile_url/bio), + funding (total_raised + rounds), competitors (name+reason), products, + recent_updates, customers. `errors` maps section name -> error message + for whichever calls failed, so a partial result is never silently + presented as complete - same transparency principle as every other + source/collector in this app.""" + + __tablename__ = "company_enrichments" + + company_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("companies.id", ondelete="CASCADE"), unique=True, index=True + ) + status: Mapped[EnrichmentStatus] = mapped_column( + Enum(EnrichmentStatus, native_enum=False, length=20), default=EnrichmentStatus.PENDING + ) + data: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) + errors: Mapped[dict[str, str]] = mapped_column(JSON, default=dict) + credits_spent: Mapped[int | None] = mapped_column(Integer, nullable=True) + fetched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + company: Mapped[Company] = relationship(back_populates="enrichment") diff --git a/apps/api/app/models/detected_change.py b/apps/api/app/models/detected_change.py new file mode 100644 index 0000000..2c9aba7 --- /dev/null +++ b/apps/api/app/models/detected_change.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import uuid +from typing import Any + +from sqlalchemy import JSON, Enum, Float, ForeignKey, String +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base, TimestampMixin, UUIDPrimaryKeyMixin +from app.models.enums import ChangeStatus, ChangeType, SeverityLevel + + +class DetectedChange(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "detected_changes" + + company_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("companies.id", ondelete="CASCADE"), index=True + ) + source_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("sources.id", ondelete="CASCADE"), index=True + ) + monitoring_run_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("monitoring_runs.id", ondelete="CASCADE"), index=True + ) + previous_snapshot_id: Mapped[uuid.UUID | None] = mapped_column( + ForeignKey("snapshots.id", ondelete="SET NULL"), nullable=True + ) + current_snapshot_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("snapshots.id", ondelete="CASCADE") + ) + change_type: Mapped[ChangeType] = mapped_column(Enum(ChangeType, native_enum=False, length=30)) + raw_diff: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) + significance_score: Mapped[float] = mapped_column(Float) + confidence_score: Mapped[float] = mapped_column(Float) + severity: Mapped[SeverityLevel] = mapped_column( + Enum(SeverityLevel, native_enum=False, length=20) + ) + status: Mapped[ChangeStatus] = mapped_column( + Enum(ChangeStatus, native_enum=False, length=20), default=ChangeStatus.NEW + ) + # Short human-readable label, e.g. "3 new job postings detected" - + # populated deterministically here; Phase 7's LLM may later add a + # richer "why it matters" narrative on top without replacing this. + summary: Mapped[str] = mapped_column(String(500)) diff --git a/apps/api/app/models/email_code.py b/apps/api/app/models/email_code.py new file mode 100644 index 0000000..4df58bd --- /dev/null +++ b/apps/api/app/models/email_code.py @@ -0,0 +1,37 @@ +"""Email verification / password-reset codes. + +Only a SHA-256 hash of the 6-digit code is stored, never the raw value - +same "never store the raw secret" precedent as RefreshToken.token_hash. A +short numeric code doesn't need Argon2's cost; it needs short expiry plus +the IP throttle system (app/services/ip_throttle_service.py) guarding how +often it can be guessed or resent. +""" + +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import TYPE_CHECKING + +from sqlalchemy import DateTime, Enum, ForeignKey, String +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.db.base import Base, TimestampMixin, UUIDPrimaryKeyMixin +from app.models.enums import EmailCodePurpose + +if TYPE_CHECKING: + from app.models.user import User + + +class EmailCode(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "email_codes" + + user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) + purpose: Mapped[EmailCodePurpose] = mapped_column( + Enum(EmailCodePurpose, native_enum=False, length=20) + ) + code_hash: Mapped[str] = mapped_column(String(64), index=True) + expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + used_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + user: Mapped[User] = relationship() diff --git a/apps/api/app/models/enums.py b/apps/api/app/models/enums.py new file mode 100644 index 0000000..18c67ca --- /dev/null +++ b/apps/api/app/models/enums.py @@ -0,0 +1,176 @@ +"""Shared string enums for ORM models. Mirrored in apps/web/lib/types.ts and +packages/shared/src/index.ts - keep those in sync by hand when changing this.""" + +from __future__ import annotations + +from enum import StrEnum + + +class CompanyStatus(StrEnum): + ACTIVE = "active" + PAUSED = "paused" + + +class MonitoringFrequency(StrEnum): + HOURLY = "hourly" + EVERY_6_HOURS = "every_6_hours" + EVERY_12_HOURS = "every_12_hours" + DAILY = "daily" + EVERY_2_DAYS = "every_2_days" + WEEKLY = "weekly" + EVERY_2_WEEKS = "every_2_weeks" + MONTHLY = "monthly" + CUSTOM = "custom" + + +# Minimum minutes represented by each non-custom frequency, used both to +# compute next_run and to enforce MINIMUM_MONITORING_INTERVAL_MINUTES. +FREQUENCY_MINUTES: dict[MonitoringFrequency, int] = { + MonitoringFrequency.HOURLY: 60, + MonitoringFrequency.EVERY_6_HOURS: 6 * 60, + MonitoringFrequency.EVERY_12_HOURS: 12 * 60, + MonitoringFrequency.DAILY: 24 * 60, + MonitoringFrequency.EVERY_2_DAYS: 2 * 24 * 60, + MonitoringFrequency.WEEKLY: 7 * 24 * 60, + MonitoringFrequency.EVERY_2_WEEKS: 14 * 24 * 60, + MonitoringFrequency.MONTHLY: 30 * 24 * 60, +} + + +class SeverityLevel(StrEnum): + CRITICAL = "critical" + HIGH = "high" + MEDIUM = "medium" + LOW = "low" + + +# Ordering for threshold comparisons (index 0 = most severe). +SEVERITY_ORDER: list[SeverityLevel] = [ + SeverityLevel.CRITICAL, + SeverityLevel.HIGH, + SeverityLevel.MEDIUM, + SeverityLevel.LOW, +] + + +class NotificationType(StrEnum): + EMAIL = "email" + SMS = "sms" + CONSOLE = "console" + + +class SourceType(StrEnum): + WEBSITE = "website" + RSS = "rss" + CUSTOM_URL = "custom_url" + SEC_EDGAR = "sec_edgar" + GITHUB = "github" + JOB_POSTING = "job_posting" + PATENT = "patent" + REVIEW = "review" + GOV_CONTRACT = "gov_contract" + + +class SourceStatus(StrEnum): + ACTIVE = "active" + DISABLED = "disabled" + RATE_LIMITED = "rate_limited" + AUTH_REQUIRED = "auth_required" + BLOCKED_BY_POLICY = "blocked_by_policy" + FAILED = "failed" + + +class MonitoringRunTrigger(StrEnum): + SCHEDULED = "scheduled" + MANUAL = "manual" + INITIAL = "initial" + RETRY = "retry" + + +class MonitoringRunStatus(StrEnum): + QUEUED = "queued" + RUNNING = "running" + SUCCESSFUL = "successful" + PARTIAL = "partial" + FAILED = "failed" + + +class ChangeType(StrEnum): + NEW_DOCUMENT = "new_document" + REMOVED_DOCUMENT = "removed_document" + CONTENT_MODIFIED = "content_modified" + PRICE_CHANGE = "price_change" + LEADERSHIP_CHANGE = "leadership_change" + FILING_NEW = "filing_new" + + +class ChangeStatus(StrEnum): + NEW = "new" + ACKNOWLEDGED = "acknowledged" + DISMISSED = "dismissed" + + +class ReportType(StrEnum): + BASELINE = "baseline" + UPDATE = "update" + MONTHLY = "monthly" + MANUAL = "manual" + + +class NotificationDeliveryStatus(StrEnum): + PENDING = "pending" + SENT = "sent" + FAILED = "failed" + + +class EnrichmentStatus(StrEnum): + PENDING = "pending" + PARTIAL = "partial" + COMPLETE = "complete" + FAILED = "failed" + + +class EmailCodePurpose(StrEnum): + VERIFY_EMAIL = "verify_email" + PASSWORD_RESET = "password_reset" + + +class ThrottleAction(StrEnum): + RESEND_VERIFICATION = "resend_verification" + RESEND_RESET = "resend_reset" + FAILED_LOGIN = "failed_login" + VERIFY_EMAIL_CODE = "verify_email_code" + CONFIRM_RESET_CODE = "confirm_reset_code" + + +class SecurityEventType(StrEnum): + LOGIN_SUCCESS = "login_success" + LOGIN_FAILED = "login_failed" + ACCOUNT_LOCKED = "account_locked" + PASSWORD_RESET_REQUESTED = "password_reset_requested" + PASSWORD_RESET_COMPLETED = "password_reset_completed" + EMAIL_VERIFICATION_SENT = "email_verification_sent" + EMAIL_VERIFIED = "email_verified" + SERVER_SECRET_UPDATED = "server_secret_updated" + API_KEY_UPDATED = "api_key_updated" + + +class ApiKeyProvider(StrEnum): + """Third-party providers a user can supply their own key for - see + app/services/user_api_key_service.py's PROVIDER_META for the matching + Settings field, display label, and credits/notes shown in Settings.""" + + ANTHROPIC = "anthropic" + BRAVE_SEARCH = "brave_search" + NINJAPEAR = "ninjapear" + USPTO = "uspto" + + +class SystemSecretKey(StrEnum): + """Server-wide (not per-user) secrets an admin can configure from the + Settings page instead of only via .env - see + app/services/system_secret_service.py's META for the matching Settings + field and display label.""" + + TURNSTILE_SITE_KEY = "turnstile_site_key" + TURNSTILE_SECRET = "turnstile_secret" diff --git a/apps/api/app/models/ip_ban.py b/apps/api/app/models/ip_ban.py new file mode 100644 index 0000000..87ae6db --- /dev/null +++ b/apps/api/app/models/ip_ban.py @@ -0,0 +1,21 @@ +"""Global per-IP bans. Deliberately separate from IpThrottleState - once any +action type escalates an IP to permanent, that IP is blocked from every +sensitive endpoint (register/login/resend/reset), not just the one action +that triggered it.""" + +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import DateTime, String +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base, TimestampMixin, UUIDPrimaryKeyMixin + + +class IpBan(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "ip_bans" + + ip_address: Mapped[str] = mapped_column(String(45), unique=True, index=True) + banned_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + reason: Mapped[str] = mapped_column(String(255)) diff --git a/apps/api/app/models/ip_throttle_state.py b/apps/api/app/models/ip_throttle_state.py new file mode 100644 index 0000000..c58fa70 --- /dev/null +++ b/apps/api/app/models/ip_throttle_state.py @@ -0,0 +1,29 @@ +"""Per-IP, per-action escalation state for the throttle/ban engine +(app/services/ip_throttle_service.py). `offense_count` is the "memory" that +survives a completed timeout cycle - only a manual admin unban resets it.""" + +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import DateTime, Enum, Integer, String, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base, TimestampMixin, UUIDPrimaryKeyMixin +from app.models.enums import ThrottleAction + + +class IpThrottleState(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "ip_throttle_state" + __table_args__ = ( + UniqueConstraint("ip_address", "action", name="uq_ip_throttle_state_ip_action"), + ) + + ip_address: Mapped[str] = mapped_column(String(45), index=True) + action: Mapped[ThrottleAction] = mapped_column( + Enum(ThrottleAction, native_enum=False, length=24) + ) + attempt_count: Mapped[int] = mapped_column(Integer, default=0) + next_allowed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + timeout_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + offense_count: Mapped[int] = mapped_column(Integer, default=0) diff --git a/apps/api/app/models/monitor_configuration.py b/apps/api/app/models/monitor_configuration.py new file mode 100644 index 0000000..810d10d --- /dev/null +++ b/apps/api/app/models/monitor_configuration.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import TYPE_CHECKING, Any + +from sqlalchemy import JSON, Boolean, DateTime, Enum, ForeignKey, Integer, String +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.db.base import Base, TimestampMixin, UUIDPrimaryKeyMixin +from app.models.enums import MonitoringFrequency, SeverityLevel + +if TYPE_CHECKING: + from app.models.company import Company + + +class MonitorConfiguration(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "monitor_configurations" + + company_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("companies.id", ondelete="CASCADE"), unique=True, index=True + ) + frequency_type: Mapped[MonitoringFrequency] = mapped_column( + Enum(MonitoringFrequency, native_enum=False, length=30), + default=MonitoringFrequency.WEEKLY, + ) + # Only meaningful when frequency_type == CUSTOM: interval_minutes takes + # precedence if set, otherwise cron_expression is used. + interval_minutes: Mapped[int | None] = mapped_column(Integer, nullable=True) + cron_expression: Mapped[str | None] = mapped_column(String(120), nullable=True) + timezone: Mapped[str] = mapped_column(String(64), default="America/New_York") + enabled: Mapped[bool] = mapped_column(Boolean, default=True) + next_run: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + last_run: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + severity_threshold: Mapped[SeverityLevel] = mapped_column( + Enum(SeverityLevel, native_enum=False, length=20), default=SeverityLevel.MEDIUM + ) + source_configuration: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) + + company: Mapped[Company] = relationship(back_populates="monitor_configuration") diff --git a/apps/api/app/models/monitoring_run.py b/apps/api/app/models/monitoring_run.py new file mode 100644 index 0000000..17b286b --- /dev/null +++ b/apps/api/app/models/monitoring_run.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import uuid +from datetime import datetime + +from sqlalchemy import DateTime, Enum, ForeignKey, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base, TimestampMixin, UUIDPrimaryKeyMixin +from app.models.enums import MonitoringRunStatus, MonitoringRunTrigger + + +class MonitoringRun(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "monitoring_runs" + + company_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("companies.id", ondelete="CASCADE"), index=True + ) + trigger_type: Mapped[MonitoringRunTrigger] = mapped_column( + Enum(MonitoringRunTrigger, native_enum=False, length=20) + ) + status: Mapped[MonitoringRunStatus] = mapped_column( + Enum(MonitoringRunStatus, native_enum=False, length=20), + default=MonitoringRunStatus.QUEUED, + ) + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + sources_attempted: Mapped[int] = mapped_column(Integer, default=0) + sources_successful: Mapped[int] = mapped_column(Integer, default=0) + sources_failed: Mapped[int] = mapped_column(Integer, default=0) + items_collected: Mapped[int] = mapped_column(Integer, default=0) + changes_detected: Mapped[int] = mapped_column(Integer, default=0) + error_summary: Mapped[str | None] = mapped_column(Text, nullable=True) + worker_task_id: Mapped[str | None] = mapped_column(String(255), nullable=True) diff --git a/apps/api/app/models/notification_delivery.py b/apps/api/app/models/notification_delivery.py new file mode 100644 index 0000000..6d95339 --- /dev/null +++ b/apps/api/app/models/notification_delivery.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import uuid +from datetime import datetime + +from sqlalchemy import DateTime, Enum, ForeignKey, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base, TimestampMixin, UUIDPrimaryKeyMixin +from app.models.enums import NotificationDeliveryStatus + + +class NotificationDelivery(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "notification_deliveries" + + alert_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("alerts.id", ondelete="CASCADE"), index=True + ) + destination_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("notification_destinations.id", ondelete="CASCADE"), index=True + ) + provider: Mapped[str] = mapped_column(String(50)) + status: Mapped[NotificationDeliveryStatus] = mapped_column( + Enum(NotificationDeliveryStatus, native_enum=False, length=20), + default=NotificationDeliveryStatus.PENDING, + ) + attempt_count: Mapped[int] = mapped_column(Integer, default=0) + last_attempt: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + external_message_id: Mapped[str | None] = mapped_column(String(255), nullable=True) + error_message: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/apps/api/app/models/notification_destination.py b/apps/api/app/models/notification_destination.py new file mode 100644 index 0000000..8d1f802 --- /dev/null +++ b/apps/api/app/models/notification_destination.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import uuid +from typing import TYPE_CHECKING + +from sqlalchemy import Boolean, Enum, ForeignKey, String +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.db.base import Base, TimestampMixin, UUIDPrimaryKeyMixin +from app.models.enums import NotificationType, SeverityLevel + +if TYPE_CHECKING: + from app.models.company import Company + + +class NotificationDestination(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "notification_destinations" + + user_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), index=True + ) + type: Mapped[NotificationType] = mapped_column( + Enum(NotificationType, native_enum=False, length=20) + ) + # Email address, phone number, or a label for the console provider. + # Not a secret, but still PII - see SECURITY.md. + destination_value: Mapped[str] = mapped_column(String(320)) + verified: Mapped[bool] = mapped_column(Boolean, default=False) + enabled: Mapped[bool] = mapped_column(Boolean, default=True) + minimum_severity: Mapped[SeverityLevel] = mapped_column( + Enum(SeverityLevel, native_enum=False, length=20), default=SeverityLevel.MEDIUM + ) + + company_links: Mapped[list[NotificationDestinationCompany]] = relationship( + back_populates="destination", cascade="all, delete-orphan" + ) + + +class NotificationDestinationCompany(Base, TimestampMixin): + """Which companies a destination receives alerts for - a destination + with zero links is orphaned and gets garbage-collected (see + notification_destination_service.py) rather than left dangling.""" + + __tablename__ = "notification_destination_companies" + + destination_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("notification_destinations.id", ondelete="CASCADE"), primary_key=True + ) + company_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("companies.id", ondelete="CASCADE"), primary_key=True + ) + + destination: Mapped[NotificationDestination] = relationship(back_populates="company_links") + # Read-only path to the company's name for display; Company.notification_links + # (used only for cascade-delete) writes the same FK from the other direction, + # hence overlaps= to tell SQLAlchemy that's intentional, not a conflict. + company: Mapped[Company] = relationship(overlaps="notification_links") diff --git a/apps/api/app/models/password_history.py b/apps/api/app/models/password_history.py new file mode 100644 index 0000000..1a0f78c --- /dev/null +++ b/apps/api/app/models/password_history.py @@ -0,0 +1,28 @@ +"""Every password hash a user has ever had active - checked on password +reset so a user can't "reset" back to a password they (or an attacker who +learned it) has used before. Never used for anything except that +membership check; nothing reads these hashes back out for display.""" + +from __future__ import annotations + +import uuid +from typing import TYPE_CHECKING + +from sqlalchemy import ForeignKey, String +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.db.base import Base, TimestampMixin, UUIDPrimaryKeyMixin + +if TYPE_CHECKING: + from app.models.user import User + + +class PasswordHistoryEntry(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "password_history_entries" + + user_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), index=True + ) + password_hash: Mapped[str] = mapped_column(String(255)) + + user: Mapped[User] = relationship() diff --git a/apps/api/app/models/refresh_token.py b/apps/api/app/models/refresh_token.py new file mode 100644 index 0000000..dac0835 --- /dev/null +++ b/apps/api/app/models/refresh_token.py @@ -0,0 +1,32 @@ +"""Refresh token records. + +Only a hash of the token's `jti` is stored - never the JWT itself - so a +database read can't be replayed as a valid refresh token. Rotation on use +(one row per issuance, `revoked_at` set when superseded) limits the blast +radius of a leaked refresh token to its remaining lifetime. +""" + +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import TYPE_CHECKING + +from sqlalchemy import DateTime, ForeignKey, String +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.db.base import Base, TimestampMixin, UUIDPrimaryKeyMixin + +if TYPE_CHECKING: + from app.models.user import User + + +class RefreshToken(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "refresh_tokens" + + user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) + token_hash: Mapped[str] = mapped_column(String(64), unique=True, index=True) + expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + user: Mapped[User] = relationship(back_populates="refresh_tokens") diff --git a/apps/api/app/models/report.py b/apps/api/app/models/report.py new file mode 100644 index 0000000..9b4bc9f --- /dev/null +++ b/apps/api/app/models/report.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import uuid +from typing import Any + +from sqlalchemy import JSON, Enum, ForeignKey, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base, TimestampMixin, UUIDPrimaryKeyMixin +from app.models.enums import ReportType + + +class Report(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "reports" + + company_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("companies.id", ondelete="CASCADE"), index=True + ) + monitoring_run_id: Mapped[uuid.UUID | None] = mapped_column( + ForeignKey("monitoring_runs.id", ondelete="SET NULL"), nullable=True, index=True + ) + report_type: Mapped[ReportType] = mapped_column(Enum(ReportType, native_enum=False, length=20)) + title: Mapped[str] = mapped_column(String(300)) + executive_summary: Mapped[str] = mapped_column(Text) + structured_report: Mapped[dict[str, Any]] = mapped_column(JSON) + markdown_content: Mapped[str] = mapped_column(Text) + model_provider: Mapped[str] = mapped_column(String(50)) + model_name: Mapped[str] = mapped_column(String(100)) + prompt_version: Mapped[str] = mapped_column(String(20), default="v1") diff --git a/apps/api/app/models/snapshot.py b/apps/api/app/models/snapshot.py new file mode 100644 index 0000000..d30b9bb --- /dev/null +++ b/apps/api/app/models/snapshot.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import uuid +from typing import Any + +from sqlalchemy import JSON, ForeignKey, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base, TimestampMixin, UUIDPrimaryKeyMixin + + +class Snapshot(Base, UUIDPrimaryKeyMixin, TimestampMixin): + """A structured, comparable summary of a source's state at a point in + time - what change detection (Phase 6) diffs against the prior snapshot + for the same source.""" + + __tablename__ = "snapshots" + + company_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("companies.id", ondelete="CASCADE"), index=True + ) + source_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("sources.id", ondelete="CASCADE"), index=True + ) + snapshot_type: Mapped[str] = mapped_column(String(50)) + hash: Mapped[str] = mapped_column(String(64), index=True) + structured_summary: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) + text_summary: Mapped[str | None] = mapped_column(Text, nullable=True) + monitoring_run_id: Mapped[uuid.UUID | None] = mapped_column( + ForeignKey("monitoring_runs.id", ondelete="SET NULL"), nullable=True, index=True + ) diff --git a/apps/api/app/models/source.py b/apps/api/app/models/source.py new file mode 100644 index 0000000..94824de --- /dev/null +++ b/apps/api/app/models/source.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any + +from sqlalchemy import JSON, Boolean, DateTime, Enum, Float, ForeignKey, Integer, String +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base, TimestampMixin, UUIDPrimaryKeyMixin +from app.models.enums import MonitoringFrequency, SourceStatus, SourceType + + +class Source(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "sources" + + company_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("companies.id", ondelete="CASCADE"), index=True + ) + source_type: Mapped[SourceType] = mapped_column(Enum(SourceType, native_enum=False, length=20)) + name: Mapped[str] = mapped_column(String(200)) + base_url: Mapped[str | None] = mapped_column(String(500), nullable=True) + active: Mapped[bool] = mapped_column(Boolean, default=True) + status: Mapped[SourceStatus] = mapped_column( + Enum(SourceStatus, native_enum=False, length=20), default=SourceStatus.ACTIVE + ) + trust_score: Mapped[float] = mapped_column(Float, default=0.7) + last_checked: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + last_successful_check: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + failure_count: Mapped[int] = mapped_column(Integer, default=0) + configuration_metadata: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) + + # Per-source check cadence override - NULL frequency_type means "inherit + # the company's default MonitorConfiguration cadence" (the behavior for + # every source before this feature existed, and still the default for + # any source that never sets an override). See app/tasks/scheduler.py + # and app/services/scheduling.py for how these combine into due-ness. + frequency_type: Mapped[MonitoringFrequency | None] = mapped_column( + Enum(MonitoringFrequency, native_enum=False, length=20), nullable=True + ) + interval_minutes: Mapped[int | None] = mapped_column(Integer, nullable=True) + cron_expression: Mapped[str | None] = mapped_column(String(120), nullable=True) + next_check: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) diff --git a/apps/api/app/models/source_document.py b/apps/api/app/models/source_document.py new file mode 100644 index 0000000..a41c413 --- /dev/null +++ b/apps/api/app/models/source_document.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any + +from sqlalchemy import JSON, DateTime, Float, ForeignKey, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base, TimestampMixin, UUIDPrimaryKeyMixin + + +class SourceDocument(Base, UUIDPrimaryKeyMixin, TimestampMixin): + """A single piece of collected content. Kept lean on purpose - raw HTML + is not stored, only extracted text - per the "avoid saving unnecessary + full HTML indefinitely" rule in the spec.""" + + __tablename__ = "source_documents" + + source_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("sources.id", ondelete="CASCADE"), index=True + ) + company_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("companies.id", ondelete="CASCADE"), index=True + ) + url: Mapped[str] = mapped_column(String(1000)) + canonical_url: Mapped[str] = mapped_column(String(1000), index=True) + title: Mapped[str | None] = mapped_column(String(500), nullable=True) + author: Mapped[str | None] = mapped_column(String(200), nullable=True) + publication_date: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + retrieved_date: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + content_text: Mapped[str] = mapped_column(Text) + content_hash: Mapped[str] = mapped_column(String(64), index=True) + metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) + language: Mapped[str | None] = mapped_column(String(16), nullable=True) + http_status: Mapped[int | None] = mapped_column(Integer, nullable=True) + extraction_method: Mapped[str] = mapped_column(String(50)) + trust_score: Mapped[float] = mapped_column(Float, default=0.7) diff --git a/apps/api/app/models/system_secret.py b/apps/api/app/models/system_secret.py new file mode 100644 index 0000000..f2953af --- /dev/null +++ b/apps/api/app/models/system_secret.py @@ -0,0 +1,24 @@ +"""A server-wide secret (e.g. Turnstile site key/secret), encrypted at rest +(app/core/crypto.py). Unlike UserApiKey, this isn't scoped to a user - it's +one value shared by the whole app, admin-editable from the Settings page +instead of only via .env. When set, +app/services/system_secret_service.py's get_effective_settings substitutes +it in place of the server's global .env-configured value - see that module +for the full fallback logic.""" + +from __future__ import annotations + +from sqlalchemy import Enum, Text +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base, TimestampMixin, UUIDPrimaryKeyMixin +from app.models.enums import SystemSecretKey + + +class SystemSecret(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "system_secrets" + + key: Mapped[SystemSecretKey] = mapped_column( + Enum(SystemSecretKey, native_enum=False, length=32), unique=True + ) + encrypted_value: Mapped[str] = mapped_column(Text) diff --git a/apps/api/app/models/unban_request.py b/apps/api/app/models/unban_request.py new file mode 100644 index 0000000..acb96d3 --- /dev/null +++ b/apps/api/app/models/unban_request.py @@ -0,0 +1,17 @@ +"""Manual unban requests from banned visitors - one per IP per 24h, enforced +in the service layer at insert time. Purely a queue for admin review; no +automated unban happens from this table.""" + +from __future__ import annotations + +from sqlalchemy import String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base, TimestampMixin, UUIDPrimaryKeyMixin + + +class UnbanRequest(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "unban_requests" + + ip_address: Mapped[str] = mapped_column(String(45), index=True) + message: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/apps/api/app/models/user.py b/apps/api/app/models/user.py new file mode 100644 index 0000000..364d154 --- /dev/null +++ b/apps/api/app/models/user.py @@ -0,0 +1,47 @@ +"""User account model. + +`password_hash` is nullable because `AUTH_MODE=local` provisions a single +fixed user with no password at all - that mode never routes through +password verification, so there's nothing to hash. +""" + +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import TYPE_CHECKING + +from sqlalchemy import Boolean, DateTime, Integer, String +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.db.base import Base, TimestampMixin, UUIDPrimaryKeyMixin + +if TYPE_CHECKING: + from app.models.refresh_token import RefreshToken + +# Fixed, deterministic user id used for AUTH_MODE=local so the same row is +# reused across restarts rather than multiplying "local dev user" rows. +LOCAL_DEV_USER_ID = uuid.UUID("00000000-0000-0000-0000-000000000001") +LOCAL_DEV_USER_EMAIL = "local@ci-agent.local" + + +class User(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "users" + + email: Mapped[str] = mapped_column(String(320), unique=True, index=True) + password_hash: Mapped[str | None] = mapped_column(String(255), nullable=True) + display_name: Mapped[str] = mapped_column(String(120)) + timezone: Mapped[str] = mapped_column(String(64), default="America/New_York") + is_active: Mapped[bool] = mapped_column(Boolean, default=True) + is_admin: Mapped[bool] = mapped_column(Boolean, default=False) + + # Phase 19: email verification + escalating failed-login lockout. The + # fixed AUTH_MODE=local user is seeded as already-verified (it never + # goes through this flow - see auth_service.get_or_create_local_user). + email_verified: Mapped[bool] = mapped_column(Boolean, default=False) + failed_login_count: Mapped[int] = mapped_column(Integer, default=0) + locked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + refresh_tokens: Mapped[list[RefreshToken]] = relationship( + back_populates="user", cascade="all, delete-orphan" + ) diff --git a/apps/api/app/models/user_api_key.py b/apps/api/app/models/user_api_key.py new file mode 100644 index 0000000..3788104 --- /dev/null +++ b/apps/api/app/models/user_api_key.py @@ -0,0 +1,36 @@ +"""A user's own API key for a given third-party provider, encrypted at +rest (app/core/crypto.py). When set, app/services/user_api_key_service.py's +get_effective_settings substitutes it in place of the server's global +.env-configured key for that user's own requests - see that module for the +full fallback logic.""" + +from __future__ import annotations + +import uuid +from typing import TYPE_CHECKING + +from sqlalchemy import Enum, ForeignKey, Text, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.db.base import Base, TimestampMixin, UUIDPrimaryKeyMixin +from app.models.enums import ApiKeyProvider + +if TYPE_CHECKING: + from app.models.user import User + + +class UserApiKey(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "user_api_keys" + __table_args__ = ( + UniqueConstraint("user_id", "provider", name="uq_user_api_keys_user_provider"), + ) + + user_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), index=True + ) + provider: Mapped[ApiKeyProvider] = mapped_column( + Enum(ApiKeyProvider, native_enum=False, length=16) + ) + encrypted_key: Mapped[str] = mapped_column(Text) + + user: Mapped[User] = relationship() diff --git a/apps/api/app/models/user_known_ip.py b/apps/api/app/models/user_known_ip.py new file mode 100644 index 0000000..2bdc509 --- /dev/null +++ b/apps/api/app/models/user_known_ip.py @@ -0,0 +1,36 @@ +"""Every distinct IP an account has ever signed in from - one row per +(user, ip) pair, first_seen_at set once and last_seen_at touched on every +subsequent sign-in from that same IP. Pure data capture for now (see +app/services/auth_service.py's sign-in paths, both real login and the +local-dev bypass) - nothing currently reads this table, but it's the +foundation a later "new device/location" security feature would query +against without needing to scan/dedupe the much larger, append-only +user_security_events log.""" + +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import TYPE_CHECKING + +from sqlalchemy import DateTime, ForeignKey, String, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.db.base import Base, TimestampMixin, UUIDPrimaryKeyMixin + +if TYPE_CHECKING: + from app.models.user import User + + +class UserKnownIp(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "user_known_ips" + __table_args__ = (UniqueConstraint("user_id", "ip_address", name="uq_user_known_ips_user_ip"),) + + user_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), index=True + ) + ip_address: Mapped[str] = mapped_column(String(45)) + first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + + user: Mapped[User] = relationship() diff --git a/apps/api/app/models/user_security_event.py b/apps/api/app/models/user_security_event.py new file mode 100644 index 0000000..88a5bd9 --- /dev/null +++ b/apps/api/app/models/user_security_event.py @@ -0,0 +1,31 @@ +"""Per-user security activity log - the user-facing counterpart to the +admin-only, app-wide Redis log feed (app/core/logging.py). Visible only to +the owning user via GET /auth/security-events.""" + +from __future__ import annotations + +import uuid +from typing import TYPE_CHECKING + +from sqlalchemy import Enum, ForeignKey, String +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.db.base import Base, TimestampMixin, UUIDPrimaryKeyMixin +from app.models.enums import SecurityEventType + +if TYPE_CHECKING: + from app.models.user import User + + +class UserSecurityEvent(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "user_security_events" + + user_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), index=True + ) + event_type: Mapped[SecurityEventType] = mapped_column( + Enum(SecurityEventType, native_enum=False, length=32) + ) + ip_address: Mapped[str] = mapped_column(String(45)) + + user: Mapped[User] = relationship() diff --git a/apps/api/app/notifications/__init__.py b/apps/api/app/notifications/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/app/notifications/base.py b/apps/api/app/notifications/base.py new file mode 100644 index 0000000..c38b608 --- /dev/null +++ b/apps/api/app/notifications/base.py @@ -0,0 +1,31 @@ +"""Notification provider interface. Every alert dispatch (app/services/ +alert_service.py) and every "test destination" action goes through this +Protocol, never a specific vendor SDK - swapping providers or adding a new +one doesn't touch the dispatch logic. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Protocol + + +@dataclass(frozen=True) +class NotificationMessage: + destination_value: str + subject: str + body_text: str + body_html: str | None = None + + +@dataclass(frozen=True) +class DeliveryResult: + success: bool + external_message_id: str | None = None + error: str | None = None + + +class NotificationProvider(Protocol): + provider_name: str + + async def send(self, message: NotificationMessage) -> DeliveryResult: ... diff --git a/apps/api/app/notifications/console.py b/apps/api/app/notifications/console.py new file mode 100644 index 0000000..9a80645 --- /dev/null +++ b/apps/api/app/notifications/console.py @@ -0,0 +1,23 @@ +"""Console provider: logs the notification instead of sending it anywhere. +Used in local dev fallback and wherever a destination type isn't yet wired +to a real transport.""" + +from __future__ import annotations + +from app.core.logging import get_logger +from app.notifications.base import DeliveryResult, NotificationMessage + +logger = get_logger(__name__) + + +class ConsoleProvider: + provider_name = "console" + + async def send(self, message: NotificationMessage) -> DeliveryResult: + logger.info( + "console_notification", + destination=message.destination_value, + subject=message.subject, + body=message.body_text, + ) + return DeliveryResult(success=True) diff --git a/apps/api/app/notifications/factory.py b/apps/api/app/notifications/factory.py new file mode 100644 index 0000000..03e7d23 --- /dev/null +++ b/apps/api/app/notifications/factory.py @@ -0,0 +1,31 @@ +"""Resolves a NotificationType to a concrete provider. Console is always +available and never costs anything; email/SMS are only meaningfully +configured when their settings are present (callers gate on +NOTIFICATION_SMS_ENABLED before ever routing to SMS - see alert_service.py). + +SMS has two interchangeable vendor providers (`SMS_PROVIDER=twilio|telnyx`), +same pattern as LLM_PROVIDER/SEARCH_PROVIDER elsewhere in the app - swapping +the setting changes which class this returns and nothing else has to change. +""" + +from __future__ import annotations + +from app.core.config import Settings +from app.models.enums import NotificationType +from app.notifications.base import NotificationProvider +from app.notifications.console import ConsoleProvider +from app.notifications.smtp_email import SmtpEmailProvider +from app.notifications.telnyx_sms import TelnyxSmsProvider +from app.notifications.twilio_sms import TwilioSmsProvider + + +def get_notification_provider( + notification_type: NotificationType, settings: Settings +) -> NotificationProvider: + if notification_type == NotificationType.EMAIL: + return SmtpEmailProvider(settings) + if notification_type == NotificationType.SMS: + if settings.sms_provider == "telnyx": + return TelnyxSmsProvider(settings) + return TwilioSmsProvider(settings) + return ConsoleProvider() diff --git a/apps/api/app/notifications/message_builder.py b/apps/api/app/notifications/message_builder.py new file mode 100644 index 0000000..89dab56 --- /dev/null +++ b/apps/api/app/notifications/message_builder.py @@ -0,0 +1,68 @@ +"""Builds the actual email/SMS/console message bodies for an alert. Kept +separate from alert_service.py's dispatch loop so the message format can be +tested in isolation. +""" + +from __future__ import annotations + +from app.core.config import Settings +from app.models.alert import Alert +from app.models.company import Company +from app.models.enums import NotificationType +from app.notifications.base import NotificationMessage + + +def build_alert_message( + notification_type: NotificationType, + destination_value: str, + company: Company, + alert: Alert, + settings: Settings, +) -> NotificationMessage: + dashboard_link = f"{settings.frontend_url}/alerts" + + if notification_type == NotificationType.SMS: + text = ( + f"CI Alert [{alert.severity.value.upper()}]: {alert.title}. " + f"Confidence {alert.confidence:.0%}. View details: {dashboard_link}" + ) + return NotificationMessage( + destination_value=destination_value, subject=alert.title, body_text=text[:480] + ) + + subject = f"[{alert.severity.value.upper()}] {company.name}: {alert.title}" + text_lines = [ + f"Company: {company.name}", + f"Alert: {alert.title}", + f"Severity: {alert.severity.value.title()}", + f"Confidence: {alert.confidence:.0%}", + "", + "What changed:", + alert.summary, + "", + "Why it matters:", + alert.why_it_matters, + "", + f"View in dashboard: {dashboard_link}", + f"Manage notification preferences: {settings.frontend_url}/settings", + ] + body_text = "\n".join(text_lines) + + body_html = ( + f"

{subject}

" + f"

Company: {company.name}
" + f"Severity: {alert.severity.value.title()}
" + f"Confidence: {alert.confidence:.0%}

" + f"

What changed:
{alert.summary}

" + f"

Why it matters:
{alert.why_it_matters}

" + f'

View in dashboard

' + f'

' + f'Manage notification preferences

' + ) + + return NotificationMessage( + destination_value=destination_value, + subject=subject, + body_text=body_text, + body_html=body_html, + ) diff --git a/apps/api/app/notifications/resend_email.py b/apps/api/app/notifications/resend_email.py new file mode 100644 index 0000000..beed26f --- /dev/null +++ b/apps/api/app/notifications/resend_email.py @@ -0,0 +1,50 @@ +"""Resend HTTP API email provider - a separate transport from SmtpEmailProvider +(which also happens to point at Resend's SMTP relay for alert notifications +in this deployment, but that's a different concern/sender identity; see +app/services/security_email_service.py). Fixed, trusted first-party vendor +endpoint - no SSRF guard needed, same reasoning as the NinjaPear/USPTO calls +elsewhere in this app. +""" + +from __future__ import annotations + +import httpx + +from app.core.config import Settings +from app.core.logging import get_logger +from app.notifications.base import DeliveryResult, NotificationMessage + +logger = get_logger(__name__) + +_RESEND_API_URL = "https://api.resend.com/emails" + + +class ResendEmailProvider: + provider_name = "resend" + + def __init__(self, settings: Settings) -> None: + self._settings = settings + + async def send(self, message: NotificationMessage) -> DeliveryResult: + payload = { + "from": self._settings.resend_security_from_email, + "to": [message.destination_value], + "subject": message.subject, + "text": message.body_text, + } + if message.body_html: + payload["html"] = message.body_html + + try: + async with httpx.AsyncClient(timeout=10) as client: + response = await client.post( + _RESEND_API_URL, + headers={"Authorization": f"Bearer {self._settings.resend_api_key}"}, + json=payload, + ) + response.raise_for_status() + data = response.json() + return DeliveryResult(success=True, external_message_id=data.get("id")) + except Exception as exc: # noqa: BLE001 - network/API failure path + logger.error("resend_send_failed", error=str(exc)) + return DeliveryResult(success=False, error=str(exc)) diff --git a/apps/api/app/notifications/smtp_email.py b/apps/api/app/notifications/smtp_email.py new file mode 100644 index 0000000..0401226 --- /dev/null +++ b/apps/api/app/notifications/smtp_email.py @@ -0,0 +1,53 @@ +"""SMTP email provider. Points at Mailpit in local dev (see docker-compose.yml) +and any real SMTP server in production - same code path either way. Uses +stdlib `smtplib` off the event loop via `asyncio.to_thread` rather than +adding an async SMTP dependency. +""" + +from __future__ import annotations + +import asyncio +import smtplib +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + +from app.core.config import Settings +from app.core.logging import get_logger +from app.notifications.base import DeliveryResult, NotificationMessage + +logger = get_logger(__name__) + + +class SmtpEmailProvider: + provider_name = "smtp" + + def __init__(self, settings: Settings) -> None: + self._settings = settings + + async def send(self, message: NotificationMessage) -> DeliveryResult: + try: + await asyncio.to_thread(self._send_sync, message) + return DeliveryResult(success=True) + except Exception as exc: # pragma: no cover - network/SMTP failure path + logger.error("smtp_send_failed", error=str(exc)) + return DeliveryResult(success=False, error=str(exc)) + + def _send_sync(self, message: NotificationMessage) -> None: + mime_message = MIMEMultipart("alternative") + mime_message["Subject"] = message.subject + mime_message["From"] = self._settings.smtp_from_email + mime_message["To"] = message.destination_value + mime_message.attach(MIMEText(message.body_text, "plain")) + if message.body_html: + mime_message.attach(MIMEText(message.body_html, "html")) + + with smtplib.SMTP(self._settings.smtp_host, self._settings.smtp_port, timeout=10) as server: + if self._settings.smtp_use_tls: + server.starttls() + if self._settings.smtp_username: + server.login(self._settings.smtp_username, self._settings.smtp_password) + server.sendmail( + self._settings.smtp_from_email, + [message.destination_value], + mime_message.as_string(), + ) diff --git a/apps/api/app/notifications/telnyx_sms.py b/apps/api/app/notifications/telnyx_sms.py new file mode 100644 index 0000000..c899dbf --- /dev/null +++ b/apps/api/app/notifications/telnyx_sms.py @@ -0,0 +1,50 @@ +"""Telnyx SMS provider via Telnyx's Programmable Messaging REST API (no SDK +dependency - just a Bearer-authenticated POST), mirroring twilio_sms.py's +shape. No-ops with a clear error if Telnyx isn't configured; the caller +(alert_service) is what additionally gates on `NOTIFICATION_SMS_ENABLED` +before ever constructing this provider. +""" + +from __future__ import annotations + +import httpx + +from app.core.config import Settings +from app.notifications.base import DeliveryResult, NotificationMessage + +_API_URL = "https://api.telnyx.com/v2/messages" + + +class TelnyxSmsProvider: + provider_name = "telnyx_sms" + + def __init__(self, settings: Settings) -> None: + self._settings = settings + + async def send(self, message: NotificationMessage) -> DeliveryResult: + settings = self._settings + if not (settings.telnyx_api_key and settings.telnyx_from_number): + return DeliveryResult(success=False, error="Telnyx is not configured") + + try: + async with httpx.AsyncClient(timeout=15) as client: + response = await client.post( + _API_URL, + headers={"Authorization": f"Bearer {settings.telnyx_api_key}"}, + json={ + "from": settings.telnyx_from_number, + "to": message.destination_value, + "text": message.body_text, + }, + ) + except httpx.HTTPError as exc: + return DeliveryResult(success=False, error=str(exc)) + + data = response.json() + if response.status_code >= 400: + errors = data.get("errors") or [] + detail = errors[0].get("detail") if errors else response.text[:200] + return DeliveryResult( + success=False, error=f"Telnyx error {response.status_code}: {detail}" + ) + return DeliveryResult(success=True, external_message_id=data.get("data", {}).get("id")) diff --git a/apps/api/app/notifications/twilio_sms.py b/apps/api/app/notifications/twilio_sms.py new file mode 100644 index 0000000..06102ba --- /dev/null +++ b/apps/api/app/notifications/twilio_sms.py @@ -0,0 +1,52 @@ +"""Twilio SMS provider via Twilio's plain REST API (no SDK dependency - +just an authenticated POST). No-ops with a clear error if Twilio isn't +configured; the caller (alert_service) is what additionally gates on +`NOTIFICATION_SMS_ENABLED` before ever constructing this provider. +""" + +from __future__ import annotations + +import httpx + +from app.core.config import Settings +from app.notifications.base import DeliveryResult, NotificationMessage + +_API_BASE = "https://api.twilio.com/2010-04-01" + + +class TwilioSmsProvider: + provider_name = "twilio_sms" + + def __init__(self, settings: Settings) -> None: + self._settings = settings + + async def send(self, message: NotificationMessage) -> DeliveryResult: + settings = self._settings + if not ( + settings.twilio_account_sid + and settings.twilio_auth_token + and settings.twilio_from_number + ): + return DeliveryResult(success=False, error="Twilio is not configured") + + url = f"{_API_BASE}/Accounts/{settings.twilio_account_sid}/Messages.json" + try: + async with httpx.AsyncClient(timeout=15) as client: + response = await client.post( + url, + auth=(settings.twilio_account_sid, settings.twilio_auth_token), + data={ + "From": settings.twilio_from_number, + "To": message.destination_value, + "Body": message.body_text, + }, + ) + except httpx.HTTPError as exc: + return DeliveryResult(success=False, error=str(exc)) + + if response.status_code >= 400: + return DeliveryResult( + success=False, error=f"Twilio error {response.status_code}: {response.text[:200]}" + ) + data = response.json() + return DeliveryResult(success=True, external_message_id=data.get("sid")) diff --git a/apps/api/app/prompts/__init__.py b/apps/api/app/prompts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/app/prompts/alert_summarization.py b/apps/api/app/prompts/alert_summarization.py new file mode 100644 index 0000000..2cefda0 --- /dev/null +++ b/apps/api/app/prompts/alert_summarization.py @@ -0,0 +1,46 @@ +"""Task F: alert summarization. Produces the concise, non-exaggerated text +shown in the alert/email/SMS (Phase 8) - short by design, honest about +confidence, no hype language. +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + +from app.analysis.llm.base import LLMProvider +from app.prompts.base import build_user_prompt + +SYSTEM_PROMPT = ( + "You write concise, factual alert summaries for a competitive intelligence tool. Never " + "exaggerate. State what changed, cite the evidence type, and note the confidence level " + "plainly. The title must be under 100 characters and must not use hype words like " + "'huge', 'massive', or 'game-changing'." +) + + +class AlertSummary(BaseModel): + title: str = Field(max_length=100) + summary: str + why_it_matters: str + + +async def summarize_alert( + llm: LLMProvider, + *, + company_name: str, + change_type: str, + change_summary: str, + severity: str, + confidence: float, + evidence_snippets: list[str], +) -> AlertSummary: + evidence = { + "company_name": company_name, + "change_type": change_type, + "change_summary": change_summary, + "severity": severity, + "confidence": confidence, + "evidence_snippets": evidence_snippets[:10], + } + user_prompt = build_user_prompt("Write a concise alert summary for this change.", evidence) + return await llm.generate_structured(SYSTEM_PROMPT, user_prompt, AlertSummary) diff --git a/apps/api/app/prompts/base.py b/apps/api/app/prompts/base.py new file mode 100644 index 0000000..d712fb9 --- /dev/null +++ b/apps/api/app/prompts/base.py @@ -0,0 +1,39 @@ +"""Shared helpers for building prompts and (for MockLLMProvider) recovering +the structured evidence a prompt was built from, without a real model call. +Every analysis task embeds its evidence as a fenced JSON block via +`build_user_prompt`, so this stays consistent across all six tasks and lets +the mock provider parse it back out deterministically. +""" + +from __future__ import annotations + +import json +from typing import Any + +_EVIDENCE_FENCE_START = "```json evidence" +_EVIDENCE_FENCE_END = "```" + + +def build_user_prompt(instructions: str, evidence: dict[str, Any]) -> str: + evidence_json = json.dumps(evidence, indent=2, default=str) + return ( + f"{instructions}\n\n" + "Evidence (only use what is provided here - never invent facts not present):\n" + f"{_EVIDENCE_FENCE_START}\n{evidence_json}\n{_EVIDENCE_FENCE_END}" + ) + + +def extract_evidence_block(user_prompt: str) -> dict[str, Any]: + """Recovers the evidence dict embedded by `build_user_prompt`. Used only + by MockLLMProvider, which has no model to actually read the prompt.""" + start = user_prompt.find(_EVIDENCE_FENCE_START) + if start == -1: + return {} + start += len(_EVIDENCE_FENCE_START) + end = user_prompt.find(_EVIDENCE_FENCE_END, start) + if end == -1: + return {} + try: + return json.loads(user_prompt[start:end].strip()) + except json.JSONDecodeError: + return {} diff --git a/apps/api/app/prompts/change_significance.py b/apps/api/app/prompts/change_significance.py new file mode 100644 index 0000000..33c717b --- /dev/null +++ b/apps/api/app/prompts/change_significance.py @@ -0,0 +1,60 @@ +"""Task E: change significance narrative. + +Severity itself stays deterministic (app/change_detection/scoring.py) - see +ARCHITECTURE.md: "the LLM narrates why a change matters; it does not decide +how much it matters." This task supplies the narrative explanation and a +second, independent opinion on whether the change looks real/meaningful and +notification-worthy, which the notification layer (Phase 8) can use as an +extra signal alongside the deterministic severity - it never overrides it. +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + +from app.analysis.llm.base import LLMProvider +from app.prompts.base import build_user_prompt + +SYSTEM_PROMPT = ( + "You are a competitive intelligence analyst reviewing one detected change between two " + "snapshots of a company's public information. Explain in plain language why this change " + "would or wouldn't matter to someone monitoring this company, given their stated focus. " + "Be honest about uncertainty - do not overstate a routine wording tweak as significant, " + "and do not undersell a genuine signal." +) + + +class ChangeSignificanceAssessment(BaseModel): + is_real_change: bool = Field(description="Does this look like a genuine change, not noise?") + is_meaningful: bool + why_it_matters: str + confidence: float = Field(ge=0.0, le=1.0) + should_notify: bool + + +async def assess_change_significance( + llm: LLMProvider, + *, + company_name: str, + monitoring_focus: str | None, + change_type: str, + change_summary: str, + added_text: list[str], + removed_text: list[str], + deterministic_severity: str, + deterministic_confidence: float, +) -> ChangeSignificanceAssessment: + evidence = { + "company_name": company_name, + "monitoring_focus": monitoring_focus, + "change_type": change_type, + "change_summary": change_summary, + "text_added": added_text[:20], + "text_removed": removed_text[:20], + "deterministic_severity": deterministic_severity, + "deterministic_confidence": deterministic_confidence, + } + user_prompt = build_user_prompt( + "Assess this detected change and explain why it does or doesn't matter.", evidence + ) + return await llm.generate_structured(SYSTEM_PROMPT, user_prompt, ChangeSignificanceAssessment) diff --git a/apps/api/app/prompts/company_profile.py b/apps/api/app/prompts/company_profile.py new file mode 100644 index 0000000..a5ffac0 --- /dev/null +++ b/apps/api/app/prompts/company_profile.py @@ -0,0 +1,72 @@ +"""Company-profile discovery extraction. Used once per company, at +onboarding time (see app/services/discovery_service.py) - never asked +"tell me everything about this company," only "given this fetched +homepage text and these search snippets, extract what's actually +supported." Leaves a field unset rather than guessing when the evidence +doesn't support it; the wizard's Review step shows unset fields as +"not found" for the user to fill in themselves. +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + +from app.analysis.llm.base import LLMProvider +from app.prompts.base import build_user_prompt + +SYSTEM_PROMPT = ( + "You are a competitive intelligence analyst building an initial company profile from " + "search results and a fetched homepage. Extract only what the evidence actually states or " + "clearly implies - never use outside knowledge, never guess. Leave a field null (or an " + "empty list) rather than filling it with a plausible-sounding guess. Aliases means other " + "names the company is or was known by (former names, common abbreviations, brand names) - " + "not synonyms or descriptions. Competitors means other named companies the evidence " + "explicitly identifies as competing in the same space." +) + + +class PublicIdentifier(BaseModel): + key: str = Field(description="e.g. 'ticker', 'linkedin_url', 'cik'") + value: str + + +class CompanyProfileExtraction(BaseModel): + description: str | None = Field( + default=None, + description="A short (1-3 sentence) factual summary of what the company does, drawn " + "strictly from the evidence - not a marketing tagline.", + ) + industry: str | None = None + country: str | None = None + region: str | None = None + headquarters: str | None = None + aliases: list[str] = Field(default_factory=list) + competitors: list[str] = Field(default_factory=list) + public_identifiers: list[PublicIdentifier] = Field( + default_factory=list, + description="Best-effort key/value pairs, e.g. ticker or linkedin_url - empty if none found. " + "A list of {key, value} pairs rather than a free-form object, since the Gemini Developer " + "API's structured-output mode rejects open-ended (additionalProperties) JSON schemas.", + ) + + +async def extract_company_profile( + llm: LLMProvider, + *, + company_name: str, + homepage_url: str | None, + homepage_text: str | None, + search_results: list[dict], +) -> CompanyProfileExtraction: + evidence = { + "company_name": company_name, + "homepage_url": homepage_url, + "homepage_text": (homepage_text or "")[:4000], + "search_results": search_results[:15], + } + user_prompt = build_user_prompt( + "Build an initial company profile (description, industry, country, region, " + "headquarters, aliases, competitors, public identifiers) strictly from this evidence.", + evidence, + ) + return await llm.generate_structured(SYSTEM_PROMPT, user_prompt, CompanyProfileExtraction) diff --git a/apps/api/app/prompts/extraction.py b/apps/api/app/prompts/extraction.py new file mode 100644 index 0000000..c5e865d --- /dev/null +++ b/apps/api/app/prompts/extraction.py @@ -0,0 +1,54 @@ +"""Task B: fact and signal extraction. Pulls discrete, evidence-anchored +signals (hiring, partnerships, leadership, financial, etc.) out of a single +document - the atomic units Task C (synthesis) and Task D (report +generation) later combine.""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + +from app.analysis.llm.base import LLMProvider +from app.prompts.base import build_user_prompt + +SYSTEM_PROMPT = ( + "You are a competitive intelligence analyst extracting discrete factual signals from a " + "single document. Extract only what the text actually states or clearly implies - never " + "add outside knowledge. Every signal must include the exact passage that supports it." +) + +_SIGNAL_TYPES = ( + "event, entity, date, location, product, leadership, partnership, hiring, investment, " + "manufacturing, technology, financial, regulatory, sentiment" +) + + +class ExtractedSignal(BaseModel): + signal_type: str = Field(description=f"One of: {_SIGNAL_TYPES}") + description: str + supporting_passage: str = Field(description="The exact quote from the document backing this") + date: str | None = None + entities: list[str] = Field(default_factory=list) + + +class ExtractionResult(BaseModel): + signals: list[ExtractedSignal] = Field(default_factory=list) + + +async def extract_signals( + llm: LLMProvider, + *, + document_title: str | None, + document_url: str, + document_text: str, +) -> ExtractionResult: + evidence = { + "document_title": document_title, + "document_url": document_url, + "document_text": document_text[:6000], + } + user_prompt = build_user_prompt( + "Extract every discrete factual signal from this document, each anchored to its " + "exact supporting passage.", + evidence, + ) + return await llm.generate_structured(SYSTEM_PROMPT, user_prompt, ExtractionResult) diff --git a/apps/api/app/prompts/relevance.py b/apps/api/app/prompts/relevance.py new file mode 100644 index 0000000..1b96e4c --- /dev/null +++ b/apps/api/app/prompts/relevance.py @@ -0,0 +1,51 @@ +"""Task A: document relevance. Decides whether a collected document is +actually about the target company and matches the user's stated focus, +before it's used as evidence for anything else.""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + +from app.analysis.llm.base import LLMProvider +from app.prompts.base import build_user_prompt + +SYSTEM_PROMPT = ( + "You are a competitive intelligence analyst. Given a single document and a company " + "profile, judge only whether the document is genuinely about that company and whether " + "it relates to the user's stated monitoring focus. Do not summarize the document's " + "content here - a later task does that. Be conservative: if the document could be about " + "a different company with a similar name, say so." +) + + +class RelevanceAssessment(BaseModel): + is_relevant: bool = Field(description="Is this document genuinely about the target company?") + matches_focus: bool = Field(description="Does it relate to the user's stated monitoring focus?") + topic_categories: list[str] = Field(default_factory=list) + source_reliability: float = Field(ge=0.0, le=1.0) + reasoning: str + + +async def assess_relevance( + llm: LLMProvider, + *, + company_name: str, + company_aliases: list[str], + monitoring_focus: str | None, + document_title: str | None, + document_text: str, + document_url: str, +) -> RelevanceAssessment: + evidence = { + "company_name": company_name, + "company_aliases": company_aliases, + "monitoring_focus": monitoring_focus, + "document_title": document_title, + "document_url": document_url, + "document_text": document_text[:4000], + } + user_prompt = build_user_prompt( + "Assess whether this document is about the target company and matches its monitoring focus.", + evidence, + ) + return await llm.generate_structured(SYSTEM_PROMPT, user_prompt, RelevanceAssessment) diff --git a/apps/api/app/prompts/report_generation.py b/apps/api/app/prompts/report_generation.py new file mode 100644 index 0000000..f34b13f --- /dev/null +++ b/apps/api/app/prompts/report_generation.py @@ -0,0 +1,135 @@ +"""Task D: report generation. Turns accumulated evidence (source documents + +detected changes) for a company into the structured CI report (spec section +17). The LLM only ever sees evidence this pipeline collected - it is +explicitly instructed not to introduce outside knowledge, and every +non-trivial claim must carry a confidence label and evidence references. +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + +from app.analysis.llm.base import LLMProvider +from app.prompts.base import build_user_prompt +from app.prompts.schemas import ConfidenceLabel, EvidenceRef, Finding + +SYSTEM_PROMPT = ( + "You are a competitive intelligence analyst producing a structured report about a " + "company, using ONLY the evidence provided - the company_profile block (discovered once " + "at onboarding from the company's real website and search results, not your own " + "knowledge), the company_enrichment block (when present - fetched once from a paid " + "third-party data provider at onboarding, also real evidence, not your own knowledge), " + "stored source documents, and previously detected changes. Never introduce " + "facts from outside knowledge, even if you recognize the company - if it isn't in the " + "evidence block, it doesn't go in the report. The company_profile and company_enrichment " + "fields ARE real evidence and should ground company_overview/market_positioning/" + "financial_signals/leadership_changes/products_and_services/competitor_comparison/" + "customer_sentiment/etc even when source_documents and detected_changes are sparse or " + "empty - do not say 'insufficient evidence' for a field company_profile or " + "company_enrichment already answers. Every finding must be " + "traceable to the evidence given and must carry an honest confidence label: confirmed " + "(the source states it directly), strongly_indicated, likely, possible, unconfirmed, or " + "insufficient_evidence. When evidence is thin or missing for a section, say so explicitly " + "in that section rather than inventing content. Distinguish clearly between what a source " + "states and what you are inferring." +) + + +class InferredProject(BaseModel): + project_name: str + status: ConfidenceLabel + summary: str + confidence: float = Field(ge=0.0, le=1.0) + evidence: list[EvidenceRef] = Field(default_factory=list) + signal_types: list[str] = Field(default_factory=list) + alternative_explanations: list[str] = Field(default_factory=list) + + +class SwotAnalysis(BaseModel): + strengths: list[str] = Field(default_factory=list) + weaknesses: list[str] = Field(default_factory=list) + opportunities: list[str] = Field(default_factory=list) + threats: list[str] = Field(default_factory=list) + + +class ReportContent(BaseModel): + executive_summary: str + company_overview: str + products_and_services: list[Finding] = Field(default_factory=list) + market_positioning: str + recent_developments: list[Finding] = Field(default_factory=list) + strategic_initiatives: list[Finding] = Field(default_factory=list) + key_inferred_projects: list[InferredProject] = Field(default_factory=list) + leadership_changes: list[Finding] = Field(default_factory=list) + hiring_signals: list[Finding] = Field(default_factory=list) + technology_signals: list[Finding] = Field(default_factory=list) + patent_signals: list[Finding] = Field(default_factory=list) + manufacturing_and_expansion_signals: list[Finding] = Field(default_factory=list) + partnerships_and_acquisitions: list[Finding] = Field(default_factory=list) + financial_signals: list[Finding] = Field(default_factory=list) + regulatory_and_legal_signals: list[Finding] = Field(default_factory=list) + customer_sentiment: str + competitor_comparison: str + swot: SwotAnalysis + risks: list[str] = Field(default_factory=list) + opportunities: list[str] = Field(default_factory=list) + unknowns_and_missing_data: list[str] = Field(default_factory=list) + monitoring_recommendations: list[str] = Field(default_factory=list) + methodology: str + limitations: str + + +async def generate_report( + llm: LLMProvider, + *, + company_name: str, + company_aliases: list[str], + competitors: list[str], + monitoring_focus: str | None, + industry: str | None, + documents: list[dict], + detected_changes: list[dict], + sources_failed: list[str], + description: str | None = None, + official_website: str | None = None, + headquarters: str | None = None, + country: str | None = None, + region: str | None = None, + public_identifiers: dict[str, str] | None = None, + enrichment: dict | None = None, +) -> ReportContent: + evidence = { + # Discovered once at onboarding (app/services/discovery_service.py) + # from the company's real website + search results - genuine + # evidence, not the LLM's own background knowledge, and the only + # evidence available before any monitoring run has collected + # source_documents/detected_changes. + "company_profile": { + "name": company_name, + "description": description, + "official_website": official_website, + "aliases": company_aliases, + "competitors": competitors, + "monitoring_focus": monitoring_focus, + "industry": industry, + "headquarters": headquarters, + "country": country, + "region": region, + "public_identifiers": public_identifiers or {}, + }, + # Fetched once, at onboarding, from a paid third-party provider + # (NinjaPear) - see app/services/enrichment_service.py. Absent for + # any company created before this feature existed, or whose + # enrichment never completed successfully - never fabricated. + "company_enrichment": enrichment or {}, + "source_documents": documents, + "detected_changes": detected_changes, + "sources_that_failed_to_collect": sources_failed, + } + user_prompt = build_user_prompt( + "Produce a full structured competitive intelligence report from this evidence. " + "If a section has no supporting evidence, say so explicitly instead of leaving it " + "generic or inventing content.", + evidence, + ) + return await llm.generate_structured(SYSTEM_PROMPT, user_prompt, ReportContent) diff --git a/apps/api/app/prompts/schemas.py b/apps/api/app/prompts/schemas.py new file mode 100644 index 0000000..96ee837 --- /dev/null +++ b/apps/api/app/prompts/schemas.py @@ -0,0 +1,35 @@ +"""Shared building blocks for analysis-task response schemas. Every finding +that claims something happened carries an explicit confidence label from +this set - never presented as bare fact (spec section 16 / "Evidence and +Anti-Hallucination Requirements").""" + +from __future__ import annotations + +from enum import StrEnum + +from pydantic import BaseModel, Field + + +class ConfidenceLabel(StrEnum): + CONFIRMED = "confirmed" + STRONGLY_INDICATED = "strongly_indicated" + LIKELY = "likely" + POSSIBLE = "possible" + UNCONFIRMED = "unconfirmed" + INSUFFICIENT_EVIDENCE = "insufficient_evidence" + + +class EvidenceRef(BaseModel): + source_document_id: str | None = None + detected_change_id: str | None = None + url: str | None = None + description: str = Field(description="What this piece of evidence shows, in one sentence") + + +class Finding(BaseModel): + headline: str + summary: str + evidence: list[EvidenceRef] = Field(default_factory=list) + confidence: ConfidenceLabel = ConfidenceLabel.UNCONFIRMED + category: str | None = None + date: str | None = None diff --git a/apps/api/app/prompts/synthesis.py b/apps/api/app/prompts/synthesis.py new file mode 100644 index 0000000..e56ff59 --- /dev/null +++ b/apps/api/app/prompts/synthesis.py @@ -0,0 +1,54 @@ +"""Task C: cross-source synthesis. Combines related signals from multiple +documents/changes into a conclusion (e.g. "hiring battery engineers" + +"filed a battery patent" + "announced a facility expansion" = "possible +battery manufacturing initiative"), always with alternative explanations and +an explicit accounting of what's still unknown. +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + +from app.analysis.llm.base import LLMProvider +from app.prompts.base import build_user_prompt + +SYSTEM_PROMPT = ( + "You are a competitive intelligence analyst. Given a set of independently-extracted " + "signals about a company, identify conclusions that multiple signals point toward " + "together. Only draw a conclusion when the evidence actually supports it - state your " + "confidence honestly, list plausible alternative explanations, and note what evidence " + "would be needed to be more certain. Do not synthesize a conclusion from a single signal." +) + + +class SynthesizedConclusion(BaseModel): + conclusion: str + evidence_summary: list[str] = Field(default_factory=list) + source_count: int = Field(ge=0) + confidence: float = Field(ge=0.0, le=1.0) + alternative_explanations: list[str] = Field(default_factory=list) + missing_information: list[str] = Field(default_factory=list) + + +class SynthesisResult(BaseModel): + conclusions: list[SynthesizedConclusion] = Field(default_factory=list) + + +async def synthesize_signals( + llm: LLMProvider, + *, + company_name: str, + monitoring_focus: str | None, + signals: list[dict], +) -> SynthesisResult: + evidence = { + "company_name": company_name, + "monitoring_focus": monitoring_focus, + "signals": signals, + } + user_prompt = build_user_prompt( + "Identify conclusions supported by two or more of these signals together. Do not " + "synthesize from a single signal alone.", + evidence, + ) + return await llm.generate_structured(SYSTEM_PROMPT, user_prompt, SynthesisResult) diff --git a/apps/api/app/repositories/__init__.py b/apps/api/app/repositories/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/app/repositories/alert_repository.py b/apps/api/app/repositories/alert_repository.py new file mode 100644 index 0000000..45f47b0 --- /dev/null +++ b/apps/api/app/repositories/alert_repository.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import uuid +from datetime import datetime + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.alert import Alert +from app.models.enums import SeverityLevel + + +class AlertRepository: + def __init__(self, db: AsyncSession) -> None: + self.db = db + + async def get(self, alert_id: uuid.UUID) -> Alert | None: + return await self.db.get(Alert, alert_id) + + async def get_for_user(self, alert_id: uuid.UUID, user_id: uuid.UUID) -> Alert | None: + result = await self.db.execute( + select(Alert).where(Alert.id == alert_id, Alert.user_id == user_id) + ) + return result.scalar_one_or_none() + + async def list_for_user( + self, + user_id: uuid.UUID, + *, + company_id: uuid.UUID | None = None, + severity: SeverityLevel | None = None, + read: bool | None = None, + resolved: bool | None = None, + since: datetime | None = None, + limit: int = 200, + ) -> list[Alert]: + stmt = select(Alert).where(Alert.user_id == user_id) + if company_id is not None: + stmt = stmt.where(Alert.company_id == company_id) + if severity is not None: + stmt = stmt.where(Alert.severity == severity) + if read is not None: + stmt = stmt.where(Alert.read == read) + if resolved is not None: + stmt = stmt.where(Alert.resolved == resolved) + if since is not None: + stmt = stmt.where(Alert.created_at >= since) + stmt = stmt.order_by(Alert.created_at.desc()).limit(limit) + result = await self.db.execute(stmt) + return list(result.scalars().all()) + + async def create(self, alert: Alert) -> Alert: + self.db.add(alert) + await self.db.flush() + return alert diff --git a/apps/api/app/repositories/company_enrichment_repository.py b/apps/api/app/repositories/company_enrichment_repository.py new file mode 100644 index 0000000..96a2591 --- /dev/null +++ b/apps/api/app/repositories/company_enrichment_repository.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.company_enrichment import CompanyEnrichment +from app.models.enums import EnrichmentStatus + + +class CompanyEnrichmentRepository: + def __init__(self, db: AsyncSession) -> None: + self.db = db + + async def get_for_company(self, company_id: uuid.UUID) -> CompanyEnrichment | None: + result = await self.db.execute( + select(CompanyEnrichment).where(CompanyEnrichment.company_id == company_id) + ) + return result.scalar_one_or_none() + + async def upsert( + self, + company_id: uuid.UUID, + *, + status: EnrichmentStatus, + data: dict[str, Any], + errors: dict[str, str], + credits_spent: int | None, + fetched_at: datetime, + ) -> CompanyEnrichment: + """One row per company (unique FK) - the second-ever call for a + company would only happen via a manual re-run, never automatically + (enrichment fires once, at onboarding - see enrichment_service).""" + existing = await self.get_for_company(company_id) + if existing is not None: + existing.status = status + existing.data = data + existing.errors = errors + existing.credits_spent = credits_spent + existing.fetched_at = fetched_at + await self.db.flush() + return existing + + enrichment = CompanyEnrichment( + company_id=company_id, + status=status, + data=data, + errors=errors, + credits_spent=credits_spent, + fetched_at=fetched_at, + ) + self.db.add(enrichment) + await self.db.flush() + return enrichment diff --git a/apps/api/app/repositories/company_repository.py b/apps/api/app/repositories/company_repository.py new file mode 100644 index 0000000..9547bc6 --- /dev/null +++ b/apps/api/app/repositories/company_repository.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import uuid +from datetime import datetime + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from app.models.company import Company, CompanyAlias, Competitor +from app.models.monitor_configuration import MonitorConfiguration + + +def _with_relations(stmt): + return stmt.options( + selectinload(Company.aliases), + selectinload(Company.competitors), + selectinload(Company.monitor_configuration), + selectinload(Company.enrichment), + # Loaded eagerly so ORM-level cascade-delete (SQLite doesn't enforce + # FK ON DELETE CASCADE without a pragma this app doesn't set) can + # actually see the children to remove when a Company is deleted. + selectinload(Company.notification_links), + ) + + +class CompanyRepository: + def __init__(self, db: AsyncSession) -> None: + self.db = db + + async def get_for_user(self, company_id: uuid.UUID, user_id: uuid.UUID) -> Company | None: + stmt = _with_relations( + select(Company).where(Company.id == company_id, Company.user_id == user_id) + ) + result = await self.db.execute(stmt) + return result.scalar_one_or_none() + + async def get_by_id(self, company_id: uuid.UUID) -> Company | None: + """Unscoped lookup for trusted internal callers (Celery tasks) that + already have the company_id from a source they control - not for + anything reachable from an HTTP request without an ownership check.""" + stmt = _with_relations(select(Company).where(Company.id == company_id)) + result = await self.db.execute(stmt) + return result.scalar_one_or_none() + + async def list_for_user(self, user_id: uuid.UUID) -> list[Company]: + stmt = _with_relations( + select(Company).where(Company.user_id == user_id).order_by(Company.created_at.desc()) + ) + result = await self.db.execute(stmt) + return list(result.scalars().all()) + + async def count_for_user(self, user_id: uuid.UUID) -> int: + result = await self.db.execute( + select(func.count()).select_from(Company).where(Company.user_id == user_id) + ) + return int(result.scalar_one()) + + async def slug_exists_for_user(self, user_id: uuid.UUID, slug: str) -> bool: + result = await self.db.execute( + select(func.count()) + .select_from(Company) + .where(Company.user_id == user_id, Company.slug == slug) + ) + return int(result.scalar_one()) > 0 + + async def name_exists_for_user(self, user_id: uuid.UUID, name: str) -> bool: + result = await self.db.execute( + select(func.count()) + .select_from(Company) + .where(Company.user_id == user_id, func.lower(Company.name) == name.lower()) + ) + return int(result.scalar_one()) > 0 + + async def create( + self, + *, + user_id: uuid.UUID, + name: str, + slug: str, + official_website: str | None, + description: str | None, + monitoring_focus: str | None, + industry: str | None, + country: str | None, + region: str | None, + headquarters: str | None = None, + public_identifiers: dict[str, str] | None = None, + alias_names: list[str], + competitor_names: list[str], + ) -> Company: + company = Company( + user_id=user_id, + name=name, + slug=slug, + official_website=official_website, + description=description, + monitoring_focus=monitoring_focus, + industry=industry, + country=country, + region=region, + headquarters=headquarters, + public_identifiers=public_identifiers or {}, + ) + company.aliases = [CompanyAlias(alias=a) for a in alias_names if a.strip()] + company.competitors = [Competitor(name=c) for c in competitor_names if c.strip()] + self.db.add(company) + await self.db.flush() + return company + + async def delete(self, company: Company) -> None: + await self.db.delete(company) + await self.db.flush() + + async def replace_aliases(self, company: Company, alias_names: list[str]) -> None: + company.aliases = [CompanyAlias(alias=a) for a in alias_names if a.strip()] + + async def replace_competitors(self, company: Company, competitor_names: list[str]) -> None: + company.competitors = [Competitor(name=c) for c in competitor_names if c.strip()] + + +class MonitorConfigurationRepository: + def __init__(self, db: AsyncSession) -> None: + self.db = db + + async def create_default( + self, + *, + company_id: uuid.UUID, + frequency_type, + interval_minutes: int | None, + cron_expression: str | None, + timezone: str, + severity_threshold, + next_run, + ) -> MonitorConfiguration: + config = MonitorConfiguration( + company_id=company_id, + frequency_type=frequency_type, + interval_minutes=interval_minutes, + cron_expression=cron_expression, + timezone=timezone, + severity_threshold=severity_threshold, + next_run=next_run, + ) + self.db.add(config) + await self.db.flush() + return config + + async def list_due(self, now: datetime) -> list[MonitorConfiguration]: + """Enabled schedules whose next_run has arrived - what Celery Beat's + sync_schedules task polls instead of requiring a static per-company + beat_schedule entry (see ARCHITECTURE.md).""" + result = await self.db.execute( + select(MonitorConfiguration).where( + MonitorConfiguration.enabled.is_(True), + MonitorConfiguration.next_run.is_not(None), + MonitorConfiguration.next_run <= now, + ) + ) + return list(result.scalars().all()) + + async def list_enabled(self) -> list[MonitorConfiguration]: + """Every enabled schedule regardless of next_run - since a company + can now also become due purely via a per-source frequency override + even when its own default next_run isn't due yet (see + SourceRepository.list_due_for_company), the scheduler needs this + broader set to check, not just the ones already due on the + company-level clock.""" + result = await self.db.execute( + select(MonitorConfiguration).where(MonitorConfiguration.enabled.is_(True)) + ) + return list(result.scalars().all()) diff --git a/apps/api/app/repositories/detected_change_repository.py b/apps/api/app/repositories/detected_change_repository.py new file mode 100644 index 0000000..d802c42 --- /dev/null +++ b/apps/api/app/repositories/detected_change_repository.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.detected_change import DetectedChange +from app.models.enums import ChangeType + + +class DetectedChangeRepository: + def __init__(self, db: AsyncSession) -> None: + self.db = db + + async def get_recent_for_source_and_type( + self, source_id: uuid.UUID, change_type: ChangeType, since: datetime + ) -> DetectedChange | None: + """Most recent DetectedChange of this type for this source within + the cooldown window - used both to flag repeats (lower significance) + and to suppress exact duplicates (see change_detection_service.py).""" + result = await self.db.execute( + select(DetectedChange) + .where( + DetectedChange.source_id == source_id, + DetectedChange.change_type == change_type, + DetectedChange.created_at >= since, + ) + .order_by(DetectedChange.created_at.desc()) + .limit(1) + ) + return result.scalar_one_or_none() + + async def create(self, **kwargs: Any) -> DetectedChange: + change = DetectedChange(**kwargs) + self.db.add(change) + await self.db.flush() + return change + + async def list_for_company( + self, company_id: uuid.UUID, limit: int = 100 + ) -> list[DetectedChange]: + result = await self.db.execute( + select(DetectedChange) + .where(DetectedChange.company_id == company_id) + .order_by(DetectedChange.created_at.desc()) + .limit(limit) + ) + return list(result.scalars().all()) diff --git a/apps/api/app/repositories/email_code_repository.py b/apps/api/app/repositories/email_code_repository.py new file mode 100644 index 0000000..ec86420 --- /dev/null +++ b/apps/api/app/repositories/email_code_repository.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import uuid +from datetime import UTC, datetime + +from sqlalchemy import select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.base import ensure_aware_utc +from app.models.email_code import EmailCode +from app.models.enums import EmailCodePurpose + + +class EmailCodeRepository: + def __init__(self, db: AsyncSession) -> None: + self.db = db + + async def create( + self, *, user_id: uuid.UUID, purpose: EmailCodePurpose, code_hash: str, expires_at: datetime + ) -> EmailCode: + record = EmailCode( + user_id=user_id, purpose=purpose, code_hash=code_hash, expires_at=expires_at + ) + self.db.add(record) + await self.db.flush() + return record + + async def get_latest_valid( + self, user_id: uuid.UUID, purpose: EmailCodePurpose, code_hash: str + ) -> EmailCode | None: + """Most recent unused, unexpired code for this user/purpose whose + hash matches. `invalidate_unused` is what actually guarantees only + the most recently issued code can ever satisfy this - this method + alone doesn't enforce that.""" + result = await self.db.execute( + select(EmailCode) + .where( + EmailCode.user_id == user_id, + EmailCode.purpose == purpose, + EmailCode.code_hash == code_hash, + EmailCode.used_at.is_(None), + ) + .order_by(EmailCode.created_at.desc()) + ) + for record in result.scalars().all(): + if ensure_aware_utc(record.expires_at) > datetime.now(UTC): + return record + return None + + async def mark_used(self, record: EmailCode) -> None: + record.used_at = datetime.now(UTC) + await self.db.flush() + + async def invalidate_unused(self, user_id: uuid.UUID, purpose: EmailCodePurpose) -> None: + """Called right before issuing a fresh code - a resend must fully + supersede every prior unused code for this purpose, not just make + them harder to guess. Without this, an old code (e.g. still sitting + in an old email) stays valid until it naturally expires, even after + the user has explicitly asked for a new one.""" + await self.db.execute( + update(EmailCode) + .where( + EmailCode.user_id == user_id, + EmailCode.purpose == purpose, + EmailCode.used_at.is_(None), + ) + .values(used_at=datetime.now(UTC)) + ) + await self.db.flush() diff --git a/apps/api/app/repositories/ip_throttle_repository.py b/apps/api/app/repositories/ip_throttle_repository.py new file mode 100644 index 0000000..b3bd2f6 --- /dev/null +++ b/apps/api/app/repositories/ip_throttle_repository.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import delete, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.enums import ThrottleAction +from app.models.ip_ban import IpBan +from app.models.ip_throttle_state import IpThrottleState + + +class IpThrottleRepository: + def __init__(self, db: AsyncSession) -> None: + self.db = db + + async def get_state(self, ip_address: str, action: ThrottleAction) -> IpThrottleState | None: + result = await self.db.execute( + select(IpThrottleState).where( + IpThrottleState.ip_address == ip_address, IpThrottleState.action == action + ) + ) + return result.scalar_one_or_none() + + async def get_or_create_state(self, ip_address: str, action: ThrottleAction) -> IpThrottleState: + state = await self.get_state(ip_address, action) + if state is not None: + return state + state = IpThrottleState(ip_address=ip_address, action=action) + self.db.add(state) + await self.db.flush() + return state + + async def get_ban(self, ip_address: str) -> IpBan | None: + result = await self.db.execute(select(IpBan).where(IpBan.ip_address == ip_address)) + return result.scalar_one_or_none() + + async def create_ban(self, ip_address: str, reason: str, banned_at: datetime) -> IpBan: + ban = IpBan(ip_address=ip_address, reason=reason, banned_at=banned_at) + self.db.add(ban) + await self.db.flush() + return ban + + async def list_bans(self) -> list[IpBan]: + result = await self.db.execute(select(IpBan).order_by(IpBan.banned_at.desc())) + return list(result.scalars().all()) + + async def clear_ban_and_state(self, ip_address: str) -> bool: + """Full pardon for an admin-approved unban - removes the ban record + and resets every throttle-state row for this IP to a clean slate + (not just lifting the terminal ban while leaving them one offense + from another).""" + ban_result = await self.db.execute(delete(IpBan).where(IpBan.ip_address == ip_address)) + result = await self.db.execute( + select(IpThrottleState).where(IpThrottleState.ip_address == ip_address) + ) + states = result.scalars().all() + for state in states: + state.attempt_count = 0 + state.next_allowed_at = None + state.timeout_until = None + state.offense_count = 0 + await self.db.flush() + return ban_result.rowcount > 0 or len(states) > 0 diff --git a/apps/api/app/repositories/monitoring_run_repository.py b/apps/api/app/repositories/monitoring_run_repository.py new file mode 100644 index 0000000..1ac8c8d --- /dev/null +++ b/apps/api/app/repositories/monitoring_run_repository.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +import uuid +from datetime import UTC, datetime + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.enums import MonitoringRunStatus, MonitoringRunTrigger +from app.models.monitoring_run import MonitoringRun + +_ACTIVE_STATUSES = (MonitoringRunStatus.QUEUED, MonitoringRunStatus.RUNNING) + + +class MonitoringRunRepository: + def __init__(self, db: AsyncSession) -> None: + self.db = db + + async def get(self, run_id: uuid.UUID) -> MonitoringRun | None: + return await self.db.get(MonitoringRun, run_id) + + async def get_for_company( + self, run_id: uuid.UUID, company_id: uuid.UUID + ) -> MonitoringRun | None: + result = await self.db.execute( + select(MonitoringRun).where( + MonitoringRun.id == run_id, MonitoringRun.company_id == company_id + ) + ) + return result.scalar_one_or_none() + + async def list_for_company(self, company_id: uuid.UUID, limit: int = 50) -> list[MonitoringRun]: + result = await self.db.execute( + select(MonitoringRun) + .where(MonitoringRun.company_id == company_id) + .order_by(MonitoringRun.created_at.desc()) + .limit(limit) + ) + return list(result.scalars().all()) + + async def get_active_for_company(self, company_id: uuid.UUID) -> MonitoringRun | None: + """The queued/running run for this company, if any - used both to + make "run now" idempotent and to stop the scheduler from double + enqueuing a company that's still mid-run.""" + result = await self.db.execute( + select(MonitoringRun) + .where( + MonitoringRun.company_id == company_id, + MonitoringRun.status.in_(_ACTIVE_STATUSES), + ) + .order_by(MonitoringRun.created_at.desc()) + .limit(1) + ) + return result.scalar_one_or_none() + + async def create( + self, *, company_id: uuid.UUID, trigger_type: MonitoringRunTrigger + ) -> MonitoringRun: + run = MonitoringRun( + company_id=company_id, + trigger_type=trigger_type, + status=MonitoringRunStatus.QUEUED, + ) + self.db.add(run) + await self.db.flush() + return run + + async def set_worker_task_id(self, run: MonitoringRun, task_id: str | None) -> None: + run.worker_task_id = task_id + await self.db.flush() + + async def mark_running(self, run: MonitoringRun) -> None: + run.status = MonitoringRunStatus.RUNNING + run.started_at = datetime.now(UTC) + await self.db.flush() + + async def update_progress( + self, + run: MonitoringRun, + *, + sources_attempted: int, + sources_successful: int, + sources_failed: int, + items_collected: int, + changes_detected: int | None = None, + ) -> None: + run.sources_attempted = sources_attempted + run.sources_successful = sources_successful + run.sources_failed = sources_failed + run.items_collected = items_collected + if changes_detected is not None: + run.changes_detected = changes_detected + await self.db.commit() + + async def mark_finished( + self, run: MonitoringRun, *, status: MonitoringRunStatus, error_summary: str | None + ) -> None: + run.status = status + run.error_summary = error_summary + run.completed_at = datetime.now(UTC) + await self.db.commit() + + async def count_for_company(self, company_id: uuid.UUID) -> int: + result = await self.db.execute( + select(func.count()) + .select_from(MonitoringRun) + .where(MonitoringRun.company_id == company_id) + ) + return int(result.scalar_one()) + + async def count_manual_since(self, company_id: uuid.UUID, since: datetime) -> int: + """Manual (user-triggered) runs for this company since `since` - + what enforces MAX_MANUAL_RUNS_PER_DAY (scheduled runs don't count + against it).""" + result = await self.db.execute( + select(func.count()) + .select_from(MonitoringRun) + .where( + MonitoringRun.company_id == company_id, + MonitoringRun.trigger_type == MonitoringRunTrigger.MANUAL, + MonitoringRun.created_at >= since, + ) + ) + return int(result.scalar_one()) diff --git a/apps/api/app/repositories/notification_delivery_repository.py b/apps/api/app/repositories/notification_delivery_repository.py new file mode 100644 index 0000000..7f32d52 --- /dev/null +++ b/apps/api/app/repositories/notification_delivery_repository.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +import uuid + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.notification_delivery import NotificationDelivery + + +class NotificationDeliveryRepository: + def __init__(self, db: AsyncSession) -> None: + self.db = db + + async def create(self, delivery: NotificationDelivery) -> NotificationDelivery: + self.db.add(delivery) + await self.db.flush() + return delivery + + async def list_for_alert(self, alert_id: uuid.UUID) -> list[NotificationDelivery]: + result = await self.db.execute( + select(NotificationDelivery).where(NotificationDelivery.alert_id == alert_id) + ) + return list(result.scalars().all()) diff --git a/apps/api/app/repositories/notification_destination_repository.py b/apps/api/app/repositories/notification_destination_repository.py new file mode 100644 index 0000000..6ba6097 --- /dev/null +++ b/apps/api/app/repositories/notification_destination_repository.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import uuid + +from sqlalchemy import delete, func, select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from app.models.enums import NotificationType, SeverityLevel +from app.models.notification_destination import ( + NotificationDestination, + NotificationDestinationCompany, +) + + +def _with_companies(stmt): + return stmt.options( + selectinload(NotificationDestination.company_links).selectinload( + NotificationDestinationCompany.company + ) + ) + + +class NotificationDestinationRepository: + def __init__(self, db: AsyncSession) -> None: + self.db = db + + async def list_for_user(self, user_id: uuid.UUID) -> list[NotificationDestination]: + result = await self.db.execute( + _with_companies( + select(NotificationDestination) + .where(NotificationDestination.user_id == user_id) + .order_by(NotificationDestination.created_at.desc()) + ) + ) + return list(result.scalars().all()) + + async def list_for_company(self, company_id: uuid.UUID) -> list[NotificationDestination]: + """Only destinations actually linked to this company - what alert + dispatch notifies, as opposed to list_for_user's "everything this + user owns" (used by the Settings page).""" + result = await self.db.execute( + select(NotificationDestination) + .join( + NotificationDestinationCompany, + NotificationDestinationCompany.destination_id == NotificationDestination.id, + ) + .where(NotificationDestinationCompany.company_id == company_id) + ) + return list(result.scalars().all()) + + async def get_for_user( + self, destination_id: uuid.UUID, user_id: uuid.UUID + ) -> NotificationDestination | None: + result = await self.db.execute( + _with_companies( + select(NotificationDestination).where( + NotificationDestination.id == destination_id, + NotificationDestination.user_id == user_id, + ) + ) + ) + return result.scalar_one_or_none() + + async def find_by_value( + self, user_id: uuid.UUID, type: NotificationType, destination_value: str + ) -> NotificationDestination | None: + """Case-insensitive for email (RFC-technically case-sensitive local + parts exist, but no real provider treats them that way and users + retype casing inconsistently), exact for everything else.""" + normalized = destination_value.strip() + stmt = select(NotificationDestination).where( + NotificationDestination.user_id == user_id, NotificationDestination.type == type + ) + if type == NotificationType.EMAIL: + stmt = stmt.where( + func.lower(NotificationDestination.destination_value) == normalized.lower() + ) + else: + stmt = stmt.where(NotificationDestination.destination_value == normalized) + result = await self.db.execute(_with_companies(stmt)) + return result.scalar_one_or_none() + + async def create( + self, + *, + user_id: uuid.UUID, + type: NotificationType, + destination_value: str, + minimum_severity: SeverityLevel, + enabled: bool, + ) -> NotificationDestination: + destination = NotificationDestination( + user_id=user_id, + type=type, + destination_value=destination_value, + minimum_severity=minimum_severity, + enabled=enabled, + ) + self.db.add(destination) + await self.db.flush() + return destination + + async def link_company(self, destination_id: uuid.UUID, company_id: uuid.UUID) -> None: + exists = await self.db.execute( + select(func.count()) + .select_from(NotificationDestinationCompany) + .where( + NotificationDestinationCompany.destination_id == destination_id, + NotificationDestinationCompany.company_id == company_id, + ) + ) + if int(exists.scalar_one()) > 0: + return + self.db.add( + NotificationDestinationCompany(destination_id=destination_id, company_id=company_id) + ) + await self.db.flush() + + async def unlink_company(self, destination_id: uuid.UUID, company_id: uuid.UUID) -> None: + await self.db.execute( + delete(NotificationDestinationCompany).where( + NotificationDestinationCompany.destination_id == destination_id, + NotificationDestinationCompany.company_id == company_id, + ) + ) + await self.db.flush() + + async def company_link_count(self, destination_id: uuid.UUID) -> int: + result = await self.db.execute( + select(func.count()) + .select_from(NotificationDestinationCompany) + .where(NotificationDestinationCompany.destination_id == destination_id) + ) + return int(result.scalar_one()) + + async def delete_orphaned_for_user(self, user_id: uuid.UUID) -> None: + """Deletes any of this user's destinations that ended up linked to + zero companies - called after a company delete, since that cascades + the join rows for it but leaves the destination row itself behind + even when it was the destination's only remaining link.""" + destinations = await self.list_for_user(user_id) + for destination in destinations: + if await self.company_link_count(destination.id) == 0: + await self.delete(destination) + + async def delete(self, destination: NotificationDestination) -> None: + await self.db.delete(destination) + await self.db.flush() diff --git a/apps/api/app/repositories/password_history_repository.py b/apps/api/app/repositories/password_history_repository.py new file mode 100644 index 0000000..e3ced76 --- /dev/null +++ b/apps/api/app/repositories/password_history_repository.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import uuid + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.password_history import PasswordHistoryEntry + + +class PasswordHistoryRepository: + def __init__(self, db: AsyncSession) -> None: + self.db = db + + async def list_hashes_for_user(self, user_id: uuid.UUID) -> list[str]: + result = await self.db.execute( + select(PasswordHistoryEntry.password_hash).where( + PasswordHistoryEntry.user_id == user_id + ) + ) + return list(result.scalars().all()) + + async def add(self, *, user_id: uuid.UUID, password_hash: str) -> None: + self.db.add(PasswordHistoryEntry(user_id=user_id, password_hash=password_hash)) + await self.db.flush() diff --git a/apps/api/app/repositories/refresh_token_repository.py b/apps/api/app/repositories/refresh_token_repository.py new file mode 100644 index 0000000..bbdaa6d --- /dev/null +++ b/apps/api/app/repositories/refresh_token_repository.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import uuid +from datetime import UTC, datetime + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.base import ensure_aware_utc +from app.models.refresh_token import RefreshToken + + +class RefreshTokenRepository: + def __init__(self, db: AsyncSession) -> None: + self.db = db + + async def create( + self, *, user_id: uuid.UUID, token_hash: str, expires_at: datetime + ) -> RefreshToken: + record = RefreshToken(user_id=user_id, token_hash=token_hash, expires_at=expires_at) + self.db.add(record) + await self.db.flush() + return record + + async def get_valid_by_hash(self, token_hash: str) -> RefreshToken | None: + result = await self.db.execute( + select(RefreshToken).where(RefreshToken.token_hash == token_hash) + ) + record = result.scalar_one_or_none() + if record is None: + return None + if record.revoked_at is not None: + return None + if ensure_aware_utc(record.expires_at) < datetime.now(UTC): + return None + return record + + async def revoke(self, record: RefreshToken) -> None: + record.revoked_at = datetime.now(UTC) + await self.db.flush() + + async def revoke_all_for_user(self, user_id: uuid.UUID) -> None: + """Invalidates every active session for this user - used after a + password reset, since an attacker who had a valid refresh token + shouldn't stay logged in past the password change that locked them + out going forward.""" + result = await self.db.execute( + select(RefreshToken).where( + RefreshToken.user_id == user_id, RefreshToken.revoked_at.is_(None) + ) + ) + now = datetime.now(UTC) + for record in result.scalars().all(): + record.revoked_at = now + await self.db.flush() diff --git a/apps/api/app/repositories/report_repository.py b/apps/api/app/repositories/report_repository.py new file mode 100644 index 0000000..e5471f8 --- /dev/null +++ b/apps/api/app/repositories/report_repository.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import uuid + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.report import Report + + +class ReportRepository: + def __init__(self, db: AsyncSession) -> None: + self.db = db + + async def get(self, report_id: uuid.UUID) -> Report | None: + return await self.db.get(Report, report_id) + + async def get_for_company(self, report_id: uuid.UUID, company_id: uuid.UUID) -> Report | None: + result = await self.db.execute( + select(Report).where(Report.id == report_id, Report.company_id == company_id) + ) + return result.scalar_one_or_none() + + async def list_for_company(self, company_id: uuid.UUID, limit: int = 50) -> list[Report]: + result = await self.db.execute( + select(Report) + .where(Report.company_id == company_id) + .order_by(Report.created_at.desc()) + .limit(limit) + ) + return list(result.scalars().all()) + + async def latest_for_company(self, company_id: uuid.UUID) -> Report | None: + result = await self.db.execute( + select(Report) + .where(Report.company_id == company_id) + .order_by(Report.created_at.desc()) + .limit(1) + ) + return result.scalar_one_or_none() + + async def count_for_company(self, company_id: uuid.UUID) -> int: + result = await self.db.execute( + select(func.count()).select_from(Report).where(Report.company_id == company_id) + ) + return int(result.scalar_one()) diff --git a/apps/api/app/repositories/source_repository.py b/apps/api/app/repositories/source_repository.py new file mode 100644 index 0000000..dabebd0 --- /dev/null +++ b/apps/api/app/repositories/source_repository.py @@ -0,0 +1,225 @@ +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any + +from sqlalchemy import delete, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.base import ensure_aware_utc +from app.models.company import Company +from app.models.enums import SourceStatus, SourceType +from app.models.snapshot import Snapshot +from app.models.source import Source +from app.models.source_document import SourceDocument + + +def _is_due(value: datetime | None, now: datetime) -> bool: + # SQLite silently drops tzinfo on read-back (Postgres does not) - any + # value read from the DB must go through ensure_aware_utc before being + # compared against an aware `now` in Python, or this comparison would + # raise on SQLite while working fine on Postgres. See db/base.py. + return value is not None and ensure_aware_utc(value) <= now + + +def _is_source_due(source: Source, now: datetime, company_next_run: datetime | None) -> bool: + """A source with its own frequency override uses its own next_check + (due immediately if never computed yet); one with no override rides + the company's own next_run clock instead.""" + if source.frequency_type is not None: + return source.next_check is None or _is_due(source.next_check, now) + return _is_due(company_next_run, now) + + +class SourceRepository: + def __init__(self, db: AsyncSession) -> None: + self.db = db + + async def list_for_company(self, company_id: uuid.UUID) -> list[Source]: + result = await self.db.execute( + select(Source).where(Source.company_id == company_id).order_by(Source.created_at) + ) + return list(result.scalars().all()) + + async def list_due_for_company( + self, company_id: uuid.UUID, now: datetime, company_next_run: datetime | None + ) -> list[Source]: + """Active sources that are due for a check right now. A source with + its own frequency override (frequency_type is not None) uses its + own next_check, and is treated as due if it's never been computed + yet (a brand-new override should get its first check immediately, + same as any brand-new source always has). A source with no + override rides the company's own next_run clock instead - this is + what keeps "no override configured" behaviorally identical to how + every source worked before per-source scheduling existed. Small + per-company table (sources per company is always small for this + app), so filtering in Python after one plain SELECT is simpler than + expressing the OR-with-NULL-fallback logic as SQL.""" + sources = await self._list_active_for_company(company_id) + return [s for s in sources if _is_source_due(s, now, company_next_run)] + + async def company_has_due_work( + self, company_id: uuid.UUID, now: datetime, company_next_run: datetime | None + ) -> bool: + """Whether the scheduler should enqueue a run for this company: a + company with zero sources yet has nothing to check per-source - it + stays gated purely by its own next_run (the company's first-ever + run, which is what triggers source discovery), same as before + per-source scheduling existed. A company with sources is due if any + active one is (see list_due_for_company).""" + sources = await self._list_active_for_company(company_id) + if not sources: + return _is_due(company_next_run, now) + return any(_is_source_due(s, now, company_next_run) for s in sources) + + async def _list_active_for_company(self, company_id: uuid.UUID) -> list[Source]: + result = await self.db.execute( + select(Source).where(Source.company_id == company_id, Source.active.is_(True)) + ) + return list(result.scalars().all()) + + async def get_for_company(self, source_id: uuid.UUID, company_id: uuid.UUID) -> Source | None: + result = await self.db.execute( + select(Source).where(Source.id == source_id, Source.company_id == company_id) + ) + return result.scalar_one_or_none() + + async def get_for_user(self, source_id: uuid.UUID, user_id: uuid.UUID) -> Source | None: + """Ownership-checked lookup that doesn't require the caller to + already know the company_id (matches the spec's `/sources/{id}` + routes, which aren't nested under `/companies/{company_id}`).""" + result = await self.db.execute( + select(Source) + .join(Company, Company.id == Source.company_id) + .where(Source.id == source_id, Company.user_id == user_id) + ) + return result.scalar_one_or_none() + + async def create( + self, + *, + company_id: uuid.UUID, + source_type: SourceType, + name: str, + base_url: str | None, + configuration_metadata: dict[str, Any] | None = None, + trust_score: float = 0.7, + ) -> Source: + source = Source( + company_id=company_id, + source_type=source_type, + name=name, + base_url=base_url, + configuration_metadata=configuration_metadata or {}, + trust_score=trust_score, + ) + self.db.add(source) + await self.db.flush() + return source + + async def delete(self, source: Source) -> None: + await self.db.delete(source) + await self.db.flush() + + async def mark_checked( + self, + source: Source, + *, + status: SourceStatus, + checked_at: datetime, + success: bool, + ) -> None: + source.status = status + source.last_checked = checked_at + if success: + source.last_successful_check = checked_at + source.failure_count = 0 + else: + source.failure_count += 1 + await self.db.flush() + + +class SourceDocumentRepository: + def __init__(self, db: AsyncSession) -> None: + self.db = db + + async def exists_with_hash(self, source_id: uuid.UUID, content_hash: str) -> bool: + result = await self.db.execute( + select(SourceDocument.id).where( + SourceDocument.source_id == source_id, + SourceDocument.content_hash == content_hash, + ) + ) + return result.scalar_one_or_none() is not None + + async def create(self, **kwargs: Any) -> SourceDocument: + document = SourceDocument(**kwargs) + self.db.add(document) + await self.db.flush() + return document + + async def latest_for_source( + self, source_id: uuid.UUID, limit: int = 50 + ) -> list[SourceDocument]: + result = await self.db.execute( + select(SourceDocument) + .where(SourceDocument.source_id == source_id) + .order_by(SourceDocument.retrieved_date.desc()) + .limit(limit) + ) + return list(result.scalars().all()) + + async def delete_older_than(self, cutoff: datetime) -> int: + """Data-retention purge target (DATA_RETENTION_DAYS). Only + SourceDocument is in scope - nothing else has a foreign key onto it + (see KNOWN_LIMITATIONS.md), so this can't cascade-delete a Snapshot, + DetectedChange, Alert, or Report a user might still want to see.""" + result = await self.db.execute( + delete(SourceDocument).where(SourceDocument.retrieved_date < cutoff) + ) + return result.rowcount or 0 + + +class SnapshotRepository: + def __init__(self, db: AsyncSession) -> None: + self.db = db + + async def list_for_company(self, company_id: uuid.UUID, limit: int = 50) -> list[Snapshot]: + result = await self.db.execute( + select(Snapshot) + .where(Snapshot.company_id == company_id) + .order_by(Snapshot.created_at.desc()) + .limit(limit) + ) + return list(result.scalars().all()) + + async def latest_for_source(self, source_id: uuid.UUID) -> Snapshot | None: + result = await self.db.execute( + select(Snapshot) + .where(Snapshot.source_id == source_id) + .order_by(Snapshot.created_at.desc()) + .limit(1) + ) + return result.scalar_one_or_none() + + async def create(self, **kwargs: Any) -> Snapshot: + snapshot = Snapshot(**kwargs) + self.db.add(snapshot) + await self.db.flush() + return snapshot + + async def get_previous(self, source_id: uuid.UUID, before: Snapshot) -> Snapshot | None: + """The snapshot immediately preceding `before` for this source - + what change detection diffs the new snapshot against.""" + result = await self.db.execute( + select(Snapshot) + .where( + Snapshot.source_id == source_id, + Snapshot.id != before.id, + Snapshot.created_at <= before.created_at, + ) + .order_by(Snapshot.created_at.desc()) + .limit(1) + ) + return result.scalar_one_or_none() diff --git a/apps/api/app/repositories/system_secret_repository.py b/apps/api/app/repositories/system_secret_repository.py new file mode 100644 index 0000000..ee7e2dc --- /dev/null +++ b/apps/api/app/repositories/system_secret_repository.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.enums import SystemSecretKey +from app.models.system_secret import SystemSecret + + +class SystemSecretRepository: + def __init__(self, db: AsyncSession) -> None: + self.db = db + + async def list_all(self) -> list[SystemSecret]: + result = await self.db.execute(select(SystemSecret)) + return list(result.scalars().all()) + + async def get(self, key: SystemSecretKey) -> SystemSecret | None: + result = await self.db.execute(select(SystemSecret).where(SystemSecret.key == key)) + return result.scalar_one_or_none() + + async def upsert(self, key: SystemSecretKey, encrypted_value: str) -> SystemSecret: + existing = await self.get(key) + if existing is not None: + existing.encrypted_value = encrypted_value + await self.db.flush() + return existing + record = SystemSecret(key=key, encrypted_value=encrypted_value) + self.db.add(record) + await self.db.flush() + return record + + async def delete(self, key: SystemSecretKey) -> None: + existing = await self.get(key) + if existing is not None: + await self.db.delete(existing) + await self.db.flush() diff --git a/apps/api/app/repositories/unban_request_repository.py b/apps/api/app/repositories/unban_request_repository.py new file mode 100644 index 0000000..932e077 --- /dev/null +++ b/apps/api/app/repositories/unban_request_repository.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import uuid +from datetime import UTC, datetime, timedelta + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.base import ensure_aware_utc +from app.models.unban_request import UnbanRequest + + +class UnbanRequestRepository: + def __init__(self, db: AsyncSession) -> None: + self.db = db + + async def get(self, request_id: uuid.UUID) -> UnbanRequest | None: + result = await self.db.execute(select(UnbanRequest).where(UnbanRequest.id == request_id)) + return result.scalar_one_or_none() + + async def delete(self, request_id: uuid.UUID) -> None: + request = await self.get(request_id) + if request is not None: + await self.db.delete(request) + await self.db.flush() + + async def most_recent_for_ip(self, ip_address: str) -> UnbanRequest | None: + result = await self.db.execute( + select(UnbanRequest) + .where(UnbanRequest.ip_address == ip_address) + .order_by(UnbanRequest.created_at.desc()) + .limit(1) + ) + return result.scalar_one_or_none() + + async def within_cooldown(self, ip_address: str, cooldown_hours: int = 24) -> bool: + latest = await self.most_recent_for_ip(ip_address) + if latest is None: + return False + cutoff = datetime.now(UTC) - timedelta(hours=cooldown_hours) + return ensure_aware_utc(latest.created_at) > cutoff + + async def create(self, ip_address: str, message: str | None) -> UnbanRequest: + record = UnbanRequest(ip_address=ip_address, message=message) + self.db.add(record) + await self.db.flush() + return record + + async def list_all(self, limit: int = 100) -> list[UnbanRequest]: + result = await self.db.execute( + select(UnbanRequest).order_by(UnbanRequest.created_at.desc()).limit(limit) + ) + return list(result.scalars().all()) diff --git a/apps/api/app/repositories/user_api_key_repository.py b/apps/api/app/repositories/user_api_key_repository.py new file mode 100644 index 0000000..d86c043 --- /dev/null +++ b/apps/api/app/repositories/user_api_key_repository.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import uuid + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.enums import ApiKeyProvider +from app.models.user_api_key import UserApiKey + + +class UserApiKeyRepository: + def __init__(self, db: AsyncSession) -> None: + self.db = db + + async def list_for_user(self, user_id: uuid.UUID) -> list[UserApiKey]: + result = await self.db.execute(select(UserApiKey).where(UserApiKey.user_id == user_id)) + return list(result.scalars().all()) + + async def get(self, user_id: uuid.UUID, provider: ApiKeyProvider) -> UserApiKey | None: + result = await self.db.execute( + select(UserApiKey).where(UserApiKey.user_id == user_id, UserApiKey.provider == provider) + ) + return result.scalar_one_or_none() + + async def upsert( + self, user_id: uuid.UUID, provider: ApiKeyProvider, encrypted_key: str + ) -> UserApiKey: + existing = await self.get(user_id, provider) + if existing is not None: + existing.encrypted_key = encrypted_key + await self.db.flush() + return existing + record = UserApiKey(user_id=user_id, provider=provider, encrypted_key=encrypted_key) + self.db.add(record) + await self.db.flush() + return record + + async def delete(self, user_id: uuid.UUID, provider: ApiKeyProvider) -> None: + existing = await self.get(user_id, provider) + if existing is not None: + await self.db.delete(existing) + await self.db.flush() diff --git a/apps/api/app/repositories/user_known_ip_repository.py b/apps/api/app/repositories/user_known_ip_repository.py new file mode 100644 index 0000000..9c48a43 --- /dev/null +++ b/apps/api/app/repositories/user_known_ip_repository.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import uuid +from datetime import datetime + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.user_known_ip import UserKnownIp + + +class UserKnownIpRepository: + def __init__(self, db: AsyncSession) -> None: + self.db = db + + async def get(self, user_id: uuid.UUID, ip_address: str) -> UserKnownIp | None: + result = await self.db.execute( + select(UserKnownIp).where( + UserKnownIp.user_id == user_id, UserKnownIp.ip_address == ip_address + ) + ) + return result.scalar_one_or_none() + + async def list_for_user(self, user_id: uuid.UUID) -> list[UserKnownIp]: + result = await self.db.execute( + select(UserKnownIp) + .where(UserKnownIp.user_id == user_id) + .order_by(UserKnownIp.last_seen_at.desc()) + ) + return list(result.scalars().all()) + + async def record_login(self, user_id: uuid.UUID, ip_address: str, now: datetime) -> bool: + """Touches last_seen_at for an already-known IP, or inserts a new + row for a genuinely new one. Returns True iff this IP was new for + this user - callers don't currently act on that, but it's the + natural hook a future "new IP" alert would use.""" + existing = await self.get(user_id, ip_address) + if existing is not None: + existing.last_seen_at = now + await self.db.flush() + return False + record = UserKnownIp( + user_id=user_id, ip_address=ip_address, first_seen_at=now, last_seen_at=now + ) + self.db.add(record) + await self.db.flush() + return True diff --git a/apps/api/app/repositories/user_repository.py b/apps/api/app/repositories/user_repository.py new file mode 100644 index 0000000..8f8a6ab --- /dev/null +++ b/apps/api/app/repositories/user_repository.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import uuid + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.user import User + + +class UserRepository: + def __init__(self, db: AsyncSession) -> None: + self.db = db + + async def get_by_id(self, user_id: uuid.UUID) -> User | None: + return await self.db.get(User, user_id) + + async def get_by_email(self, email: str) -> User | None: + result = await self.db.execute(select(User).where(User.email == email.lower())) + return result.scalar_one_or_none() + + async def list_admin_emails(self) -> list[str]: + result = await self.db.execute(select(User.email).where(User.is_admin.is_(True))) + return list(result.scalars().all()) + + async def create( + self, + *, + email: str, + password_hash: str | None, + display_name: str, + timezone: str, + user_id: uuid.UUID | None = None, + is_admin: bool = False, + email_verified: bool = False, + ) -> User: + user = User( + id=user_id or uuid.uuid4(), + email=email.lower(), + password_hash=password_hash, + display_name=display_name, + timezone=timezone, + is_admin=is_admin, + email_verified=email_verified, + ) + self.db.add(user) + await self.db.flush() + return user diff --git a/apps/api/app/repositories/user_security_event_repository.py b/apps/api/app/repositories/user_security_event_repository.py new file mode 100644 index 0000000..e31e5c9 --- /dev/null +++ b/apps/api/app/repositories/user_security_event_repository.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import uuid + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.enums import SecurityEventType +from app.models.user_security_event import UserSecurityEvent + + +class UserSecurityEventRepository: + def __init__(self, db: AsyncSession) -> None: + self.db = db + + async def create( + self, *, user_id: uuid.UUID, event_type: SecurityEventType, ip_address: str + ) -> UserSecurityEvent: + record = UserSecurityEvent(user_id=user_id, event_type=event_type, ip_address=ip_address) + self.db.add(record) + await self.db.flush() + return record + + async def most_recent_of_type( + self, user_id: uuid.UUID, event_type: SecurityEventType + ) -> UserSecurityEvent | None: + result = await self.db.execute( + select(UserSecurityEvent) + .where(UserSecurityEvent.user_id == user_id, UserSecurityEvent.event_type == event_type) + .order_by(UserSecurityEvent.created_at.desc()) + .limit(1) + ) + return result.scalar_one_or_none() + + async def list_for_user(self, user_id: uuid.UUID, limit: int = 100) -> list[UserSecurityEvent]: + result = await self.db.execute( + select(UserSecurityEvent) + .where(UserSecurityEvent.user_id == user_id) + .order_by(UserSecurityEvent.created_at.desc()) + .limit(limit) + ) + return list(result.scalars().all()) diff --git a/apps/api/app/schemas/__init__.py b/apps/api/app/schemas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/app/schemas/alert.py b/apps/api/app/schemas/alert.py new file mode 100644 index 0000000..8bc7ff9 --- /dev/null +++ b/apps/api/app/schemas/alert.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import uuid +from datetime import datetime + +from pydantic import BaseModel, ConfigDict + +from app.models.enums import NotificationDeliveryStatus, SeverityLevel + + +class NotificationDeliverySummary(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: uuid.UUID + destination_id: uuid.UUID + provider: str + status: NotificationDeliveryStatus + external_message_id: str | None + error_message: str | None + + +class AlertResponse(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: uuid.UUID + company_id: uuid.UUID + detected_change_id: uuid.UUID + title: str + summary: str + why_it_matters: str + severity: SeverityLevel + confidence: float + read: bool + resolved: bool + created_at: datetime + + +class AlertDetailResponse(AlertResponse): + deliveries: list[NotificationDeliverySummary] + + +class AlertUpdate(BaseModel): + read: bool | None = None + resolved: bool | None = None diff --git a/apps/api/app/schemas/auth.py b/apps/api/app/schemas/auth.py new file mode 100644 index 0000000..d4cbfe7 --- /dev/null +++ b/apps/api/app/schemas/auth.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from datetime import datetime + +from pydantic import BaseModel, EmailStr, Field, field_validator + + +def _validate_password_strength(value: str) -> str: + has_letter = any(c.isalpha() for c in value) + has_digit = any(c.isdigit() for c in value) + if not (has_letter and has_digit): + raise ValueError("Password must contain at least one letter and one digit") + return value + + +class RegisterRequest(BaseModel): + email: EmailStr + password: str = Field(min_length=10, max_length=128) + display_name: str = Field(min_length=1, max_length=120) + timezone: str = Field(default="America/New_York", max_length=64) + turnstile_token: str | None = None + + @field_validator("password") + @classmethod + def _password_strength(cls, value: str) -> str: + return _validate_password_strength(value) + + +class LoginRequest(BaseModel): + email: EmailStr + password: str = Field(min_length=1, max_length=128) + turnstile_token: str | None = None + + +class RefreshRequest(BaseModel): + refresh_token: str + + +class LogoutRequest(BaseModel): + refresh_token: str + + +class TokenResponse(BaseModel): + access_token: str + refresh_token: str + token_type: str = "bearer" + expires_in_minutes: int + + +class VerifyEmailRequest(BaseModel): + email: EmailStr + code: str = Field(min_length=6, max_length=6) + + +class ResendVerificationRequest(BaseModel): + email: EmailStr + turnstile_token: str | None = None + + +class RequestPasswordResetRequest(BaseModel): + email: EmailStr + turnstile_token: str | None = None + + +class ConfirmPasswordResetRequest(BaseModel): + email: EmailStr + code: str = Field(min_length=6, max_length=6) + new_password: str = Field(min_length=10, max_length=128) + + @field_validator("new_password") + @classmethod + def _password_strength(cls, value: str) -> str: + return _validate_password_strength(value) + + +class SecurityEventResponse(BaseModel): + event_type: str + ip_address: str + created_at: datetime + + model_config = {"from_attributes": True} diff --git a/apps/api/app/schemas/company.py b/apps/api/app/schemas/company.py new file mode 100644 index 0000000..8782c05 --- /dev/null +++ b/apps/api/app/schemas/company.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any, Self + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from app.models.company import Company +from app.models.enums import CompanyStatus, EnrichmentStatus, MonitoringFrequency, SeverityLevel + + +def _normalize_website(value: str | None) -> str | None: + if value is None or value.strip() == "": + return None + value = value.strip() + if not (value.startswith("http://") or value.startswith("https://")): + value = f"https://{value}" + return value + + +class CompanyCreate(BaseModel): + name: str = Field(min_length=1, max_length=200) + official_website: str | None = Field(default=None, max_length=500) + description: str | None = Field(default=None, max_length=4000) + monitoring_focus: str | None = Field(default=None, max_length=2000) + industry: str | None = Field(default=None, max_length=120) + country: str | None = Field(default=None, max_length=120) + region: str | None = Field(default=None, max_length=120) + headquarters: str | None = Field(default=None, max_length=200) + public_identifiers: dict[str, str] = Field(default_factory=dict) + competitor_names: list[str] = Field(default_factory=list, max_length=25) + alias_names: list[str] = Field(default_factory=list, max_length=25) + + frequency_type: MonitoringFrequency = MonitoringFrequency.WEEKLY + interval_minutes: int | None = Field(default=None, ge=1) + cron_expression: str | None = Field(default=None, max_length=120) + timezone: str = Field(default="America/New_York", max_length=64) + severity_threshold: SeverityLevel = SeverityLevel.MEDIUM + + @field_validator("official_website") + @classmethod + def _validate_website(cls, value: str | None) -> str | None: + return _normalize_website(value) + + +class CompanyUpdate(BaseModel): + name: str | None = Field(default=None, min_length=1, max_length=200) + official_website: str | None = Field(default=None, max_length=500) + description: str | None = Field(default=None, max_length=4000) + monitoring_focus: str | None = Field(default=None, max_length=2000) + industry: str | None = Field(default=None, max_length=120) + country: str | None = Field(default=None, max_length=120) + region: str | None = Field(default=None, max_length=120) + headquarters: str | None = Field(default=None, max_length=200) + public_identifiers: dict[str, str] | None = None + competitor_names: list[str] | None = Field(default=None, max_length=25) + alias_names: list[str] | None = Field(default=None, max_length=25) + + @field_validator("official_website") + @classmethod + def _validate_website(cls, value: str | None) -> str | None: + return _normalize_website(value) + + +class MonitorConfigurationResponse(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: uuid.UUID + frequency_type: MonitoringFrequency + interval_minutes: int | None + cron_expression: str | None + timezone: str + enabled: bool + next_run: datetime | None + last_run: datetime | None + severity_threshold: SeverityLevel + + +class MonitorConfigurationUpdate(BaseModel): + frequency_type: MonitoringFrequency | None = None + interval_minutes: int | None = Field(default=None, ge=1) + cron_expression: str | None = Field(default=None, max_length=120) + timezone: str | None = Field(default=None, max_length=64) + enabled: bool | None = None + severity_threshold: SeverityLevel | None = None + + +class CompanyEnrichmentResponse(BaseModel): + model_config = ConfigDict(from_attributes=True) + + status: EnrichmentStatus + data: dict[str, Any] = Field(default_factory=dict) + errors: dict[str, str] = Field(default_factory=dict) + credits_spent: int | None + fetched_at: datetime | None + + +class CompanyResponse(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: uuid.UUID + name: str + slug: str + official_website: str | None + description: str | None + monitoring_focus: str | None + industry: str | None + country: str | None + region: str | None + headquarters: str | None + public_identifiers: dict[str, str] = Field(default_factory=dict) + status: CompanyStatus + created_at: datetime + updated_at: datetime + aliases: list[str] = Field(default_factory=list) + competitors: list[str] = Field(default_factory=list) + monitor_configuration: MonitorConfigurationResponse | None = None + # None until the onboarding-time enrichment task finishes (or if + # NINJAPEAR_API_KEY was never configured) - see enrichment_service.py. + enrichment: CompanyEnrichmentResponse | None = None + # Report/alert counts wire up once those models exist (Phases 7-8). + report_count: int = 0 + unresolved_alert_count: int = 0 + + @classmethod + def from_company( + cls, + company: Company, + *, + report_count: int = 0, + unresolved_alert_count: int = 0, + ) -> Self: + return cls( + id=company.id, + name=company.name, + slug=company.slug, + official_website=company.official_website, + description=company.description, + monitoring_focus=company.monitoring_focus, + industry=company.industry, + country=company.country, + region=company.region, + headquarters=company.headquarters, + public_identifiers=company.public_identifiers, + status=company.status, + created_at=company.created_at, + updated_at=company.updated_at, + aliases=[a.alias for a in company.aliases], + competitors=[c.name for c in company.competitors], + monitor_configuration=( + MonitorConfigurationResponse.model_validate(company.monitor_configuration) + if company.monitor_configuration + else None + ), + enrichment=( + CompanyEnrichmentResponse.model_validate(company.enrichment) + if company.enrichment + else None + ), + report_count=report_count, + unresolved_alert_count=unresolved_alert_count, + ) diff --git a/apps/api/app/schemas/dashboard.py b/apps/api/app/schemas/dashboard.py new file mode 100644 index 0000000..4477e33 --- /dev/null +++ b/apps/api/app/schemas/dashboard.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import uuid +from datetime import datetime + +from pydantic import BaseModel + + +class RunsByDayPoint(BaseModel): + date: str + successful: int + failed: int + other: int + + +class RecentSignal(BaseModel): + id: uuid.UUID + company_id: uuid.UUID + company_name: str + change_type: str + severity: str + confidence_score: float + summary: str + created_at: datetime + + +class DashboardAnalytics(BaseModel): + changes_by_type: dict[str, int] + alerts_by_severity: dict[str, int] + sources_by_status: dict[str, int] + runs_by_day: list[RunsByDayPoint] + recent_signals: list[RecentSignal] diff --git a/apps/api/app/schemas/discovery.py b/apps/api/app/schemas/discovery.py new file mode 100644 index 0000000..c21c267 --- /dev/null +++ b/apps/api/app/schemas/discovery.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from pydantic import BaseModel, Field + +from app.models.enums import SourceType + + +class DiscoverCompanyRequest(BaseModel): + name: str = Field(min_length=1, max_length=200) + official_website: str | None = Field(default=None, max_length=500) + monitoring_focus: str | None = Field(default=None, max_length=2000) + competitor_names: list[str] = Field(default_factory=list, max_length=25) + alias_names: list[str] = Field(default_factory=list, max_length=25) + + +class PotentialSource(BaseModel): + source_type: SourceType + name: str + base_url: str | None + + +class DiscoveredCompanyProfile(BaseModel): + name: str + official_website: str | None + description: str | None + monitoring_focus: str | None + industry: str | None + country: str | None + region: str | None + headquarters: str | None + aliases: list[str] + competitors: list[str] + public_identifiers: dict[str, str] + potential_sources: list[PotentialSource] + sources_consulted: list[str] diff --git a/apps/api/app/schemas/monitoring.py b/apps/api/app/schemas/monitoring.py new file mode 100644 index 0000000..4286aa5 --- /dev/null +++ b/apps/api/app/schemas/monitoring.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +import uuid +from datetime import datetime + +from pydantic import BaseModel, ConfigDict + +from app.models.enums import MonitoringRunStatus, MonitoringRunTrigger + + +class MonitoringRunResponse(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: uuid.UUID + company_id: uuid.UUID + trigger_type: MonitoringRunTrigger + status: MonitoringRunStatus + started_at: datetime | None + completed_at: datetime | None + sources_attempted: int + sources_successful: int + sources_failed: int + items_collected: int + changes_detected: int + error_summary: str | None + created_at: datetime diff --git a/apps/api/app/schemas/notification_destination.py b/apps/api/app/schemas/notification_destination.py new file mode 100644 index 0000000..1d77c99 --- /dev/null +++ b/apps/api/app/schemas/notification_destination.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import re +import uuid +from datetime import datetime +from typing import Self + +from email_validator import EmailNotValidError, validate_email +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from app.models.enums import NotificationType, SeverityLevel +from app.models.notification_destination import NotificationDestination + +_PHONE_RE = re.compile(r"^\+?[1-9]\d{7,14}$") + + +class NotificationDestinationCreate(BaseModel): + type: NotificationType + destination_value: str = Field(min_length=1, max_length=320) + minimum_severity: SeverityLevel = SeverityLevel.MEDIUM + enabled: bool = True + company_ids: list[uuid.UUID] = Field( + min_length=1, + description="Which companies this destination receives alerts for. If a destination " + "with the same type/value already exists for this user, it's reused (linked to these " + "companies too) rather than duplicated.", + ) + + @model_validator(mode="after") + def _validate_destination_value(self) -> Self: + if self.type == NotificationType.EMAIL: + try: + validate_email(self.destination_value, check_deliverability=False) + except EmailNotValidError as exc: + raise ValueError(f"Invalid email address: {exc}") from exc + elif self.type == NotificationType.SMS: + if not _PHONE_RE.match(self.destination_value): + raise ValueError("Phone number must be in E.164 format, e.g. +15551234567") + return self + + +class NotificationDestinationUpdate(BaseModel): + destination_value: str | None = Field(default=None, min_length=1, max_length=320) + enabled: bool | None = None + minimum_severity: SeverityLevel | None = None + + +class LinkedCompany(BaseModel): + id: uuid.UUID + name: str + + +class NotificationDestinationResponse(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: uuid.UUID + type: NotificationType + destination_value: str + verified: bool + enabled: bool + minimum_severity: SeverityLevel + created_at: datetime + companies: list[LinkedCompany] = Field(default_factory=list) + + @classmethod + def from_destination(cls, destination: NotificationDestination) -> Self: + return cls( + id=destination.id, + type=destination.type, + destination_value=destination.destination_value, + verified=destination.verified, + enabled=destination.enabled, + minimum_severity=destination.minimum_severity, + created_at=destination.created_at, + companies=[ + LinkedCompany(id=link.company.id, name=link.company.name) + for link in destination.company_links + ], + ) + + +class NotificationTestResult(BaseModel): + success: bool + error: str | None = None diff --git a/apps/api/app/schemas/report.py b/apps/api/app/schemas/report.py new file mode 100644 index 0000000..8def2be --- /dev/null +++ b/apps/api/app/schemas/report.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any + +from pydantic import BaseModel, ConfigDict + +from app.models.enums import ReportType + + +class ReportListItem(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: uuid.UUID + report_type: ReportType + title: str + executive_summary: str + model_provider: str + model_name: str + created_at: datetime + + +class ReportResponse(ReportListItem): + company_id: uuid.UUID + monitoring_run_id: uuid.UUID | None + structured_report: dict[str, Any] + prompt_version: str diff --git a/apps/api/app/schemas/snapshot.py b/apps/api/app/schemas/snapshot.py new file mode 100644 index 0000000..80c023d --- /dev/null +++ b/apps/api/app/schemas/snapshot.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any + +from pydantic import BaseModel, ConfigDict + + +class SnapshotResponse(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: uuid.UUID + source_id: uuid.UUID + snapshot_type: str + hash: str + text_summary: str | None + structured_summary: dict[str, Any] + monitoring_run_id: uuid.UUID | None + created_at: datetime diff --git a/apps/api/app/schemas/source.py b/apps/api/app/schemas/source.py new file mode 100644 index 0000000..e48bf20 --- /dev/null +++ b/apps/api/app/schemas/source.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import uuid +from datetime import datetime + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from app.models.enums import MonitoringFrequency, SourceStatus, SourceType + +# Only these types accept a user-supplied URL via the API. Every other type +# (website/github/sec_edgar/job_posting) is populated by discovery instead, +# and patent/review only ever activate through a configured fixture key. +_USER_CREATABLE_TYPES = {SourceType.CUSTOM_URL, SourceType.RSS} + + +class SourceCreate(BaseModel): + source_type: SourceType + name: str = Field(min_length=1, max_length=200) + base_url: str = Field(min_length=1, max_length=500) + + @field_validator("source_type") + @classmethod + def _validate_type(cls, value: SourceType) -> SourceType: + if value not in _USER_CREATABLE_TYPES: + raise ValueError( + f"{value.value} sources are created by discovery, not added directly. " + f"Only {', '.join(t.value for t in _USER_CREATABLE_TYPES)} may be added here." + ) + return value + + @field_validator("base_url") + @classmethod + def _normalize_url(cls, value: str) -> str: + value = value.strip() + if not (value.startswith("http://") or value.startswith("https://")): + value = f"https://{value}" + return value + + +class SourceUpdate(BaseModel): + name: str | None = Field(default=None, min_length=1, max_length=200) + active: bool | None = None + # None means "inherit the company's default cadence" - the default for + # every source. Sending frequency_type: null explicitly clears an + # existing override back to that default. + frequency_type: MonitoringFrequency | None = None + interval_minutes: int | None = Field(default=None, ge=1) + cron_expression: str | None = Field(default=None, max_length=120) + + +class SourceResponse(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: uuid.UUID + source_type: SourceType + name: str + base_url: str | None + active: bool + status: SourceStatus + trust_score: float + last_checked: datetime | None + last_successful_check: datetime | None + failure_count: int + frequency_type: MonitoringFrequency | None + interval_minutes: int | None + cron_expression: str | None + next_check: datetime | None + + +class SourceTestResult(BaseModel): + status: SourceStatus + documents_found: int + error: str | None diff --git a/apps/api/app/schemas/system_secret.py b/apps/api/app/schemas/system_secret.py new file mode 100644 index 0000000..85e14c7 --- /dev/null +++ b/apps/api/app/schemas/system_secret.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from pydantic import BaseModel, Field + + +class SystemSecretStatus(BaseModel): + key: str + label: str + configured: bool + value: str | None + + +class SetSystemSecretRequest(BaseModel): + # Blank clears the stored override, falling back to the server's + # .env-configured value again. + value: str = Field(default="", max_length=2000) diff --git a/apps/api/app/schemas/unban.py b/apps/api/app/schemas/unban.py new file mode 100644 index 0000000..04e6444 --- /dev/null +++ b/apps/api/app/schemas/unban.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import ipaddress +import uuid +from datetime import datetime + +from pydantic import BaseModel, Field, field_validator + + +class UnbanRequestPayload(BaseModel): + message: str | None = Field(default=None, max_length=2000) + + +class IpBanResponse(BaseModel): + ip_address: str + banned_at: datetime + reason: str + + model_config = {"from_attributes": True} + + +class UnbanRequestResponse(BaseModel): + id: uuid.UUID + ip_address: str + message: str | None + created_at: datetime + + model_config = {"from_attributes": True} + + +class BanIpRequest(BaseModel): + ip_address: str + + @field_validator("ip_address") + @classmethod + def _validate_ip(cls, value: str) -> str: + try: + ipaddress.ip_address(value.strip()) + except ValueError as exc: + raise ValueError("Enter a valid IPv4 or IPv6 address.") from exc + return value.strip() diff --git a/apps/api/app/schemas/user.py b/apps/api/app/schemas/user.py new file mode 100644 index 0000000..8a9f9bc --- /dev/null +++ b/apps/api/app/schemas/user.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import uuid + +from pydantic import BaseModel, ConfigDict + + +class UserResponse(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: uuid.UUID + email: str + display_name: str + timezone: str + is_active: bool + is_admin: bool + + +class MeResponse(UserResponse): + auth_mode: str diff --git a/apps/api/app/schemas/user_api_key.py b/apps/api/app/schemas/user_api_key.py new file mode 100644 index 0000000..1738b6d --- /dev/null +++ b/apps/api/app/schemas/user_api_key.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from pydantic import BaseModel, Field + + +class UserApiKeyStatus(BaseModel): + provider: str + label: str + configured: bool + value: str | None + credits: int | None + credits_note: str | None + free: bool + requires_government_id: bool + + +class SetUserApiKeyRequest(BaseModel): + # Blank clears the user's override, falling back to the server's + # global key for that provider again. + key: str = Field(default="", max_length=500) diff --git a/apps/api/app/search/__init__.py b/apps/api/app/search/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/app/search/base.py b/apps/api/app/search/base.py new file mode 100644 index 0000000..179363e --- /dev/null +++ b/apps/api/app/search/base.py @@ -0,0 +1,26 @@ +"""Search provider interface. Answers "where should we look?" - discovery +only, never a replacement for the LLM or for a SourceCollector. Every +company-discovery query (official website, aliases, competitors, HQ) goes +through this Protocol, never a specific vendor SDK - swapping +`SEARCH_PROVIDER` changes which class `get_search_provider()` returns and +nothing else has to change. See app/services/discovery_service.py for the +only caller. +""" + +from __future__ import annotations + +from typing import Protocol + +from pydantic import BaseModel + + +class SearchResult(BaseModel): + title: str + url: str + snippet: str + + +class SearchProvider(Protocol): + provider_name: str + + async def search(self, query: str, *, count: int = 5) -> list[SearchResult]: ... diff --git a/apps/api/app/search/brave.py b/apps/api/app/search/brave.py new file mode 100644 index 0000000..6274746 --- /dev/null +++ b/apps/api/app/search/brave.py @@ -0,0 +1,42 @@ +"""Brave Search API provider. A fixed, trusted, first-party integration +endpoint (like Twilio/Resend) - calls httpx directly rather than through +`safe_fetch`, which exists specifically to guard arbitrary/user-supplied +collector targets, not our own known-safe API integrations.""" + +from __future__ import annotations + +import httpx + +from app.core.config import Settings +from app.search.base import SearchResult + +_API_URL = "https://api.search.brave.com/res/v1/web/search" + + +class BraveSearchProvider: + provider_name = "brave" + + def __init__(self, settings: Settings) -> None: + self._settings = settings + + async def search(self, query: str, *, count: int = 5) -> list[SearchResult]: + async with httpx.AsyncClient(timeout=15) as client: + response = await client.get( + _API_URL, + params={"q": query, "count": count}, + headers={ + "Accept": "application/json", + "X-Subscription-Token": self._settings.brave_search_api_key, + }, + ) + response.raise_for_status() + data = response.json() + results = data.get("web", {}).get("results", []) + return [ + SearchResult( + title=r.get("title", ""), + url=r.get("url", ""), + snippet=r.get("description", ""), + ) + for r in results[:count] + ] diff --git a/apps/api/app/search/factory.py b/apps/api/app/search/factory.py new file mode 100644 index 0000000..04b7562 --- /dev/null +++ b/apps/api/app/search/factory.py @@ -0,0 +1,21 @@ +"""Resolves `SEARCH_PROVIDER` to a concrete provider instance. Never +imported directly by discovery_service - always go through +`get_search_provider()` so swapping providers stays a one-line config +change.""" + +from __future__ import annotations + +from app.core.config import Settings, get_settings +from app.search.base import SearchProvider +from app.search.mock import MockSearchProvider + + +def get_search_provider(settings: Settings | None = None) -> SearchProvider: + settings = settings or get_settings() + + if settings.search_provider == "brave": + from app.search.brave import BraveSearchProvider + + return BraveSearchProvider(settings) + + return MockSearchProvider() diff --git a/apps/api/app/search/mock.py b/apps/api/app/search/mock.py new file mode 100644 index 0000000..9607d14 --- /dev/null +++ b/apps/api/app/search/mock.py @@ -0,0 +1,62 @@ +"""Deterministic mock search provider - the default (`SEARCH_PROVIDER=mock`) +and what every automated test runs against. Never calls a network. + +Unlike `MockLLMProvider` (which fabricates plausible-looking structured +output from real evidence), this provider has no real evidence to work +from - a company name alone isn't enough to honestly guess an industry, +headquarters, or competitor list. So except for the one query type where a +name-derived guess is genuinely meaningful (official website - many +companies really do live at a domain close to their name), every other +query honestly reports "no information available" as its snippet text. +That honesty flows through `discovery_service.py` and the company-profile +LLM task: fields with no real evidence come back `None`/`[]`, and the +wizard's Review step shows them as "not found - fill in yourself" rather +than presenting fabricated data as fact. +""" + +from __future__ import annotations + +import re + +from app.search.base import SearchResult + +_DOMAIN_STRIP_RE = re.compile(r"[^a-z0-9]+") +_WEBSITE_SUFFIX = " official website" + + +def _guess_domain(name: str) -> str: + slug = _DOMAIN_STRIP_RE.sub("", name.lower()) + return f"{slug}.com" if slug else "example.com" + + +class MockSearchProvider: + provider_name = "mock" + + async def search(self, query: str, *, count: int = 5) -> list[SearchResult]: + if query.lower().endswith(_WEBSITE_SUFFIX): + name = query[: -len(_WEBSITE_SUFFIX)] + domain = _guess_domain(name) + results = [ + SearchResult( + title=f"{name} - Official Site", + url=f"https://{domain}", + snippet=( + f"A likely official website for {name}, guessed from its name. " + "Mock search - no real web search performed; set SEARCH_PROVIDER=brave " + "for real results." + ), + ) + ] + else: + results = [ + SearchResult( + title=f"Mock search: {query}", + url="https://example.com/mock-search", + snippet=( + "No information available - SEARCH_PROVIDER=mock is the dev/test " + "default and performs no real web search. Set SEARCH_PROVIDER=brave " + "for real discovery." + ), + ) + ] + return results[:count] diff --git a/apps/api/app/services/__init__.py b/apps/api/app/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/app/services/alert_service.py b/apps/api/app/services/alert_service.py new file mode 100644 index 0000000..da5e252 --- /dev/null +++ b/apps/api/app/services/alert_service.py @@ -0,0 +1,202 @@ +"""Turns a DetectedChange that crosses the company's alert threshold into an +Alert, generates its summary via Task F (LLM), and dispatches it to every +enabled notification destination that also meets its own severity threshold +- recording one NotificationDelivery per attempt. Two independent +thresholds by design: MonitorConfiguration.severity_threshold gates whether +an Alert is created at all; NotificationDestination.minimum_severity then +gates whether *this* destination gets notified about it (spec section 6I/22). +""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.analysis.llm.base import LLMProvider +from app.core.config import Settings +from app.core.errors import NotFoundError +from app.models.alert import Alert +from app.models.company import Company +from app.models.detected_change import DetectedChange +from app.models.enums import ( + SEVERITY_ORDER, + NotificationDeliveryStatus, + NotificationType, + SeverityLevel, +) +from app.models.notification_delivery import NotificationDelivery +from app.notifications.base import DeliveryResult, NotificationMessage +from app.notifications.factory import get_notification_provider +from app.notifications.message_builder import build_alert_message +from app.prompts.alert_summarization import summarize_alert +from app.repositories.alert_repository import AlertRepository +from app.repositories.notification_delivery_repository import NotificationDeliveryRepository +from app.repositories.notification_destination_repository import NotificationDestinationRepository + + +def _meets_threshold(severity: SeverityLevel, threshold: SeverityLevel) -> bool: + """SEVERITY_ORDER[0] is most severe, so meeting a threshold means being + at least as severe - i.e. an index <= the threshold's index.""" + return SEVERITY_ORDER.index(severity) <= SEVERITY_ORDER.index(threshold) + + +async def create_alert_for_change( + db: AsyncSession, settings: Settings, llm: LLMProvider, change: DetectedChange, company: Company +) -> Alert | None: + config = company.monitor_configuration + if config is None or not _meets_threshold(change.severity, config.severity_threshold): + return None + + evidence_snippets = [ + *change.raw_diff.get("text_added_lines", [])[:5], + *change.raw_diff.get("structured_added", [])[:5], + ] + summary = await summarize_alert( + llm, + company_name=company.name, + change_type=change.change_type.value, + change_summary=change.summary, + severity=change.severity.value, + confidence=change.confidence_score, + evidence_snippets=evidence_snippets, + ) + + alert = Alert( + company_id=company.id, + detected_change_id=change.id, + user_id=company.user_id, + title=summary.title, + summary=summary.summary, + why_it_matters=summary.why_it_matters, + severity=change.severity, + confidence=change.confidence_score, + ) + await AlertRepository(db).create(alert) + + destinations = await NotificationDestinationRepository(db).list_for_company(company.id) + delivery_repo = NotificationDeliveryRepository(db) + for destination in destinations: + if not destination.enabled: + continue + if not _meets_threshold(change.severity, destination.minimum_severity): + continue + if destination.type == NotificationType.SMS and not settings.notification_sms_enabled: + continue + + provider = get_notification_provider(destination.type, settings) + message = build_alert_message( + destination.type, destination.destination_value, company, alert, settings + ) + delivery = NotificationDelivery( + alert_id=alert.id, destination_id=destination.id, provider=provider.provider_name + ) + await delivery_repo.create(delivery) + + result = await provider.send(message) + delivery.attempt_count = 1 + delivery.last_attempt = datetime.now(UTC) + delivery.status = ( + NotificationDeliveryStatus.SENT if result.success else NotificationDeliveryStatus.FAILED + ) + delivery.external_message_id = result.external_message_id + delivery.error_message = result.error + + await db.commit() + await db.refresh(alert) + return alert + + +async def list_alerts( + db: AsyncSession, + user_id: uuid.UUID, + *, + company_id: uuid.UUID | None = None, + severity: SeverityLevel | None = None, + read: bool | None = None, + resolved: bool | None = None, +) -> list[Alert]: + return await AlertRepository(db).list_for_user( + user_id, company_id=company_id, severity=severity, read=read, resolved=resolved + ) + + +async def get_alert(db: AsyncSession, user_id: uuid.UUID, alert_id: uuid.UUID) -> Alert: + alert = await AlertRepository(db).get_for_user(alert_id, user_id) + if alert is None: + raise NotFoundError("Alert not found") + return alert + + +async def mark_read(db: AsyncSession, user_id: uuid.UUID, alert_id: uuid.UUID) -> Alert: + alert = await get_alert(db, user_id, alert_id) + alert.read = True + await db.commit() + await db.refresh(alert) + return alert + + +async def mark_resolved(db: AsyncSession, user_id: uuid.UUID, alert_id: uuid.UUID) -> Alert: + alert = await get_alert(db, user_id, alert_id) + alert.resolved = True + await db.commit() + await db.refresh(alert) + return alert + + +async def update_alert( + db: AsyncSession, + user_id: uuid.UUID, + alert_id: uuid.UUID, + *, + read: bool | None, + resolved: bool | None, +) -> Alert: + alert = await get_alert(db, user_id, alert_id) + if read is not None: + alert.read = read + if resolved is not None: + alert.resolved = resolved + await db.commit() + await db.refresh(alert) + return alert + + +async def get_alert_with_deliveries( + db: AsyncSession, user_id: uuid.UUID, alert_id: uuid.UUID +) -> tuple[Alert, list[NotificationDelivery]]: + alert = await get_alert(db, user_id, alert_id) + deliveries = await NotificationDeliveryRepository(db).list_for_alert(alert.id) + return alert, deliveries + + +async def send_test_notification( + db: AsyncSession, settings: Settings, user_id: uuid.UUID, destination_id: uuid.UUID +): + destination = await NotificationDestinationRepository(db).get_for_user(destination_id, user_id) + if destination is None: + raise NotFoundError("Notification destination not found") + + if destination.type == NotificationType.SMS and not settings.notification_sms_enabled: + return DeliveryResult( + success=False, + error="SMS delivery is currently disabled (NOTIFICATION_SMS_ENABLED=false) - no " + "message was sent.", + ) + + provider = get_notification_provider(destination.type, settings) + if destination.type == NotificationType.SMS: + message = NotificationMessage( + destination_value=destination.destination_value, + subject="CI Agent test", + body_text="CI Agent test SMS: your notification destination is configured correctly.", + ) + else: + message = NotificationMessage( + destination_value=destination.destination_value, + subject="CI Agent test notification", + body_text="This is a test notification from CI Agent. Your destination is configured correctly.", + body_html="

This is a test notification from CI Agent. Your destination is configured correctly.

", + ) + return await provider.send(message) diff --git a/apps/api/app/services/analytics_service.py b/apps/api/app/services/analytics_service.py new file mode 100644 index 0000000..8f30b91 --- /dev/null +++ b/apps/api/app/services/analytics_service.py @@ -0,0 +1,105 @@ +"""Dashboard analytics: aggregate counts across every company a user owns, +scoped by joining through Company.user_id (none of the source tables carry +user_id directly except Alert). Every bucketed dict is pre-seeded with every +enum member at 0 so the frontend never has to guess which keys might be +missing. +""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime, timedelta + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.alert import Alert +from app.models.company import Company +from app.models.detected_change import DetectedChange +from app.models.enums import ChangeType, MonitoringRunStatus, SeverityLevel, SourceStatus +from app.models.monitoring_run import MonitoringRun +from app.models.source import Source +from app.schemas.dashboard import DashboardAnalytics, RecentSignal, RunsByDayPoint + + +async def get_dashboard_analytics( + db: AsyncSession, user_id: uuid.UUID, *, days: int = 30, recent_limit: int = 10 +) -> DashboardAnalytics: + since = datetime.now(UTC) - timedelta(days=days) + + changes_by_type = {ct.value: 0 for ct in ChangeType} + changes_result = await db.execute( + select(DetectedChange.change_type, func.count()) + .join(Company, DetectedChange.company_id == Company.id) + .where(Company.user_id == user_id, DetectedChange.created_at >= since) + .group_by(DetectedChange.change_type) + ) + for change_type, count in changes_result.all(): + changes_by_type[change_type.value] = count + + alerts_by_severity = {s.value: 0 for s in SeverityLevel} + alerts_result = await db.execute( + select(Alert.severity, func.count()) + .where(Alert.user_id == user_id, Alert.created_at >= since) + .group_by(Alert.severity) + ) + for severity, count in alerts_result.all(): + alerts_by_severity[severity.value] = count + + sources_by_status = {s.value: 0 for s in SourceStatus} + sources_result = await db.execute( + select(Source.status, func.count()) + .join(Company, Source.company_id == Company.id) + .where(Company.user_id == user_id) + .group_by(Source.status) + ) + for source_status, count in sources_result.all(): + sources_by_status[source_status.value] = count + + runs_result = await db.execute( + select(MonitoringRun.created_at, MonitoringRun.status) + .join(Company, MonitoringRun.company_id == Company.id) + .where(Company.user_id == user_id, MonitoringRun.created_at >= since) + ) + day_buckets: dict[str, dict[str, int]] = {} + for created_at, run_status in runs_result.all(): + day = created_at.date().isoformat() + bucket = day_buckets.setdefault(day, {"successful": 0, "failed": 0, "other": 0}) + if run_status == MonitoringRunStatus.SUCCESSFUL: + bucket["successful"] += 1 + elif run_status == MonitoringRunStatus.FAILED: + bucket["failed"] += 1 + else: + bucket["other"] += 1 + runs_by_day = [ + RunsByDayPoint(date=day, **counts) for day, counts in sorted(day_buckets.items()) + ] + + signals_result = await db.execute( + select(DetectedChange, Company.name) + .join(Company, DetectedChange.company_id == Company.id) + .where(Company.user_id == user_id) + .order_by(DetectedChange.created_at.desc()) + .limit(recent_limit) + ) + recent_signals = [ + RecentSignal( + id=change.id, + company_id=change.company_id, + company_name=company_name, + change_type=change.change_type.value, + severity=change.severity.value, + confidence_score=change.confidence_score, + summary=change.summary, + created_at=change.created_at, + ) + for change, company_name in signals_result.all() + ] + + return DashboardAnalytics( + changes_by_type=changes_by_type, + alerts_by_severity=alerts_by_severity, + sources_by_status=sources_by_status, + runs_by_day=runs_by_day, + recent_signals=recent_signals, + ) diff --git a/apps/api/app/services/auth_service.py b/apps/api/app/services/auth_service.py new file mode 100644 index 0000000..463346d --- /dev/null +++ b/apps/api/app/services/auth_service.py @@ -0,0 +1,428 @@ +"""Registration/login/refresh/logout/verify-email/password-reset business +logic. + +Kept independent of FastAPI so it's reachable from tests and (later) from +Celery tasks or an admin script without pulling in the HTTP layer. +""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime, timedelta + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.config import Settings +from app.core.errors import AuthenticationError, ConflictError, ThrottledError, ValidationAppError +from app.core.security import ( + InvalidTokenError, + TokenType, + create_access_token, + create_refresh_token, + decode_token, + generate_email_code, + hash_email_code, + hash_password, + hash_token_identifier, + verify_password, +) +from app.db.base import ensure_aware_utc +from app.models.enums import EmailCodePurpose, SecurityEventType, ThrottleAction +from app.models.user import LOCAL_DEV_USER_EMAIL, LOCAL_DEV_USER_ID, User +from app.models.user_security_event import UserSecurityEvent +from app.repositories.email_code_repository import EmailCodeRepository +from app.repositories.password_history_repository import PasswordHistoryRepository +from app.repositories.refresh_token_repository import RefreshTokenRepository +from app.repositories.user_known_ip_repository import UserKnownIpRepository +from app.repositories.user_repository import UserRepository +from app.repositories.user_security_event_repository import UserSecurityEventRepository +from app.schemas.auth import ( + ConfirmPasswordResetRequest, + LoginRequest, + RegisterRequest, + RequestPasswordResetRequest, + ResendVerificationRequest, + TokenResponse, + VerifyEmailRequest, +) +from app.services import ip_throttle_service, security_email_service + +EMAIL_CODE_VALID_HOURS = 36 + +# The local-dev bypass (AUTH_MODE=local + loopback, see +# app/auth/dependencies.py) has no real login step - get_or_create_local_user +# runs on every authenticated request. Logging a login_success event on every +# single request would flood Account activity, so a fresh one is only +# recorded once per this cooldown window, treated as a proxy for "a new +# session" rather than every request within one. +LOCAL_DEV_LOGIN_LOG_COOLDOWN_MINUTES = 30 + + +async def get_or_create_local_user(db: AsyncSession, client_ip: str) -> User: + repo = UserRepository(db) + user = await repo.get_by_id(LOCAL_DEV_USER_ID) + if user is None: + user = await repo.create( + email=LOCAL_DEV_USER_EMAIL, + password_hash=None, + display_name="Local Developer", + timezone="America/New_York", + user_id=LOCAL_DEV_USER_ID, + is_admin=True, + email_verified=True, + ) + await db.commit() + + event_repo = UserSecurityEventRepository(db) + last_login = await event_repo.most_recent_of_type(user.id, SecurityEventType.LOGIN_SUCCESS) + cutoff = datetime.now(UTC) - timedelta(minutes=LOCAL_DEV_LOGIN_LOG_COOLDOWN_MINUTES) + if last_login is None or ensure_aware_utc(last_login.created_at) < cutoff: + await event_repo.create( + user_id=user.id, event_type=SecurityEventType.LOGIN_SUCCESS, ip_address=client_ip + ) + await UserKnownIpRepository(db).record_login(user.id, client_ip, datetime.now(UTC)) + await db.commit() + return user + + +async def _issue_and_send_email_code( + db: AsyncSession, settings: Settings, user: User, purpose: EmailCodePurpose +) -> None: + code = generate_email_code() + code_repo = EmailCodeRepository(db) + # A fresh code must fully supersede every prior unused one for this + # purpose - otherwise an old code (e.g. still sitting in an earlier + # email) stays valid until it naturally expires, even after the user + # explicitly asked for a new one. + await code_repo.invalidate_unused(user.id, purpose) + await code_repo.create( + user_id=user.id, + purpose=purpose, + code_hash=hash_email_code(code), + expires_at=datetime.now(UTC) + timedelta(hours=EMAIL_CODE_VALID_HOURS), + ) + if purpose == EmailCodePurpose.VERIFY_EMAIL: + await security_email_service.send_verification_code_email(settings, user.email, code) + else: + await security_email_service.send_password_reset_email(settings, user.email, code) + + +async def register( + db: AsyncSession, settings: Settings, client_ip: str, payload: RegisterRequest +) -> User: + if await ip_throttle_service.is_banned(db, client_ip): + raise ThrottledError("This IP address has been temporarily blocked.") + + repo = UserRepository(db) + existing = await repo.get_by_email(payload.email) + if existing is not None: + raise ConflictError("An account with this email already exists") + + user = await repo.create( + email=payload.email, + password_hash=hash_password(payload.password), + display_name=payload.display_name, + timezone=payload.timezone, + # Test suite has no inbox to read a real code from - same + # app_env == "test" precedent already used to disable rate limiting + # (app/core/rate_limit.py). The code-generation/sending/throttle + # path below still runs unconditionally either way, so it's still + # exercised by every test that registers a user, not skipped. + email_verified=settings.app_env == "test", + ) + await _issue_and_send_email_code(db, settings, user, EmailCodePurpose.VERIFY_EMAIL) + # Deliberately does NOT call ip_throttle_service.record_attempt here - + # this is per-IP, not per-account (see ip_throttle_service's docstring), + # so unconditionally charging every new registration against it would + # let unrelated people sharing an IP (an office, a NAT'd network) drive + # each other toward a shared ban purely through legitimate signups. The + # escalation ladder starts from the first *manual* resend click instead + # (resend_verification endpoint) - "first resend allowed in 30s" is a + # frontend-only initial cooldown after registration, not a backend- + # enforced one; the backend ladder governs resend #2 onward. + await UserSecurityEventRepository(db).create( + user_id=user.id, event_type=SecurityEventType.EMAIL_VERIFICATION_SENT, ip_address=client_ip + ) + await db.commit() + return user + + +async def verify_email(db: AsyncSession, client_ip: str, payload: VerifyEmailRequest) -> None: + """Generic failure for both 'no such account' and 'wrong/expired code' - + never distinguishes the two to the caller. Guessing the code itself is + IP-throttled the same way a login password guess is - a 6-digit code + has only 1M possible values, so without this an attacker could + brute-force it well within its 36-hour validity window.""" + throttle = await ip_throttle_service.peek_throttle( + db, client_ip, ThrottleAction.VERIFY_EMAIL_CODE + ) + if not throttle.allowed: + _raise_throttled(throttle) + + user_repo = UserRepository(db) + code_repo = EmailCodeRepository(db) + user = await user_repo.get_by_email(payload.email) + record = ( + await code_repo.get_latest_valid( + user.id, EmailCodePurpose.VERIFY_EMAIL, hash_email_code(payload.code) + ) + if user is not None + else None + ) + if record is None: + await ip_throttle_service.record_attempt( + db, + client_ip, + ThrottleAction.VERIFY_EMAIL_CODE, + ip_throttle_service.LOGIN_BACKOFF_SECONDS, + ) + await db.commit() + raise AuthenticationError("Invalid or expired verification code") + + await code_repo.mark_used(record) + user.email_verified = True + await ip_throttle_service.reset_on_success(db, client_ip, ThrottleAction.VERIFY_EMAIL_CODE) + await UserSecurityEventRepository(db).create( + user_id=user.id, event_type=SecurityEventType.EMAIL_VERIFIED, ip_address=client_ip + ) + await db.commit() + + +async def resend_verification( + db: AsyncSession, settings: Settings, client_ip: str, payload: ResendVerificationRequest +) -> None: + throttle = await ip_throttle_service.peek_throttle( + db, client_ip, ThrottleAction.RESEND_VERIFICATION + ) + if not throttle.allowed: + _raise_throttled(throttle) + + user_repo = UserRepository(db) + user = await user_repo.get_by_email(payload.email) + # Always record the attempt, whether or not the account exists, so the + # throttle behavior itself can never leak account existence. + await ip_throttle_service.record_attempt( + db, + client_ip, + ThrottleAction.RESEND_VERIFICATION, + ip_throttle_service.RESEND_BACKOFF_SECONDS, + ) + if user is not None and not user.email_verified: + await _issue_and_send_email_code(db, settings, user, EmailCodePurpose.VERIFY_EMAIL) + await UserSecurityEventRepository(db).create( + user_id=user.id, + event_type=SecurityEventType.EMAIL_VERIFICATION_SENT, + ip_address=client_ip, + ) + await db.commit() + + +async def request_password_reset( + db: AsyncSession, settings: Settings, client_ip: str, payload: RequestPasswordResetRequest +) -> None: + throttle = await ip_throttle_service.peek_throttle(db, client_ip, ThrottleAction.RESEND_RESET) + if not throttle.allowed: + _raise_throttled(throttle) + + user_repo = UserRepository(db) + user = await user_repo.get_by_email(payload.email) + await ip_throttle_service.record_attempt( + db, client_ip, ThrottleAction.RESEND_RESET, ip_throttle_service.RESEND_BACKOFF_SECONDS + ) + if user is not None: + await _issue_and_send_email_code(db, settings, user, EmailCodePurpose.PASSWORD_RESET) + await UserSecurityEventRepository(db).create( + user_id=user.id, + event_type=SecurityEventType.PASSWORD_RESET_REQUESTED, + ip_address=client_ip, + ) + # Always a generic success, regardless of whether the account exists - + # the classic enumeration-safe pattern. + await db.commit() + + +async def confirm_password_reset( + db: AsyncSession, client_ip: str, payload: ConfirmPasswordResetRequest +) -> None: + """Same code-guess throttling rationale as verify_email - a reset code + is just as brute-forceable if left unthrottled.""" + throttle = await ip_throttle_service.peek_throttle( + db, client_ip, ThrottleAction.CONFIRM_RESET_CODE + ) + if not throttle.allowed: + _raise_throttled(throttle) + + user_repo = UserRepository(db) + code_repo = EmailCodeRepository(db) + user = await user_repo.get_by_email(payload.email) + record = ( + await code_repo.get_latest_valid( + user.id, EmailCodePurpose.PASSWORD_RESET, hash_email_code(payload.code) + ) + if user is not None + else None + ) + if record is None: + await ip_throttle_service.record_attempt( + db, + client_ip, + ThrottleAction.CONFIRM_RESET_CODE, + ip_throttle_service.LOGIN_BACKOFF_SECONDS, + ) + await db.commit() + raise AuthenticationError("Invalid or expired reset code") + + # Checked before consuming the code, and before recording any throttle + # attempt, so a rejected-for-reuse password never burns the (correctly + # entered) code - the user can immediately retry with a different one. + history_repo = PasswordHistoryRepository(db) + previous_hashes = await history_repo.list_hashes_for_user(user.id) + if user.password_hash is not None: + previous_hashes.append(user.password_hash) + if any(verify_password(payload.new_password, h) for h in previous_hashes): + raise ValidationAppError("You've used this password before. Choose a different one.") + + await code_repo.mark_used(record) + if user.password_hash is not None: + await history_repo.add(user_id=user.id, password_hash=user.password_hash) + user.password_hash = hash_password(payload.new_password) + # This is the documented unlock mechanism - a successful reset always + # clears any prior lockout, regardless of how it got there. + user.locked_at = None + user.failed_login_count = 0 + + await ip_throttle_service.reset_on_success(db, client_ip, ThrottleAction.CONFIRM_RESET_CODE) + await RefreshTokenRepository(db).revoke_all_for_user(user.id) + await UserSecurityEventRepository(db).create( + user_id=user.id, event_type=SecurityEventType.PASSWORD_RESET_COMPLETED, ip_address=client_ip + ) + await db.commit() + + +def _raise_throttled(throttle: ip_throttle_service.ThrottleResult) -> None: + if throttle.banned: + raise ThrottledError("This IP address has been temporarily blocked.") + raise ThrottledError( + f"Too many attempts. Try again in {throttle.retry_after_seconds} seconds.", + retry_after_seconds=throttle.retry_after_seconds, + ) + + +async def _issue_token_pair(db: AsyncSession, settings: Settings, user: User) -> TokenResponse: + access_token = create_access_token(user.id, settings) + refresh_jwt, jti, expires_at = create_refresh_token(user.id, settings) + + token_repo = RefreshTokenRepository(db) + await token_repo.create( + user_id=user.id, token_hash=hash_token_identifier(jti), expires_at=expires_at + ) + await db.commit() + + return TokenResponse( + access_token=access_token, + refresh_token=refresh_jwt, + expires_in_minutes=settings.jwt_access_token_minutes, + ) + + +# 5 free instant retries, then 5s/15s/30s/60s/2min/5min/15min - the 12th +# failure (index past the end of LOGIN_BACKOFF_SECONDS) is also where the +# account itself locks, in lockstep with the IP throttle's own escalation. +LOGIN_LOCKOUT_THRESHOLD = len(ip_throttle_service.LOGIN_BACKOFF_SECONDS) + 1 + + +async def login( + db: AsyncSession, settings: Settings, client_ip: str, payload: LoginRequest +) -> TokenResponse: + # Enforced *before* password verification - a correct password during a + # penalty window must still be rejected, or the delay is meaningless. + throttle = await ip_throttle_service.peek_throttle(db, client_ip, ThrottleAction.FAILED_LOGIN) + if not throttle.allowed: + _raise_throttled(throttle) + + repo = UserRepository(db) + user = await repo.get_by_email(payload.email) + + if user is not None and user.locked_at is not None: + # Generic message either way - never confirms the account exists. + raise AuthenticationError("Account locked. Reset your password to unlock it.") + + password_ok = ( + user is not None + and user.password_hash is not None + and verify_password(payload.password, user.password_hash) + ) + + if not password_ok: + await ip_throttle_service.record_attempt( + db, client_ip, ThrottleAction.FAILED_LOGIN, ip_throttle_service.LOGIN_BACKOFF_SECONDS + ) + if user is not None: + user.failed_login_count += 1 + await UserSecurityEventRepository(db).create( + user_id=user.id, event_type=SecurityEventType.LOGIN_FAILED, ip_address=client_ip + ) + if user.failed_login_count >= LOGIN_LOCKOUT_THRESHOLD: + user.locked_at = datetime.now(UTC) + await UserSecurityEventRepository(db).create( + user_id=user.id, + event_type=SecurityEventType.ACCOUNT_LOCKED, + ip_address=client_ip, + ) + await security_email_service.send_account_locked_email(settings, user.email) + await db.commit() + raise AuthenticationError("Invalid email or password") + + assert user is not None # password_ok implies this + if not user.is_active: + raise AuthenticationError("This account has been deactivated") + if not user.email_verified: + raise AuthenticationError("Verify your email before signing in.") + + user.failed_login_count = 0 + await ip_throttle_service.reset_on_success(db, client_ip, ThrottleAction.FAILED_LOGIN) + await UserSecurityEventRepository(db).create( + user_id=user.id, event_type=SecurityEventType.LOGIN_SUCCESS, ip_address=client_ip + ) + await UserKnownIpRepository(db).record_login(user.id, client_ip, datetime.now(UTC)) + return await _issue_token_pair(db, settings, user) + + +async def refresh(db: AsyncSession, settings: Settings, raw_refresh_token: str) -> TokenResponse: + try: + decoded = decode_token(raw_refresh_token, settings, TokenType.REFRESH) + except InvalidTokenError as exc: + raise AuthenticationError("Invalid or expired refresh token") from exc + + token_repo = RefreshTokenRepository(db) + stored = await token_repo.get_valid_by_hash(hash_token_identifier(decoded.jti)) + if stored is None: + raise AuthenticationError("Invalid or expired refresh token") + + user_repo = UserRepository(db) + user = await user_repo.get_by_id(decoded.user_id) + if user is None or not user.is_active: + raise AuthenticationError("Invalid or expired refresh token") + + # Rotate: revoke the token just used before issuing a new pair. + await token_repo.revoke(stored) + return await _issue_token_pair(db, settings, user) + + +async def logout(db: AsyncSession, settings: Settings, raw_refresh_token: str) -> None: + try: + decoded = decode_token(raw_refresh_token, settings, TokenType.REFRESH) + except InvalidTokenError: + return # Already unusable; logout is idempotent. + + token_repo = RefreshTokenRepository(db) + stored = await token_repo.get_valid_by_hash(hash_token_identifier(decoded.jti)) + if stored is not None: + await token_repo.revoke(stored) + await db.commit() + + +async def list_security_events(db: AsyncSession, user_id: uuid.UUID) -> list[UserSecurityEvent]: + """The calling user's own security activity - the user-facing + counterpart to the admin-only app-wide log feed (core/logging.py).""" + return await UserSecurityEventRepository(db).list_for_user(user_id) diff --git a/apps/api/app/services/change_detection_service.py b/apps/api/app/services/change_detection_service.py new file mode 100644 index 0000000..a4b2661 --- /dev/null +++ b/apps/api/app/services/change_detection_service.py @@ -0,0 +1,172 @@ +"""Orchestrates the change-detection layers (hash -> structured diff -> text +diff -> scoring) into a single `DetectedChange` row, or nothing if the +change isn't real/meaningful/novel enough to record. + +Cross-source corroboration (Task C in the spec's LLM analysis section) is +Phase 7 scope - `independent_source_count` is always 1 here. See +KNOWN_LIMITATIONS.md. +""" + +from __future__ import annotations + +import re +import uuid +from datetime import UTC, datetime, timedelta + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.change_detection.extractors import extract_prices, mentions_leadership_title +from app.change_detection.scoring import classify_severity, compute_confidence, compute_significance +from app.change_detection.structured_diff import StructuredDiff, diff_item_sets +from app.change_detection.text_diff import TextDiffResult, bounded_text_diff +from app.models.company import Company +from app.models.detected_change import DetectedChange +from app.models.enums import ChangeType, SourceType +from app.models.snapshot import Snapshot +from app.models.source import Source +from app.repositories.detected_change_repository import DetectedChangeRepository +from app.repositories.source_repository import SnapshotRepository + +_COOLDOWN = timedelta(hours=24) +_MIN_CONTENT_DIFF_RATIO = 0.05 + +_EXTRACTION_CONFIDENCE: dict[ChangeType, float] = { + ChangeType.LEADERSHIP_CHANGE: 0.6, + ChangeType.FILING_NEW: 0.95, + ChangeType.PRICE_CHANGE: 0.6, + ChangeType.NEW_DOCUMENT: 0.85, + ChangeType.REMOVED_DOCUMENT: 0.85, + ChangeType.CONTENT_MODIFIED: 0.7, +} + + +async def detect_change_for_source( + db: AsyncSession, + source: Source, + company: Company, + current_snapshot: Snapshot, + monitoring_run_id: uuid.UUID, +) -> DetectedChange | None: + snapshot_repo = SnapshotRepository(db) + previous = await snapshot_repo.get_previous(source.id, current_snapshot) + if previous is None: + return None # Baseline snapshot - nothing to compare against yet. + + # Layer 1: exact hash comparison. + if previous.hash == current_snapshot.hash: + return None + + structured_diff = diff_item_sets( + previous.structured_summary.get("urls", []), + current_snapshot.structured_summary.get("urls", []), + ) + text_diff = bounded_text_diff(previous.text_summary or "", current_snapshot.text_summary or "") + + change_type = _classify_change(source, structured_diff, text_diff) + if change_type is None: + return None # The hash differed, but only noise (Layer 3 already strips it). + + # Always drawn from the actual new text (not the URL/title evidence used + # for display) - that's what's meaningful to compare against the user's + # stated focus, regardless of which change_type it triggered. + focus_evidence_text = "\n".join(text_diff.added_lines) + + raw_diff = { + "structured_added": structured_diff.added, + "structured_removed": structured_diff.removed, + "text_diff_ratio": text_diff.diff_ratio, + "text_added_lines": text_diff.added_lines, + "text_removed_lines": text_diff.removed_lines, + } + + change_repo = DetectedChangeRepository(db) + since = datetime.now(UTC) - _COOLDOWN + recent = await change_repo.get_recent_for_source_and_type(source.id, change_type, since) + if recent is not None and recent.raw_diff == raw_diff: + return None # Exact repeat within the cooldown window - nothing new to report. + is_repeat = recent is not None + + significance = compute_significance( + change_type=change_type, + source_trust_score=source.trust_score, + independent_source_count=1, + focus_match=_matches_focus(company.monitoring_focus, focus_evidence_text), + is_repeat=is_repeat, + diff_ratio=text_diff.diff_ratio if change_type is ChangeType.CONTENT_MODIFIED else None, + ) + confidence = compute_confidence( + extraction_confidence=_EXTRACTION_CONFIDENCE[change_type], + source_trust_score=source.trust_score, + independent_source_count=1, + ) + severity = classify_severity(significance, confidence) + + change = await change_repo.create( + company_id=company.id, + source_id=source.id, + monitoring_run_id=monitoring_run_id, + previous_snapshot_id=previous.id, + current_snapshot_id=current_snapshot.id, + change_type=change_type, + raw_diff=raw_diff, + significance_score=significance, + confidence_score=confidence, + severity=severity, + summary=_build_summary(change_type, structured_diff, text_diff), + ) + await db.commit() + return change + + +def _classify_change( + source: Source, structured_diff: StructuredDiff, text_diff: TextDiffResult +) -> ChangeType | None: + added_text = "\n".join(text_diff.added_lines) + + if mentions_leadership_title(added_text): + return ChangeType.LEADERSHIP_CHANGE + + if source.source_type is SourceType.SEC_EDGAR and structured_diff.added: + return ChangeType.FILING_NEW + + price_delta = extract_prices(added_text) | extract_prices("\n".join(text_diff.removed_lines)) + if price_delta: + return ChangeType.PRICE_CHANGE + + if structured_diff.added: + return ChangeType.NEW_DOCUMENT + + if structured_diff.removed: + return ChangeType.REMOVED_DOCUMENT + + if text_diff.diff_ratio >= _MIN_CONTENT_DIFF_RATIO: + return ChangeType.CONTENT_MODIFIED + + return None + + +def _matches_focus(monitoring_focus: str | None, evidence_text: str) -> bool: + if not monitoring_focus: + return False + focus_words = {w.lower() for w in re.findall(r"[a-zA-Z]{5,}", monitoring_focus)} + evidence_words = {w.lower() for w in re.findall(r"[a-zA-Z]{5,}", evidence_text)} + return bool(focus_words & evidence_words) + + +def _build_summary( + change_type: ChangeType, structured_diff: StructuredDiff, text_diff: TextDiffResult +) -> str: + if change_type is ChangeType.NEW_DOCUMENT: + n = len(structured_diff.added) + return f"{n} new item{'s' if n != 1 else ''} detected" + if change_type is ChangeType.REMOVED_DOCUMENT: + n = len(structured_diff.removed) + return f"{n} item{'s' if n != 1 else ''} removed" + if change_type is ChangeType.PRICE_CHANGE: + return "Pricing information changed" + if change_type is ChangeType.LEADERSHIP_CHANGE: + return "Possible leadership change mentioned" + if change_type is ChangeType.FILING_NEW: + n = len(structured_diff.added) + return f"{n} new regulatory filing{'s' if n != 1 else ''}" + return f"Content changed ({text_diff.diff_ratio:.0%} different)" diff --git a/apps/api/app/services/collection_service.py b/apps/api/app/services/collection_service.py new file mode 100644 index 0000000..b6d38c0 --- /dev/null +++ b/apps/api/app/services/collection_service.py @@ -0,0 +1,198 @@ +"""Orchestrates collectors against the database: turns `DiscoveredSource`s +into `Source` rows, and a collector's `CollectionResult` into persisted +`SourceDocument` + `Snapshot` rows. Collectors themselves stay +database-free (see collectors/base.py) so they're trivially unit-testable; +this module is the seam where that plain-dataclass world meets the ORM. +""" + +from __future__ import annotations + +import hashlib +import uuid +from datetime import UTC, datetime + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.collectors.base import CollectionResult, CompanyContext, SourceConfig +from app.collectors.registry import get_collector +from app.core.config import Settings +from app.core.logging import get_logger +from app.models.company import Company +from app.models.enums import EnrichmentStatus, SourceStatus, SourceType +from app.models.source import Source +from app.repositories.source_repository import ( + SnapshotRepository, + SourceDocumentRepository, + SourceRepository, +) + +logger = get_logger(__name__) + +# Collector types with real (non-fixture) auto-discovery. +_DISCOVERABLE_TYPES = ( + SourceType.WEBSITE, + SourceType.GITHUB, + SourceType.SEC_EDGAR, + SourceType.JOB_POSTING, + SourceType.RSS, + SourceType.GOV_CONTRACT, + SourceType.PATENT, +) + + +def _leadership_names(company: Company) -> list[str]: + enrichment = company.enrichment + if enrichment is None or enrichment.status == EnrichmentStatus.FAILED: + return [] + return [ + member["name"] + for member in enrichment.data.get("leadership_team", []) or [] + if member.get("name") + ] + + +def to_company_context(company: Company, settings: Settings | None = None) -> CompanyContext: + return CompanyContext( + id=str(company.id), + name=company.name, + official_website=company.official_website, + monitoring_focus=company.monitoring_focus, + aliases=[a.alias for a in company.aliases], + competitors=[c.name for c in company.competitors], + leadership_names=_leadership_names(company), + uspto_api_key=settings.uspto_api_key if settings is not None else None, + ) + + +def _to_source_config(source: Source) -> SourceConfig: + return SourceConfig( + id=str(source.id), + source_type=source.source_type, + name=source.name, + base_url=source.base_url, + configuration_metadata=source.configuration_metadata or {}, + ) + + +async def discover_sources_for_company( + db: AsyncSession, company: Company, settings: Settings | None = None +) -> list[Source]: + """Runs discovery for every collector type capable of it and creates a + `Source` row per suggestion, skipping ones that already exist for this + company (same type + base_url).""" + repo = SourceRepository(db) + existing = await repo.list_for_company(company.id) + existing_keys = {(s.source_type, s.base_url) for s in existing} + + context = to_company_context(company, settings) + created: list[Source] = [] + + for source_type in _DISCOVERABLE_TYPES: + collector = get_collector(source_type) + try: + discovered = await collector.discover(context) + except Exception as exc: # pragma: no cover - defensive, discovery is best-effort + logger.warning("source_discovery_failed", source_type=source_type, error=str(exc)) + continue + + for candidate in discovered: + key = (candidate.source_type, candidate.base_url) + if key in existing_keys: + continue + source = await repo.create( + company_id=company.id, + source_type=candidate.source_type, + name=candidate.name, + base_url=candidate.base_url, + configuration_metadata=candidate.configuration_metadata, + ) + existing_keys.add(key) + created.append(source) + + return created + + +def _summarize_documents(documents) -> dict: + return { + "document_count": len(documents), + "titles": [d.title for d in documents if d.title][:50], + "urls": [d.url for d in documents][:50], + "content_hashes": [d.content_hash for d in documents][:50], + } + + +def _build_text_summary(documents) -> str: + """Concatenated per-document excerpts used for bounded text diffing + (change_detection's text-diff layer) - a title-only summary is too thin + to catch wording-level changes within a page.""" + parts = [] + for doc in documents[:8]: + title = doc.title or doc.url + excerpt = doc.content_text[:600] + parts.append(f"### {title}\n{excerpt}") + return "\n\n".join(parts) + + +async def collect_source( + db: AsyncSession, + settings: Settings, + source: Source, + company: Company, + *, + monitoring_run_id: uuid.UUID | None = None, +) -> CollectionResult: + """Runs one source's collector, persists new documents (deduped by + content hash within the source), writes a Snapshot summarizing the + batch, and updates the Source's health/status fields.""" + collector = get_collector(source.source_type) + context = to_company_context(company, settings) + config = _to_source_config(source) + + result = await collector.collect(config, context) + + doc_repo = SourceDocumentRepository(db) + persisted_hashes: list[str] = [] + for doc in result.documents: + if await doc_repo.exists_with_hash(source.id, doc.content_hash): + continue + await doc_repo.create( + source_id=source.id, + company_id=company.id, + url=doc.url, + canonical_url=doc.canonical_url, + title=doc.title, + author=doc.author, + publication_date=doc.publication_date, + retrieved_date=doc.retrieved_date, + content_text=doc.content_text, + content_hash=doc.content_hash, + metadata_json=doc.metadata, + language=doc.language, + http_status=doc.http_status, + extraction_method=doc.extraction_method, + trust_score=doc.trust_score, + ) + persisted_hashes.append(doc.content_hash) + + now = datetime.now(UTC) + if result.documents: + batch_hash = hashlib.sha256( + "|".join(sorted(d.content_hash for d in result.documents)).encode("utf-8") + ).hexdigest() + snapshot_repo = SnapshotRepository(db) + await snapshot_repo.create( + company_id=company.id, + source_id=source.id, + snapshot_type=source.source_type.value, + hash=batch_hash, + structured_summary=_summarize_documents(result.documents), + text_summary=_build_text_summary(result.documents), + monitoring_run_id=monitoring_run_id, + ) + + source_repo = SourceRepository(db) + success = result.status in (SourceStatus.ACTIVE,) + await source_repo.mark_checked(source, status=result.status, checked_at=now, success=success) + + await db.commit() + return result diff --git a/apps/api/app/services/company_service.py b/apps/api/app/services/company_service.py new file mode 100644 index 0000000..798d226 --- /dev/null +++ b/apps/api/app/services/company_service.py @@ -0,0 +1,243 @@ +"""Company + monitor configuration business logic. + +Every lookup here is scoped to `user_id` at the query level (see +CompanyRepository), so a company that exists but belongs to another user +raises NotFoundError exactly like one that doesn't exist - this avoids +leaking existence via a 403-vs-404 timing/response difference. +""" + +from __future__ import annotations + +import uuid + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.config import Settings +from app.core.errors import ConflictError, NotFoundError +from app.core.text import slugify +from app.models.company import Company +from app.models.enums import CompanyStatus, EnrichmentStatus +from app.models.monitor_configuration import MonitorConfiguration +from app.repositories.company_enrichment_repository import CompanyEnrichmentRepository +from app.repositories.company_repository import CompanyRepository, MonitorConfigurationRepository +from app.repositories.notification_destination_repository import ( + NotificationDestinationRepository, +) +from app.schemas.company import CompanyCreate, CompanyUpdate, MonitorConfigurationUpdate +from app.services.scheduling import validate_and_compute_next_run + + +async def _unique_slug(repo: CompanyRepository, user_id: uuid.UUID, name: str) -> str: + base = slugify(name) + slug = base + suffix = 1 + while await repo.slug_exists_for_user(user_id, slug): + suffix += 1 + slug = f"{base}-{suffix}" + return slug + + +async def _unique_display_name(repo: CompanyRepository, user_id: uuid.UUID, name: str) -> str: + """Guarantees the created row's name is unique for this user, the same + way a filesystem silently renames a colliding file - "Stripe" stays + "Stripe" unless the user already has one, in which case this becomes + "Stripe (2)", "Stripe (3)", etc. The wizard warns about likely + duplicates *before* this ever runs (see the frontend's own near-duplicate + check) so this is a last-resort guarantee, not the primary UX.""" + if not await repo.name_exists_for_user(user_id, name): + return name + suffix = 2 + while await repo.name_exists_for_user(user_id, f"{name} ({suffix})"): + suffix += 1 + return f"{name} ({suffix})" + + +async def list_companies(db: AsyncSession, user_id: uuid.UUID) -> list[Company]: + repo = CompanyRepository(db) + return await repo.list_for_user(user_id) + + +async def get_company(db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID) -> Company: + repo = CompanyRepository(db) + company = await repo.get_for_user(company_id, user_id) + if company is None: + raise NotFoundError("Company not found") + return company + + +async def create_company( + db: AsyncSession, settings: Settings, user_id: uuid.UUID, payload: CompanyCreate +) -> Company: + company_repo = CompanyRepository(db) + monitor_repo = MonitorConfigurationRepository(db) + + existing_count = await company_repo.count_for_user(user_id) + if existing_count >= settings.max_companies_per_user: + raise ConflictError( + f"You've reached the maximum of {settings.max_companies_per_user} monitored companies" + ) + + name = await _unique_display_name(company_repo, user_id, payload.name) + slug = await _unique_slug(company_repo, user_id, payload.name) + + next_run = validate_and_compute_next_run( + frequency_type=payload.frequency_type, + interval_minutes=payload.interval_minutes, + cron_expression=payload.cron_expression, + tz_name=payload.timezone, + minimum_interval_minutes=settings.minimum_monitoring_interval_minutes, + ) + + company = await company_repo.create( + user_id=user_id, + name=name, + slug=slug, + official_website=payload.official_website, + description=payload.description, + monitoring_focus=payload.monitoring_focus, + industry=payload.industry, + country=payload.country, + region=payload.region, + headquarters=payload.headquarters, + public_identifiers=payload.public_identifiers, + alias_names=payload.alias_names, + competitor_names=payload.competitor_names, + ) + await monitor_repo.create_default( + company_id=company.id, + frequency_type=payload.frequency_type, + interval_minutes=payload.interval_minutes, + cron_expression=payload.cron_expression, + timezone=payload.timezone, + severity_threshold=payload.severity_threshold, + next_run=next_run, + ) + if settings.ninjapear_api_key: + # A PENDING row is created up front (not just enqueued) so the + # frontend has something real to poll on - otherwise "not yet + # enriched" and "never configured" would look identical (both + # `enrichment: null`). enrichment_service.enrich_company updates + # this same row in place once the task finishes. + await CompanyEnrichmentRepository(db).upsert( + company.id, + status=EnrichmentStatus.PENDING, + data={}, + errors={}, + credits_spent=None, + fetched_at=None, + ) + await db.commit() + + if settings.ninjapear_api_key: + # Fire-and-forget, onboarding-only enrichment - gated on the key + # itself (not just falling through to a mock provider) so a user + # who never configured NinjaPear gets zero extra background-task + # volume. See app/services/enrichment_service.py. + from app.tasks.enrichment import ( + enrich_company, # local import: keeps Celery out of API startup path + ) + + enrich_company.delay(str(company.id)) + + refreshed = await company_repo.get_for_user(company.id, user_id) + assert refreshed is not None + return refreshed + + +async def update_company( + db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID, payload: CompanyUpdate +) -> Company: + company_repo = CompanyRepository(db) + company = await get_company(db, user_id, company_id) + + updates = payload.model_dump(exclude_unset=True, exclude={"alias_names", "competitor_names"}) + for field, value in updates.items(): + setattr(company, field, value) + + if payload.alias_names is not None: + await company_repo.replace_aliases(company, payload.alias_names) + if payload.competitor_names is not None: + await company_repo.replace_competitors(company, payload.competitor_names) + + await db.commit() + refreshed = await company_repo.get_for_user(company_id, user_id) + assert refreshed is not None + return refreshed + + +async def delete_company(db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID) -> None: + company_repo = CompanyRepository(db) + company = await get_company(db, user_id, company_id) + await company_repo.delete(company) + await db.commit() + + # Garbage-collect any notification destination that was only ever + # linked to this now-deleted company - a destination with zero company + # links left behind is dead weight, not a valid "applies to nothing" + # state (see NotificationDestinationRepository.delete_orphaned_for_user). + await NotificationDestinationRepository(db).delete_orphaned_for_user(user_id) + await db.commit() + + +async def _set_company_status( + db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID, status: CompanyStatus +) -> Company: + company = await get_company(db, user_id, company_id) + company.status = status + if company.monitor_configuration is not None: + company.monitor_configuration.enabled = status == CompanyStatus.ACTIVE + await db.commit() + company_repo = CompanyRepository(db) + refreshed = await company_repo.get_for_user(company_id, user_id) + assert refreshed is not None + return refreshed + + +async def pause_company(db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID) -> Company: + return await _set_company_status(db, user_id, company_id, CompanyStatus.PAUSED) + + +async def resume_company(db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID) -> Company: + return await _set_company_status(db, user_id, company_id, CompanyStatus.ACTIVE) + + +async def update_monitor_configuration( + db: AsyncSession, + settings: Settings, + user_id: uuid.UUID, + company_id: uuid.UUID, + payload: MonitorConfigurationUpdate, +) -> MonitorConfiguration: + company = await get_company(db, user_id, company_id) + config = company.monitor_configuration + if config is None: + raise NotFoundError("Monitor configuration not found") + + updates = payload.model_dump(exclude_unset=True) + frequency_type = updates.get("frequency_type", config.frequency_type) + interval_minutes = updates.get("interval_minutes", config.interval_minutes) + cron_expression = updates.get("cron_expression", config.cron_expression) + tz_name = updates.get("timezone", config.timezone) + + schedule_changed = any( + key in updates + for key in ("frequency_type", "interval_minutes", "cron_expression", "timezone") + ) + if schedule_changed: + config.next_run = validate_and_compute_next_run( + frequency_type=frequency_type, + interval_minutes=interval_minutes, + cron_expression=cron_expression, + tz_name=tz_name, + minimum_interval_minutes=settings.minimum_monitoring_interval_minutes, + ) + + for field, value in updates.items(): + setattr(config, field, value) + + if "enabled" in updates: + company.status = CompanyStatus.ACTIVE if updates["enabled"] else CompanyStatus.PAUSED + + await db.commit() + await db.refresh(config) + return config diff --git a/apps/api/app/services/discovery_service.py b/apps/api/app/services/discovery_service.py new file mode 100644 index 0000000..145061b --- /dev/null +++ b/apps/api/app/services/discovery_service.py @@ -0,0 +1,190 @@ +"""Company-metadata discovery: given just a name (plus optional user +hints), proposes official_website/industry/country/region/headquarters/ +aliases/competitors/public_identifiers and a preview of sources the +pipeline would start monitoring - all before anything is persisted. This +is the "System performs company discovery" step between "user enters a +name" and "user confirms/edits" in the onboarding flow. + +Two independent, deliberately-separated concerns per the architecture +direction: +- SearchProvider answers "where should we look" (this module's job). +- The LLM only ever analyzes evidence this module already gathered - see + app/prompts/company_profile.py's docstring. It is never asked to recall + facts about the company from its own training data. + +Runs exactly once, at onboarding time, driven by an explicit user action +(the wizard's "Discover" step) - never re-triggered by scheduled monitoring +runs. Source *persistence* still happens exactly as it already did before +this module existed: lazily, on the company's first monitoring run (see +tasks/collection.py). This module only ever previews what that step would +find, via the same collector.discover() calls, without writing anything. +""" + +from __future__ import annotations + +from app.analysis.llm.base import LLMProvider +from app.collectors.base import CompanyContext +from app.collectors.extraction import extract_readable_text +from app.collectors.registry import get_collector +from app.collectors.robots import is_allowed +from app.core.config import Settings +from app.core.http import FetchError, SsrfBlockedError, fetch_with_retries +from app.core.logging import get_logger +from app.models.enums import SourceType +from app.prompts.company_profile import extract_company_profile +from app.schemas.discovery import DiscoveredCompanyProfile, PotentialSource +from app.search.base import SearchProvider + +logger = get_logger(__name__) + +# A "{name} official website" search frequently ranks a reference/social +# page above the company's own domain for well-known companies (observed +# live: Brave's top result for "Stripe official website" was Stripe's +# Wikipedia article, not stripe.com). Picking that as `official_website` +# then feeds a wrong base domain into every downstream source-preview +# collector. Skip these hosts when a better-ranked alternative exists in +# the same result set, rather than blindly taking the top hit. +_NON_CORPORATE_HOSTS = ( + "wikipedia.org", + "linkedin.com", + "crunchbase.com", + "bloomberg.com", + "facebook.com", + "twitter.com", + "x.com", + "youtube.com", + "reddit.com", + "glassdoor.com", +) + + +def _is_non_corporate_host(url: str) -> bool: + host = url.split("//", 1)[-1].split("/", 1)[0].lower() + return any(host == d or host.endswith(f".{d}") for d in _NON_CORPORATE_HOSTS) + + +# Same set collection_service.py's discover_sources_for_company already +# discovers from for a real company - kept in sync deliberately, not +# imported, since this module previews without a persisted Company/Source +# and the coupling would only make both harder to read. +_PREVIEWABLE_TYPES = ( + SourceType.WEBSITE, + SourceType.GITHUB, + SourceType.SEC_EDGAR, + SourceType.JOB_POSTING, + SourceType.RSS, + SourceType.GOV_CONTRACT, + SourceType.PATENT, +) + + +async def _resolve_official_website( + search: SearchProvider, name: str, hint: str | None +) -> tuple[str | None, list[str]]: + if hint: + return hint, [] + results = await search.search(f"{name} official website", count=3) + if not results: + return None, [] + best = next((r for r in results if not _is_non_corporate_host(r.url)), results[0]) + return best.url, [best.url] + + +async def _fetch_homepage_text( + settings: Settings, official_website: str | None +) -> tuple[str | None, list[str]]: + if not official_website: + return None, [] + try: + if not await is_allowed(official_website, settings=settings): + return None, [] + result = await fetch_with_retries(official_website, settings=settings, max_attempts=1) + except (FetchError, SsrfBlockedError) as exc: + logger.info("discovery_homepage_fetch_failed", url=official_website, error=str(exc)) + return None, [] + + if result.status_code >= 400: + return None, [] + + text, _method = extract_readable_text(result.text, official_website) + return (text or None), [official_website] + + +async def _preview_sources( + context: CompanyContext, +) -> list[PotentialSource]: + previews: list[PotentialSource] = [] + for source_type in _PREVIEWABLE_TYPES: + collector = get_collector(source_type) + try: + discovered = await collector.discover(context) + except Exception as exc: # pragma: no cover - defensive, preview is best-effort + logger.warning( + "discovery_source_preview_failed", source_type=source_type, error=str(exc) + ) + continue + previews.extend( + PotentialSource(source_type=d.source_type, name=d.name, base_url=d.base_url) + for d in discovered + ) + return previews + + +async def discover_company_profile( + search: SearchProvider, + llm: LLMProvider, + settings: Settings, + *, + name: str, + official_website: str | None, + monitoring_focus: str | None, + competitor_names: list[str], + alias_names: list[str], +) -> DiscoveredCompanyProfile: + resolved_website, consulted_website = await _resolve_official_website( + search, name, official_website + ) + homepage_text, consulted_homepage = await _fetch_homepage_text(settings, resolved_website) + + search_results = [] + consulted_queries: list[str] = [] + for query in (f"{name} headquarters", f"{name} competitors", f"{name} formerly known as"): + results = await search.search(query, count=3) + search_results.extend({"query": query, **r.model_dump()} for r in results) + consulted_queries.append(query) + + extraction = await extract_company_profile( + llm, + company_name=name, + homepage_url=resolved_website, + homepage_text=homepage_text, + search_results=search_results, + ) + + context = CompanyContext( + id="pending", + name=name, + official_website=resolved_website, + monitoring_focus=monitoring_focus, + uspto_api_key=settings.uspto_api_key, + ) + # Individual collectors already handle a missing official_website + # gracefully (e.g. JobPostingCollector.discover returns [] rather than + # raising) - GitHub/SEC EDGAR search by name and don't need one at all. + potential_sources = await _preview_sources(context) + + return DiscoveredCompanyProfile( + name=name, + official_website=resolved_website, + description=extraction.description, + monitoring_focus=monitoring_focus, + industry=extraction.industry, + country=extraction.country, + region=extraction.region, + headquarters=extraction.headquarters, + aliases=alias_names or extraction.aliases, + competitors=competitor_names or extraction.competitors, + public_identifiers={pi.key: pi.value for pi in extraction.public_identifiers}, + potential_sources=potential_sources, + sources_consulted=[*consulted_website, *consulted_homepage, *consulted_queries], + ) diff --git a/apps/api/app/services/enrichment_service.py b/apps/api/app/services/enrichment_service.py new file mode 100644 index 0000000..d531e73 --- /dev/null +++ b/apps/api/app/services/enrichment_service.py @@ -0,0 +1,176 @@ +"""Onboarding-time company enrichment via a paid third-party provider +(NinjaPear/nubela.co) - fires exactly once per company, never on a +recurring schedule (see app/tasks/enrichment.py and +company_service.create_company, which gates the enqueue itself on +NINJAPEAR_API_KEY being set). + +Orchestrates several independent, per-endpoint provider calls; one bad +call must never sink the others (same principle as tasks/collection.py's +per-source loop) - every failure is recorded in `errors` rather than +silently dropped or allowed to fail the whole enrichment. Person-level +lookups (work email, profile) are capped at +`settings.ninjapear_max_leadership_lookups` to bound the fan-out from a +large leadership team. +""" + +from __future__ import annotations + +from datetime import UTC, datetime + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.config import Settings +from app.core.logging import get_logger +from app.enrichment.base import EnrichmentProvider +from app.models.company import Company +from app.models.company_enrichment import CompanyEnrichment +from app.models.enums import EnrichmentStatus +from app.repositories.company_enrichment_repository import CompanyEnrichmentRepository + +logger = get_logger(__name__) + +# Approximate credit cost per call per NinjaPear's published pricing - +# tracked for user-facing cost transparency (Settings page) only, not +# billed or enforced by this app. +_CREDIT_COSTS = { + "details": 5, # 3 base + 2 for the employee-count add-on + "funding": 3, # 2 base + a rough per-investor estimate + "updates": 2, + "competitors": 5, # NinjaPear's documented minimum per request + "products": 3, + "customers": 3, # 1 base + a rough per-company estimate + "work_email": 2, + "person_profile": 3, +} +_PER_COMPANY_SECTIONS = ("details", "funding", "updates", "competitors", "products", "customers") +_PER_LEADER_SECTIONS = ("work_email", "person_profile") + + +def estimate_max_credits_per_company(max_leadership_lookups: int) -> int: + """Worst-case credit ceiling for one company's enrichment - every + company-level section succeeding plus every leadership-lookup slot + used. Shown to the user before they commit to creating a company (see + the add-company wizard's Confirm step) so the real cost is never a + surprise.""" + base = sum(_CREDIT_COSTS[section] for section in _PER_COMPANY_SECTIONS) + per_leader = sum(_CREDIT_COSTS[section] for section in _PER_LEADER_SECTIONS) + return base + per_leader * max_leadership_lookups + + +async def enrich_company( + db: AsyncSession, settings: Settings, provider: EnrichmentProvider, company: Company +) -> CompanyEnrichment: + website = company.official_website + if not website: + # NinjaPear identifies a company by website only - every call would + # fail the same way, so skip straight to FAILED instead of burning + # credits on N doomed requests. + repo = CompanyEnrichmentRepository(db) + enrichment = await repo.upsert( + company.id, + status=EnrichmentStatus.FAILED, + data={}, + errors={"details": "No official_website on file - NinjaPear requires one"}, + credits_spent=0, + fetched_at=datetime.now(UTC), + ) + await db.commit() + return enrichment + + data: dict = {} + errors: dict[str, str] = {} + credits_spent = 0 + attempted = 0 + succeeded = 0 + + async def _run(section: str, coro): + nonlocal credits_spent, attempted, succeeded + attempted += 1 + try: + result = await coro + except Exception as exc: # noqa: BLE001 - one bad call must never sink the rest + errors[section] = str(exc) + logger.warning( + "enrichment_section_failed", + section=section, + company_id=str(company.id), + error=str(exc), + ) + return None + credits_spent += _CREDIT_COSTS.get(section, 0) + succeeded += 1 + return result + + details = await _run("details", provider.get_company_details(company.name, website)) + leadership_team: list[dict] = [] + if details is not None: + data["employee_count"] = details.employee_count_range + data["description"] = details.description + data["industry"] = details.industry + data["founded_year"] = details.founded_year + data["specialties"] = details.specialties + leadership_team = [m.model_dump() for m in details.leadership_team] + + funding = await _run("funding", provider.get_funding(company.name, website)) + if funding is not None: + data["funding"] = funding.model_dump() + + updates = await _run("updates", provider.get_updates(company.name, website)) + if updates is not None: + data["recent_updates"] = [u.model_dump() for u in updates] + + competitors = await _run("competitors", provider.get_competitors(company.name, website)) + if competitors is not None: + data["competitors"] = [c.model_dump() for c in competitors] + + products = await _run("products", provider.get_products(company.name, website)) + if products is not None: + data["products"] = [p.model_dump() for p in products] + + customers = await _run("customers", provider.get_customers(company.name, website)) + if customers is not None: + data["customers"] = [c.model_dump() for c in customers] + + if leadership_team and website: + cap = settings.ninjapear_max_leadership_lookups + for member in leadership_team[:cap]: + person_name = member.get("name") + if not person_name: + continue + email = await _run("work_email", provider.get_work_email(person_name, website)) + if email: + member["work_email"] = email + profile = await _run( + "person_profile", provider.get_person_profile(person_name, website) + ) + if profile is not None: + profile_url, bio = profile + member["profile_url"] = member.get("profile_url") or profile_url + member["bio"] = member.get("bio") or bio + data["leadership_team"] = leadership_team + + if attempted == 0 or succeeded == 0: + status = EnrichmentStatus.FAILED + elif succeeded == attempted: + status = EnrichmentStatus.COMPLETE + else: + status = EnrichmentStatus.PARTIAL + + repo = CompanyEnrichmentRepository(db) + enrichment = await repo.upsert( + company.id, + status=status, + data=data, + errors=errors, + credits_spent=credits_spent, + fetched_at=datetime.now(UTC), + ) + await db.commit() + logger.info( + "company_enrichment_finished", + company_id=str(company.id), + status=status.value, + credits_spent=credits_spent, + failed_sections=list(errors.keys()), + ) + return enrichment diff --git a/apps/api/app/services/ip_throttle_service.py b/apps/api/app/services/ip_throttle_service.py new file mode 100644 index 0000000..fdcb99b --- /dev/null +++ b/apps/api/app/services/ip_throttle_service.py @@ -0,0 +1,151 @@ +"""Generic, per-IP escalation engine shared by resend-verification, +resend-password-reset, and failed-login throttling. + +Two functions, not one, so login can enforce the gate *before* verifying a +password (a correct password during a penalty window must still be +rejected, or the delay is meaningless) without a check-then-act race: + +- `peek_throttle` - read-only. Is this IP allowed to attempt `action` right + now? Also lazily resets a completed timeout cycle (memory of the offense + survives via `offense_count`; only the attempt stage resets - see + `IpThrottleState`). +- `record_attempt` - call only after the gated action actually happens (a + failed login, or a resend that's being sent). Advances the stage/backoff, + and escalates into a timeout (and eventually a permanent ban) once the + stage array is exhausted. + +All time comparisons go through `_now()` so tests can monkeypatch it +directly to walk the whole escalation ladder in milliseconds of real time - +no waiting, no manual brute-forcing. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.base import ensure_aware_utc +from app.models.enums import ThrottleAction +from app.repositories.ip_throttle_repository import IpThrottleRepository + +# 30s, 1min, 2min, 5min, 5min - shared identically by resend_verification and +# resend_reset. The *first* code send (at registration / at a reset request) +# also advances this, so "first resend allowed in 30s" is measured from that +# original send, not from a first manual resend click. +RESEND_BACKOFF_SECONDS: list[int] = [30, 60, 120, 300, 300] + +# A stage's value is the wait *before the next* attempt (recording attempt K +# sets the delay gating attempt K+1) - so 5 truly free attempts (1-5, no +# wait before any of them) needs only 4 leading zeros (gating attempts +# 2-5), then the 7 real delays gate attempts 6-12 (5s/15s/30s/60s/2min/5min/ +# 15min). Recording attempt 12 itself lands on stage_index=11, past the end +# of this 11-entry array - exhausted, which is exactly the intended "12th +# failure locks the account and starts the IP's timeout ladder" behavior. +LOGIN_BACKOFF_SECONDS: list[int] = [0, 0, 0, 0, 5, 15, 30, 60, 120, 300, 900] + +# 30min, 1h, 2h, 3h, 4h, 5h - indexed by offense_count. Exceeding this length +# is a permanent ban. +TIMEOUT_LADDER_SECONDS: list[int] = [1800, 3600, 7200, 10800, 14400, 18000] + + +def _now() -> datetime: + return datetime.now(UTC) + + +@dataclass(frozen=True) +class ThrottleResult: + allowed: bool + retry_after_seconds: int | None = None + banned: bool = False + + +async def is_banned(db: AsyncSession, ip_address: str) -> bool: + """Ban-only check, decoupled from any specific action's stage/timeout + state - for call sites like register() that have no throttle action of + their own but still must never let a banned IP through.""" + return await IpThrottleRepository(db).get_ban(ip_address) is not None + + +async def peek_throttle( + db: AsyncSession, ip_address: str, action: ThrottleAction +) -> ThrottleResult: + repo = IpThrottleRepository(db) + + ban = await repo.get_ban(ip_address) + if ban is not None: + return ThrottleResult(allowed=False, banned=True) + + state = await repo.get_or_create_state(ip_address, action) + now = _now() + + if state.timeout_until is not None: + timeout_until = ensure_aware_utc(state.timeout_until) + if timeout_until > now: + return ThrottleResult( + allowed=False, retry_after_seconds=int((timeout_until - now).total_seconds()) + ) + # Timeout has elapsed - cycle reset. offense_count (the memory of + # this IP's history) is deliberately left untouched. + state.timeout_until = None + state.attempt_count = 0 + state.next_allowed_at = None + await db.flush() + + if state.next_allowed_at is not None: + next_allowed_at = ensure_aware_utc(state.next_allowed_at) + if next_allowed_at > now: + return ThrottleResult( + allowed=False, retry_after_seconds=int((next_allowed_at - now).total_seconds()) + ) + + return ThrottleResult(allowed=True) + + +async def record_attempt( + db: AsyncSession, ip_address: str, action: ThrottleAction, backoff_stages: list[int] +) -> None: + repo = IpThrottleRepository(db) + state = await repo.get_or_create_state(ip_address, action) + now = _now() + + stage_index = state.attempt_count + state.attempt_count += 1 + + if stage_index < len(backoff_stages): + delay = backoff_stages[stage_index] + state.next_allowed_at = now + timedelta(seconds=delay) + await db.flush() + return + + # Backoff stages exhausted - enter a timeout, escalating in length with + # each repeat offense. + if state.offense_count < len(TIMEOUT_LADDER_SECONDS): + duration = TIMEOUT_LADDER_SECONDS[state.offense_count] + state.timeout_until = now + timedelta(seconds=duration) + state.next_allowed_at = None + state.offense_count += 1 + await db.flush() + return + + # Offended again after exhausting the entire timeout ladder - permanent. + state.offense_count += 1 + state.timeout_until = None + state.next_allowed_at = None + await repo.create_ban(ip_address, reason=action.value, banned_at=now) + await db.flush() + + +async def reset_on_success(db: AsyncSession, ip_address: str, action: ThrottleAction) -> None: + """Login-only convenience: a correct login ends that specific attack + scenario for this IP, so the stage/cooldown resets - but `offense_count` + (this IP's history) is never cleared by a success, only by an admin + unban.""" + repo = IpThrottleRepository(db) + state = await repo.get_state(ip_address, action) + if state is None: + return + state.attempt_count = 0 + state.next_allowed_at = None + await db.flush() diff --git a/apps/api/app/services/monitoring_service.py b/apps/api/app/services/monitoring_service.py new file mode 100644 index 0000000..82886bc --- /dev/null +++ b/apps/api/app/services/monitoring_service.py @@ -0,0 +1,66 @@ +"""Run-now / run-history service. Enqueuing goes through Celery +(`run_monitoring.delay`); this module only ever touches the `MonitoringRun` +row and ownership checks - the actual collection work happens in the task +(app/tasks/collection.py) and collection_service. +""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime, timedelta + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.config import Settings +from app.core.errors import NotFoundError, RateLimitedError +from app.models.enums import MonitoringRunTrigger +from app.models.monitoring_run import MonitoringRun +from app.repositories.monitoring_run_repository import MonitoringRunRepository +from app.services import company_service + + +async def enqueue_run_now( + db: AsyncSession, settings: Settings, user_id: uuid.UUID, company_id: uuid.UUID +) -> MonitoringRun: + company = await company_service.get_company(db, user_id, company_id) + run_repo = MonitoringRunRepository(db) + + # Idempotent: a company already mid-run returns that run rather than + # queuing a duplicate (spec: "unique job keys to prevent duplicate + # concurrent runs"). + active = await run_repo.get_active_for_company(company.id) + if active is not None: + return active + + since = datetime.now(UTC) - timedelta(days=1) + manual_count = await run_repo.count_manual_since(company.id, since) + if manual_count >= settings.max_manual_runs_per_day: + raise RateLimitedError( + f"This company has reached the maximum of {settings.max_manual_runs_per_day} " + "manual runs per day. Scheduled runs are unaffected." + ) + + run = await run_repo.create(company_id=company.id, trigger_type=MonitoringRunTrigger.MANUAL) + await db.commit() + + from app.tasks.collection import ( + run_monitoring, + ) # local import: keeps Celery out of API startup path + + run_monitoring.delay(str(run.id)) + return run + + +async def list_runs( + db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID +) -> list[MonitoringRun]: + await company_service.get_company(db, user_id, company_id) + return await MonitoringRunRepository(db).list_for_company(company_id) + + +async def get_run(db: AsyncSession, user_id: uuid.UUID, run_id: uuid.UUID) -> MonitoringRun: + run = await MonitoringRunRepository(db).get(run_id) + if run is None: + raise NotFoundError("Monitoring run not found") + await company_service.get_company(db, user_id, run.company_id) # ownership check + return run diff --git a/apps/api/app/services/notification_destination_service.py b/apps/api/app/services/notification_destination_service.py new file mode 100644 index 0000000..199a581 --- /dev/null +++ b/apps/api/app/services/notification_destination_service.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import uuid + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.errors import NotFoundError, ValidationAppError +from app.models.notification_destination import NotificationDestination +from app.repositories.company_repository import CompanyRepository +from app.repositories.notification_destination_repository import ( + NotificationDestinationRepository, +) +from app.schemas.notification_destination import ( + NotificationDestinationCreate, + NotificationDestinationUpdate, +) + + +async def list_destinations(db: AsyncSession, user_id: uuid.UUID) -> list[NotificationDestination]: + repo = NotificationDestinationRepository(db) + return await repo.list_for_user(user_id) + + +async def get_destination( + db: AsyncSession, user_id: uuid.UUID, destination_id: uuid.UUID +) -> NotificationDestination: + repo = NotificationDestinationRepository(db) + destination = await repo.get_for_user(destination_id, user_id) + if destination is None: + raise NotFoundError("Notification destination not found") + return destination + + +async def create_destination( + db: AsyncSession, user_id: uuid.UUID, payload: NotificationDestinationCreate +) -> NotificationDestination: + """Reuses an existing destination for the same (user, type, value) + rather than creating a duplicate row - this is the fix for the wizard + previously creating a fresh row per company even when the email/phone + was already registered. Either way, the result ends up linked to every + company in payload.company_ids.""" + company_repo = CompanyRepository(db) + for company_id in payload.company_ids: + if await company_repo.get_for_user(company_id, user_id) is None: + raise ValidationAppError(f"Company {company_id} not found") + + repo = NotificationDestinationRepository(db) + destination = await repo.find_by_value(user_id, payload.type, payload.destination_value) + if destination is None: + destination = await repo.create( + user_id=user_id, + type=payload.type, + destination_value=payload.destination_value, + minimum_severity=payload.minimum_severity, + enabled=payload.enabled, + ) + + for company_id in payload.company_ids: + await repo.link_company(destination.id, company_id) + + await db.commit() + refreshed = await repo.get_for_user(destination.id, user_id) + assert refreshed is not None + return refreshed + + +async def update_destination( + db: AsyncSession, + user_id: uuid.UUID, + destination_id: uuid.UUID, + payload: NotificationDestinationUpdate, +) -> NotificationDestination: + destination = await get_destination(db, user_id, destination_id) + updates = payload.model_dump(exclude_unset=True) + for field, value in updates.items(): + setattr(destination, field, value) + if field == "destination_value": + # Changing the destination value invalidates any prior verification. + destination.verified = False + await db.commit() + # Re-fetch (rather than db.refresh) so company_links stays eager-loaded - + # refresh() would expire it, and a bare lazy-load isn't safe under + # SQLAlchemy's async ORM without an active await context. + refreshed = await get_destination(db, user_id, destination_id) + return refreshed + + +async def delete_destination( + db: AsyncSession, user_id: uuid.UUID, destination_id: uuid.UUID +) -> None: + repo = NotificationDestinationRepository(db) + destination = await get_destination(db, user_id, destination_id) + await repo.delete(destination) + await db.commit() + + +async def unlink_company( + db: AsyncSession, user_id: uuid.UUID, destination_id: uuid.UUID, company_id: uuid.UUID +) -> None: + """Removes this one company's link, not the destination itself - a + destination can be shared across companies (see create_destination's + dedup-by-value). If this was its last remaining link, it's an orphan + now and delete_orphaned_for_user removes it outright, same as already + happens when a company itself is deleted.""" + await get_destination(db, user_id, destination_id) # ownership check + repo = NotificationDestinationRepository(db) + await repo.unlink_company(destination_id, company_id) + await repo.delete_orphaned_for_user(user_id) + await db.commit() diff --git a/apps/api/app/services/report_markdown.py b/apps/api/app/services/report_markdown.py new file mode 100644 index 0000000..ed395c6 --- /dev/null +++ b/apps/api/app/services/report_markdown.py @@ -0,0 +1,117 @@ +"""Renders a ReportContent (structured, from the LLM) into the Markdown +document the API/UI serve alongside the JSON - see spec section 6G for the +16-section layout this follows. +""" + +from __future__ import annotations + +from app.prompts.report_generation import Finding, InferredProject, ReportContent + + +def _render_findings(findings: list[Finding]) -> str: + if not findings: + return "_No findings for this section from the current evidence._\n" + lines = [] + for f in findings: + lines.append(f"- **{f.headline}** _(confidence: {f.confidence.value.replace('_', ' ')})_") + lines.append(f" {f.summary}") + if f.date: + lines.append(f" _Date: {f.date}_") + return "\n".join(lines) + "\n" + + +def _render_projects(projects: list[InferredProject]) -> str: + if not projects: + return "_No inferred strategic projects from the current evidence._\n" + lines = [] + for p in projects: + lines.append( + f"- **{p.project_name}** _({p.status.value.replace('_', ' ')}, confidence {p.confidence:.0%})_" + ) + lines.append(f" {p.summary}") + if p.alternative_explanations: + lines.append(f" Alternative explanations: {'; '.join(p.alternative_explanations)}") + return "\n".join(lines) + "\n" + + +def _render_list(items: list[str]) -> str: + if not items: + return "_None noted._\n" + return "\n".join(f"- {item}" for item in items) + "\n" + + +def render_report_markdown( + content: ReportContent, + *, + company_name: str, + generated_at: str, + model_provider: str, + model_name: str, + sources: list[dict], +) -> str: + parts = [ + f"# Competitive Intelligence Report: {company_name}", + f"_Generated {generated_at} · {model_provider}/{model_name}_", + "", + "## 1. Executive Summary", + content.executive_summary, + "", + "## 2. Company Overview", + content.company_overview, + "", + "## 3. Products and Service Landscape", + _render_findings(content.products_and_services), + "## 4. Recent Developments", + _render_findings(content.recent_developments), + "## 5. Strategic Initiatives", + _render_findings(content.strategic_initiatives), + "## 6. Key Project Signals", + _render_projects(content.key_inferred_projects), + "## 7. Competitive Positioning", + content.market_positioning, + "", + content.competitor_comparison, + "", + "## 8. SWOT Analysis", + "**Strengths**", + _render_list(content.swot.strengths), + "**Weaknesses**", + _render_list(content.swot.weaknesses), + "**Opportunities**", + _render_list(content.swot.opportunities), + "**Threats**", + _render_list(content.swot.threats), + "## 9. Hiring Signals", + _render_findings(content.hiring_signals), + "## 10. Product and Technology Signals", + _render_findings(content.technology_signals + content.patent_signals), + "## 11. Customer Sentiment", + content.customer_sentiment, + "", + "## 12. Financial and Regulatory Signals", + _render_findings(content.financial_signals + content.regulatory_and_legal_signals), + "## 13. Risks and Opportunities", + "**Risks**", + _render_list(content.risks), + "**Opportunities**", + _render_list(content.opportunities), + "## 14. Important Unknowns", + _render_list(content.unknowns_and_missing_data), + "## 15. Sources", + _render_sources(sources), + "## 16. Methodology and Limitations", + content.methodology, + "", + content.limitations, + ] + return "\n".join(str(p) for p in parts) + + +def _render_sources(sources: list[dict]) -> str: + if not sources: + return "_No sources recorded for this report._\n" + lines = [ + f"- [{s.get('title') or s.get('url')}]({s.get('url')}) — retrieved {s.get('retrieved_date')}" + for s in sources + ] + return "\n".join(lines) + "\n" diff --git a/apps/api/app/services/report_service.py b/apps/api/app/services/report_service.py new file mode 100644 index 0000000..d6e6fda --- /dev/null +++ b/apps/api/app/services/report_service.py @@ -0,0 +1,188 @@ +"""Generates and persists a company's CI report from accumulated evidence. + +Evidence gathering happens here, not inside the LLM prompt module - the +model only ever sees data this pipeline actually collected (recent +SourceDocuments + DetectedChanges), so it cannot introduce facts we never +stored. See app/prompts/report_generation.py for the schema/prompt itself. +""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.analysis.llm.base import LLMProvider +from app.core.config import Settings +from app.core.errors import NotFoundError +from app.models.company import Company +from app.models.detected_change import DetectedChange +from app.models.enums import EnrichmentStatus, ReportType, SourceStatus +from app.models.report import Report +from app.models.source import Source +from app.models.source_document import SourceDocument +from app.prompts.report_generation import generate_report +from app.repositories.company_enrichment_repository import CompanyEnrichmentRepository +from app.repositories.report_repository import ReportRepository +from app.services import company_service +from app.services.report_markdown import render_report_markdown + +_MAX_DOCUMENTS = 40 +_MAX_CHANGES = 20 + + +async def _gather_documents(db: AsyncSession, company_id: uuid.UUID) -> list[dict]: + result = await db.execute( + select(SourceDocument, Source.source_type) + .join(Source, Source.id == SourceDocument.source_id) + .where(SourceDocument.company_id == company_id) + .order_by(SourceDocument.retrieved_date.desc()) + .limit(_MAX_DOCUMENTS) + ) + return [ + { + "id": str(doc.id), + "title": doc.title, + "url": doc.url, + "excerpt": doc.content_text[:500], + "source_type": source_type.value, + "retrieved_date": doc.retrieved_date.isoformat(), + } + for doc, source_type in result.all() + ] + + +async def _gather_changes(db: AsyncSession, company_id: uuid.UUID) -> list[dict]: + result = await db.execute( + select(DetectedChange) + .where(DetectedChange.company_id == company_id) + .order_by(DetectedChange.created_at.desc()) + .limit(_MAX_CHANGES) + ) + return [ + { + "id": str(c.id), + "summary": c.summary, + "change_type": c.change_type.value, + "severity": c.severity.value, + "confidence_score": c.confidence_score, + "created_at": c.created_at.isoformat(), + } + for c in result.scalars().all() + ] + + +async def _gather_failed_source_names(db: AsyncSession, company_id: uuid.UUID) -> list[str]: + result = await db.execute( + select(Source.name).where( + Source.company_id == company_id, Source.status != SourceStatus.ACTIVE + ) + ) + return list(result.scalars().all()) + + +async def _gather_enrichment(db: AsyncSession, company_id: uuid.UUID) -> dict | None: + enrichment = await CompanyEnrichmentRepository(db).get_for_company(company_id) + if enrichment is None or enrichment.status == EnrichmentStatus.FAILED: + return None + return enrichment.data + + +def _model_name(settings: Settings, llm: LLMProvider) -> str: + if llm.provider_name == "anthropic": + return settings.anthropic_model + if llm.provider_name == "ollama": + return settings.ollama_model + return "mock" + + +async def generate_and_persist_report( + db: AsyncSession, + settings: Settings, + llm: LLMProvider, + company: Company, + *, + report_type: ReportType, + monitoring_run_id: uuid.UUID | None = None, +) -> Report: + documents = await _gather_documents(db, company.id) + changes = await _gather_changes(db, company.id) + failed_sources = await _gather_failed_source_names(db, company.id) + enrichment = await _gather_enrichment(db, company.id) + + content = await generate_report( + llm, + company_name=company.name, + company_aliases=[a.alias for a in company.aliases], + competitors=[c.name for c in company.competitors], + monitoring_focus=company.monitoring_focus, + industry=company.industry, + documents=documents, + detected_changes=changes, + sources_failed=failed_sources, + description=company.description, + official_website=company.official_website, + headquarters=company.headquarters, + country=company.country, + region=company.region, + public_identifiers=company.public_identifiers, + enrichment=enrichment, + ) + + generated_at = datetime.now(UTC) + model_name = _model_name(settings, llm) + markdown = render_report_markdown( + content, + company_name=company.name, + generated_at=generated_at.isoformat(), + model_provider=llm.provider_name, + model_name=model_name, + sources=[ + {"title": d["title"], "url": d["url"], "retrieved_date": d["retrieved_date"]} + for d in documents + ], + ) + + report = Report( + company_id=company.id, + monitoring_run_id=monitoring_run_id, + report_type=report_type, + title=f"{company.name} — Competitive Intelligence Report", + executive_summary=content.executive_summary, + structured_report=content.model_dump(mode="json"), + markdown_content=markdown, + model_provider=llm.provider_name, + model_name=model_name, + ) + db.add(report) + await db.commit() + await db.refresh(report) + return report + + +async def list_reports(db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID) -> list[Report]: + await company_service.get_company(db, user_id, company_id) + return await ReportRepository(db).list_for_company(company_id) + + +async def get_report(db: AsyncSession, user_id: uuid.UUID, report_id: uuid.UUID) -> Report: + report = await ReportRepository(db).get(report_id) + if report is None: + raise NotFoundError("Report not found") + await company_service.get_company(db, user_id, report.company_id) # ownership check + return report + + +async def generate_report_now( + db: AsyncSession, + settings: Settings, + llm: LLMProvider, + user_id: uuid.UUID, + company_id: uuid.UUID, +) -> Report: + company = await company_service.get_company(db, user_id, company_id) + return await generate_and_persist_report( + db, settings, llm, company, report_type=ReportType.MANUAL + ) diff --git a/apps/api/app/services/scheduling.py b/apps/api/app/services/scheduling.py new file mode 100644 index 0000000..d66b351 --- /dev/null +++ b/apps/api/app/services/scheduling.py @@ -0,0 +1,83 @@ +"""Schedule validation + next-run computation. + +Shared between company creation/monitor-config updates (this phase) and the +Celery Beat dynamic schedule sync (Phase 5) so both paths agree on what a +valid schedule is and when it next fires. +""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError + +from croniter import CroniterBadCronError, croniter + +from app.core.errors import ValidationAppError +from app.models.enums import FREQUENCY_MINUTES, MonitoringFrequency + + +def validate_and_compute_next_run( + *, + frequency_type: MonitoringFrequency, + interval_minutes: int | None, + cron_expression: str | None, + tz_name: str, + minimum_interval_minutes: int, + from_time: datetime | None = None, +) -> datetime: + now = from_time or datetime.now(UTC) + + try: + tzinfo = ZoneInfo(tz_name) + except ZoneInfoNotFoundError as exc: + raise ValidationAppError(f"Unknown timezone: {tz_name}") from exc + + if frequency_type == MonitoringFrequency.CUSTOM: + return _compute_custom_next_run( + interval_minutes=interval_minutes, + cron_expression=cron_expression, + tzinfo=tzinfo, + minimum_interval_minutes=minimum_interval_minutes, + now=now, + ) + + frequency_minutes = FREQUENCY_MINUTES[frequency_type] + if frequency_minutes < minimum_interval_minutes: + raise ValidationAppError( + f"{frequency_type.value} monitoring is more frequent than the minimum allowed " + f"interval of {minimum_interval_minutes} minutes" + ) + return now + timedelta(minutes=frequency_minutes) + + +def _compute_custom_next_run( + *, + interval_minutes: int | None, + cron_expression: str | None, + tzinfo: ZoneInfo, + minimum_interval_minutes: int, + now: datetime, +) -> datetime: + if interval_minutes is not None: + if interval_minutes < minimum_interval_minutes: + raise ValidationAppError( + f"Custom interval must be at least {minimum_interval_minutes} minutes" + ) + return now + timedelta(minutes=interval_minutes) + + if cron_expression: + try: + base = now.astimezone(tzinfo) + next_local = croniter(cron_expression, base).get_next(datetime) + except (CroniterBadCronError, ValueError) as exc: + raise ValidationAppError(f"Invalid cron expression: {cron_expression}") from exc + + next_utc = next_local.astimezone(UTC) + if (next_utc - now) < timedelta(minutes=minimum_interval_minutes): + raise ValidationAppError( + "Custom schedule resolves to less than the minimum allowed interval of " + f"{minimum_interval_minutes} minutes" + ) + return next_utc + + raise ValidationAppError("Custom frequency requires either interval_minutes or cron_expression") diff --git a/apps/api/app/services/security_email_service.py b/apps/api/app/services/security_email_service.py new file mode 100644 index 0000000..2ccff30 --- /dev/null +++ b/apps/api/app/services/security_email_service.py @@ -0,0 +1,77 @@ +"""Transactional account-security email (verification codes, password +reset codes, lockout notices) - deliberately separate from the alert- +notification path in app/services/alert_service.py, which dispatches to +user-configured destinations via app/notifications/factory.py. This is +always sent to the account's own registered email, from a distinct sender +identity (settings.resend_security_from_email, e.g. security@ciagent.org +vs. alerts@ciagent.org). + +Picks the Resend HTTP API provider when RESEND_API_KEY is configured, else +falls back to the existing SmtpEmailProvider. `resolve_provider` is also +reused by app.services.unban_service for the admin unban-request +notification, which has the same "Resend if configured, else SMTP" needs. +""" + +from __future__ import annotations + +from app.core.config import Settings +from app.notifications.base import DeliveryResult, NotificationMessage +from app.notifications.resend_email import ResendEmailProvider +from app.notifications.smtp_email import SmtpEmailProvider + + +def resolve_provider(settings: Settings): + if settings.resend_api_key: + return ResendEmailProvider(settings) + return SmtpEmailProvider(settings) + + +async def send_verification_code_email(settings: Settings, to: str, code: str) -> DeliveryResult: + message = NotificationMessage( + destination_value=to, + subject="Verify your CI Agent account", + body_text=( + f"Your verification code is {code}.\n\n" + "This code expires in 36 hours. If you didn't request this, you can ignore this email." + ), + body_html=( + f"

Your verification code is {code}.

" + "

This code expires in 36 hours. If you didn't request this, you can ignore this email.

" + ), + ) + return await resolve_provider(settings).send(message) + + +async def send_password_reset_email(settings: Settings, to: str, code: str) -> DeliveryResult: + message = NotificationMessage( + destination_value=to, + subject="Reset your CI Agent password", + body_text=( + f"Your password reset code is {code}.\n\n" + "This code expires in 36 hours. If you didn't request this, you can ignore this email " + "and your password will stay unchanged." + ), + body_html=( + f"

Your password reset code is {code}.

" + "

This code expires in 36 hours. If you didn't request this, you can ignore this " + "email and your password will stay unchanged.

" + ), + ) + return await resolve_provider(settings).send(message) + + +async def send_account_locked_email(settings: Settings, to: str) -> DeliveryResult: + reset_link = f"{settings.frontend_url}/forgot-password" + message = NotificationMessage( + destination_value=to, + subject="Your CI Agent account was locked", + body_text=( + "Your account was locked after repeated failed login attempts.\n\n" + f"To unlock it, reset your password: {reset_link}" + ), + body_html=( + "

Your account was locked after repeated failed login attempts.

" + f'

To unlock it, reset your password.

' + ), + ) + return await resolve_provider(settings).send(message) diff --git a/apps/api/app/services/snapshot_service.py b/apps/api/app/services/snapshot_service.py new file mode 100644 index 0000000..8569b29 --- /dev/null +++ b/apps/api/app/services/snapshot_service.py @@ -0,0 +1,20 @@ +"""Read-only snapshot history for a company. Snapshots themselves are only +ever written by collection_service.py during a monitoring run - this module +just lists what's already there, ownership-checked.""" + +from __future__ import annotations + +import uuid + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.snapshot import Snapshot +from app.repositories.source_repository import SnapshotRepository +from app.services import company_service + + +async def list_snapshots( + db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID +) -> list[Snapshot]: + await company_service.get_company(db, user_id, company_id) + return await SnapshotRepository(db).list_for_company(company_id) diff --git a/apps/api/app/services/source_service.py b/apps/api/app/services/source_service.py new file mode 100644 index 0000000..776a94d --- /dev/null +++ b/apps/api/app/services/source_service.py @@ -0,0 +1,115 @@ +"""Source CRUD + the ad-hoc "test this source" action. Collection +orchestration itself (persisting documents/snapshots) lives in +collection_service.py; this module is the ownership-checked API-facing +layer on top of it. +""" + +from __future__ import annotations + +import uuid + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.config import Settings +from app.core.errors import NotFoundError +from app.models.company import Company +from app.models.source import Source +from app.repositories.company_repository import CompanyRepository +from app.repositories.source_repository import SourceRepository +from app.schemas.source import SourceCreate, SourceUpdate +from app.services.collection_service import CollectionResult, collect_source +from app.services.scheduling import validate_and_compute_next_run + + +async def _get_owned_company( + db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID +) -> Company: + company = await CompanyRepository(db).get_for_user(company_id, user_id) + if company is None: + raise NotFoundError("Company not found") + return company + + +async def list_sources(db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID) -> list[Source]: + await _get_owned_company(db, user_id, company_id) + return await SourceRepository(db).list_for_company(company_id) + + +async def create_source( + db: AsyncSession, user_id: uuid.UUID, company_id: uuid.UUID, payload: SourceCreate +) -> Source: + await _get_owned_company(db, user_id, company_id) + source = await SourceRepository(db).create( + company_id=company_id, + source_type=payload.source_type, + name=payload.name, + base_url=payload.base_url, + ) + await db.commit() + return source + + +async def _get_owned_source(db: AsyncSession, user_id: uuid.UUID, source_id: uuid.UUID) -> Source: + source = await SourceRepository(db).get_for_user(source_id, user_id) + if source is None: + raise NotFoundError("Source not found") + return source + + +async def update_source( + db: AsyncSession, + settings: Settings, + user_id: uuid.UUID, + source_id: uuid.UUID, + payload: SourceUpdate, +) -> Source: + source = await _get_owned_source(db, user_id, source_id) + updates = payload.model_dump(exclude_unset=True) + + schedule_changed = any( + key in updates for key in ("frequency_type", "interval_minutes", "cron_expression") + ) + if schedule_changed: + frequency_type = updates.get("frequency_type", source.frequency_type) + if frequency_type is not None: + company = await CompanyRepository(db).get_for_user(source.company_id, user_id) + tz_name = ( + company.monitor_configuration.timezone + if company is not None and company.monitor_configuration is not None + else "UTC" + ) + # Validate only - the actual next_check is computed for real the + # next time this source is collected (tasks/collection.py), same + # as a brand-new source. Resetting it to None here means a + # changed override takes effect on the very next scheduler tick + # rather than waiting out whatever cadence was previously set. + validate_and_compute_next_run( + frequency_type=frequency_type, + interval_minutes=updates.get("interval_minutes", source.interval_minutes), + cron_expression=updates.get("cron_expression", source.cron_expression), + tz_name=tz_name, + minimum_interval_minutes=settings.minimum_monitoring_interval_minutes, + ) + source.next_check = None + + for field, value in updates.items(): + setattr(source, field, value) + await db.commit() + await db.refresh(source) + return source + + +async def delete_source(db: AsyncSession, user_id: uuid.UUID, source_id: uuid.UUID) -> None: + source = await _get_owned_source(db, user_id, source_id) + await SourceRepository(db).delete(source) + await db.commit() + + +async def test_source( + db: AsyncSession, settings: Settings, user_id: uuid.UUID, source_id: uuid.UUID +) -> CollectionResult: + source = await _get_owned_source(db, user_id, source_id) + company = await CompanyRepository(db).get_for_user(source.company_id, user_id) + if company is None: # pragma: no cover - defensive, implied by _get_owned_source + raise NotFoundError("Company not found") + return await collect_source(db, settings, source, company) diff --git a/apps/api/app/services/system_secret_service.py b/apps/api/app/services/system_secret_service.py new file mode 100644 index 0000000..df6d219 --- /dev/null +++ b/apps/api/app/services/system_secret_service.py @@ -0,0 +1,97 @@ +"""Server-wide secrets an admin can configure from the Settings page +instead of only via .env - today just the Cloudflare Turnstile site key +and secret (app/services/turnstile_service.py). Storage is encrypted at +rest (app/core/crypto.py). Unlike per-user API keys +(user_api_key_service.py), there's exactly one value per key, shared by +the whole app - visible/editable only to admins (see +app/api/v1/system.py's require_admin gate), never per-user. +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.config import Settings +from app.core.crypto import decrypt_secret, encrypt_secret +from app.models.enums import SecurityEventType, SystemSecretKey +from app.repositories.system_secret_repository import SystemSecretRepository +from app.repositories.user_security_event_repository import UserSecurityEventRepository + + +@dataclass(frozen=True) +class SecretMeta: + label: str + settings_field: str + + +META: dict[SystemSecretKey, SecretMeta] = { + SystemSecretKey.TURNSTILE_SITE_KEY: SecretMeta( + label="Cloudflare Turnstile Site Key", settings_field="turnstile_site_key" + ), + SystemSecretKey.TURNSTILE_SECRET: SecretMeta( + label="Cloudflare Turnstile Secret Key", settings_field="turnstile_secret" + ), +} + + +async def list_status(db: AsyncSession, settings: Settings) -> list[dict[str, Any]]: + repo = SystemSecretRepository(db) + stored = {row.key: row for row in await repo.list_all()} + + results: list[dict[str, Any]] = [] + for key, meta in META.items(): + row = stored.get(key) + value = decrypt_secret(row.encrypted_value, settings) if row is not None else None + results.append( + { + "key": key.value, + "label": meta.label, + "configured": row is not None, + "value": value, + } + ) + return results + + +async def set_secret( + db: AsyncSession, + key: SystemSecretKey, + plaintext_value: str, + settings: Settings, + *, + admin_user_id: uuid.UUID, + client_ip: str, +) -> None: + """A blank value clears the stored override, falling back to the + server's .env-configured value again. Every update - including a clear - + is logged to the acting admin's own Account activity, since this is a + security-sensitive, app-wide change (Turnstile keys today).""" + repo = SystemSecretRepository(db) + stripped = plaintext_value.strip() + if not stripped: + await repo.delete(key) + else: + await repo.upsert(key, encrypt_secret(stripped, settings)) + await UserSecurityEventRepository(db).create( + user_id=admin_user_id, + event_type=SecurityEventType.SERVER_SECRET_UPDATED, + ip_address=client_ip, + ) + await db.commit() + + +async def get_effective_settings(db: AsyncSession, settings: Settings) -> Settings: + """A copy of the global settings with any admin-stored secret + substituted in for the matching field - keys with no stored override + keep using the server's .env-configured default.""" + rows = await SystemSecretRepository(db).list_all() + if not rows: + return settings + overrides = { + META[row.key].settings_field: decrypt_secret(row.encrypted_value, settings) for row in rows + } + return settings.model_copy(update=overrides) diff --git a/apps/api/app/services/turnstile_service.py b/apps/api/app/services/turnstile_service.py new file mode 100644 index 0000000..61ed398 --- /dev/null +++ b/apps/api/app/services/turnstile_service.py @@ -0,0 +1,71 @@ +"""Cloudflare Turnstile server-side verification. Canonical siteverify +contract per Cloudflare's own reference: POST {secret, response, remoteip} +to the fixed challenges.cloudflare.com endpoint, check `success === true`. +Fails closed on any network error or non-2xx response - a Cloudflare outage +must never silently let requests through unverified. + +One deliberate exception: a misconfigured *secret* (typo'd/invalid, as +opposed to a genuinely bad/expired user token) fails open instead. Cloudflare +reports this distinctly via `error-codes` (`invalid-input-secret` / +`missing-input-secret`) rather than as an ambiguous non-2xx/network failure, +so it's a real, detectable "the admin's config is broken" signal, not "we +couldn't tell if this passed." Locking out every real register/login/ +password-reset attempt because of an admin's own copy-paste mistake is a +worse outcome than briefly running with reduced bot protection - especially +since Turnstile is one layer among several here (see SECURITY.md's IP +throttle/ban and account-lockout layers, which stay fully active either +way). +""" + +from __future__ import annotations + +import httpx + +from app.core.config import Settings +from app.core.logging import get_logger + +logger = get_logger(__name__) + +_SITEVERIFY_URL = "https://challenges.cloudflare.com/turnstile/v0/siteverify" + +# Cloudflare's own error-code vocabulary for a broken *secret* - distinct +# from user-token-level codes like invalid-input-response/timeout-or- +# duplicate/missing-input-response, which are legitimate rejections and +# must keep failing closed. +_SECRET_MISCONFIGURED_CODES = {"invalid-input-secret", "missing-input-secret"} + + +async def verify_turnstile(token: str, remote_ip: str, settings: Settings) -> bool: + try: + async with httpx.AsyncClient(timeout=10) as client: + response = await client.post( + _SITEVERIFY_URL, + data={ + "secret": settings.turnstile_secret, + "response": token, + "remoteip": remote_ip, + }, + ) + response.raise_for_status() + data = response.json() + if data.get("success") is True: + return True + + error_codes = set(data.get("error-codes") or []) + if error_codes & _SECRET_MISCONFIGURED_CODES: + logger.error( + "turnstile_secret_misconfigured", + error_codes=sorted(error_codes), + ) + return True # fail open - this is a config problem, not the caller's + return False + except Exception as exc: # noqa: BLE001 - fail closed on any error + logger.warning("turnstile_verify_failed", error=str(exc)) + return False + + +def turnstile_required(is_localhost: bool, settings: Settings) -> bool: + """Skipped entirely for a loopback caller, or when no secret is + configured at all (matches this app's usual optional-provider + convention - e.g. NinjaPear/USPTO/Brave all no-op when unset).""" + return not is_localhost and bool(settings.turnstile_secret) diff --git a/apps/api/app/services/unban_service.py b/apps/api/app/services/unban_service.py new file mode 100644 index 0000000..7a2ff95 --- /dev/null +++ b/apps/api/app/services/unban_service.py @@ -0,0 +1,103 @@ +"""Public unban-request intake + admin resolution. One request per IP per +24h (enforced here), notifying every admin account's own email (Resend if +configured, else SMTP - same provider selection as the rest of security +email, see security_email_service.resolve_provider) and via the existing +admin-only Redis log feed (app/core/logging.py), so it surfaces in the +Settings page's Logging box too. +""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.config import Settings +from app.core.errors import ConflictError, NotFoundError, RateLimitedError +from app.core.logging import get_logger +from app.models.ip_ban import IpBan +from app.notifications.base import NotificationMessage +from app.repositories.ip_throttle_repository import IpThrottleRepository +from app.repositories.unban_request_repository import UnbanRequestRepository +from app.repositories.user_repository import UserRepository +from app.services.security_email_service import resolve_provider + +logger = get_logger(__name__) + +UNBAN_REQUEST_COOLDOWN_HOURS = 24 + + +async def submit_unban_request( + db: AsyncSession, settings: Settings, ip_address: str, message: str | None +) -> None: + repo = UnbanRequestRepository(db) + if await repo.within_cooldown(ip_address, UNBAN_REQUEST_COOLDOWN_HOURS): + raise RateLimitedError( + f"Only one unban request is allowed per {UNBAN_REQUEST_COOLDOWN_HOURS} hours." + ) + + await repo.create(ip_address, message) + await db.commit() + + logger.warning("unban_request_received", ip=ip_address, message=message or "") + + admin_emails = await UserRepository(db).list_admin_emails() + if not admin_emails: + logger.warning("unban_request_no_admin_to_notify", ip=ip_address) + return + + provider = resolve_provider(settings) + for admin_email in admin_emails: + email_message = NotificationMessage( + destination_value=admin_email, + subject=f"Unban request from {ip_address}", + body_text=f"IP: {ip_address}\n\nMessage:\n{message or '(none)'}", + ) + await provider.send(email_message) + + +async def list_ip_bans(db: AsyncSession) -> list[IpBan]: + return await IpThrottleRepository(db).list_bans() + + +async def unban_ip(db: AsyncSession, ip_address: str) -> bool: + repo = IpThrottleRepository(db) + cleared = await repo.clear_ban_and_state(ip_address) + await db.commit() + return cleared + + +async def ban_ip(db: AsyncSession, ip_address: str, reason: str = "manual_admin_ban") -> IpBan: + """Admin-initiated ban, bypassing the usual offense-count escalation + ladder (ip_throttle_service) entirely - a deliberate manual override, + not something the automated abuse-detection path produces.""" + repo = IpThrottleRepository(db) + if await repo.get_ban(ip_address) is not None: + raise ConflictError(f"{ip_address} is already banned.") + ban = await repo.create_ban(ip_address, reason, datetime.now(UTC)) + await db.commit() + return ban + + +async def accept_unban_request(db: AsyncSession, request_id: uuid.UUID) -> None: + """Unbans the requester's IP and clears the request from the pending + queue - a real pardon (see clear_ban_and_state), not just acknowledging + the request was read.""" + repo = UnbanRequestRepository(db) + request = await repo.get(request_id) + if request is None: + raise NotFoundError("Unban request not found.") + await IpThrottleRepository(db).clear_ban_and_state(request.ip_address) + await repo.delete(request_id) + await db.commit() + + +async def reject_unban_request(db: AsyncSession, request_id: uuid.UUID) -> None: + """Dismisses the request without touching the ban - the IP stays + banned.""" + repo = UnbanRequestRepository(db) + if await repo.get(request_id) is None: + raise NotFoundError("Unban request not found.") + await repo.delete(request_id) + await db.commit() diff --git a/apps/api/app/services/user_api_key_service.py b/apps/api/app/services/user_api_key_service.py new file mode 100644 index 0000000..94cf1d0 --- /dev/null +++ b/apps/api/app/services/user_api_key_service.py @@ -0,0 +1,132 @@ +"""Per-user API keys - each user can supply their own key for a provider, +used in place of the server's global .env-configured key for their own +requests (see get_effective_settings, and its call sites in companies.py, +reports.py, tasks/collection.py, tasks/enrichment.py, and +collection_service.to_company_context for USPTO patents). + +Storage is encrypted at rest (app/core/crypto.py). A key is only ever +decrypted for the owning user's own list/set calls or to actually place a +provider call on their behalf - never exposed to any other user, admin or +not (this is a deliberately different visibility model from the +admin-only, localhost-only *server* key box in app/api/v1/system.py). +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.config import Settings +from app.core.crypto import decrypt_secret, encrypt_secret +from app.models.enums import ApiKeyProvider, SecurityEventType +from app.repositories.user_api_key_repository import UserApiKeyRepository +from app.repositories.user_security_event_repository import UserSecurityEventRepository + + +@dataclass(frozen=True) +class ProviderMeta: + label: str + settings_field: str + credits_note: str + free: bool = False + requires_government_id: bool = False + + +PROVIDER_META: dict[ApiKeyProvider, ProviderMeta] = { + ApiKeyProvider.ANTHROPIC: ProviderMeta( + label="Anthropic", + settings_field="anthropic_api_key", + credits_note="Anthropic doesn't expose a credit/usage-balance API.", + ), + ApiKeyProvider.BRAVE_SEARCH: ProviderMeta( + label="Brave Search", + settings_field="brave_search_api_key", + credits_note=( + "Brave Search API has no metered balance endpoint (plan-based, not prepaid credits)." + ), + ), + ApiKeyProvider.NINJAPEAR: ProviderMeta( + label="NinjaPear", + settings_field="ninjapear_api_key", + credits_note="Credit balance shown in System configuration below.", + ), + ApiKeyProvider.USPTO: ProviderMeta( + label="USPTO", + settings_field="uspto_api_key", + credits_note="Free - USPTO Open Data Portal has no usage limit or credit cost.", + free=True, + requires_government_id=True, + ), +} + + +async def list_status( + db: AsyncSession, user_id: uuid.UUID, settings: Settings +) -> list[dict[str, Any]]: + """Never fetches a live NinjaPear credit balance itself - the frontend + sources that number from /system/status's own already-fetched + ninjapear_credit_balance (see the Settings page's System configuration + box) instead of this endpoint making a second, redundant live call.""" + repo = UserApiKeyRepository(db) + stored = {row.provider: row for row in await repo.list_for_user(user_id)} + + results: list[dict[str, Any]] = [] + for provider, meta in PROVIDER_META.items(): + row = stored.get(provider) + value = decrypt_secret(row.encrypted_key, settings) if row is not None else None + results.append( + { + "provider": provider.value, + "label": meta.label, + "configured": row is not None, + "value": value, + "credits": None, + "credits_note": meta.credits_note, + "free": meta.free, + "requires_government_id": meta.requires_government_id, + } + ) + return results + + +async def set_key( + db: AsyncSession, + user_id: uuid.UUID, + provider: ApiKeyProvider, + plaintext_key: str, + settings: Settings, + *, + client_ip: str, +) -> None: + """An empty/blank key clears the user's override, falling back to the + server's global key for that provider again. Every update - including a + clear - is logged to this same user's own Account activity.""" + repo = UserApiKeyRepository(db) + stripped = plaintext_key.strip() + if not stripped: + await repo.delete(user_id, provider) + else: + await repo.upsert(user_id, provider, encrypt_secret(stripped, settings)) + await UserSecurityEventRepository(db).create( + user_id=user_id, event_type=SecurityEventType.API_KEY_UPDATED, ip_address=client_ip + ) + await db.commit() + + +async def get_effective_settings( + db: AsyncSession, user_id: uuid.UUID, settings: Settings +) -> Settings: + """A copy of the global settings with any of this user's own stored + keys substituted in for the matching field - providers they haven't + set their own key for keep using the server's global default.""" + rows = await UserApiKeyRepository(db).list_for_user(user_id) + if not rows: + return settings + overrides = { + PROVIDER_META[row.provider].settings_field: decrypt_secret(row.encrypted_key, settings) + for row in rows + } + return settings.model_copy(update=overrides) diff --git a/apps/api/app/tasks/__init__.py b/apps/api/app/tasks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/app/tasks/base.py b/apps/api/app/tasks/base.py new file mode 100644 index 0000000..156f0b2 --- /dev/null +++ b/apps/api/app/tasks/base.py @@ -0,0 +1,33 @@ +"""Bridges Celery's sync task functions into the app's async service layer. + +In production, a Celery worker process has no asyncio event loop running +when a task executes, so a plain `asyncio.run(...)` is enough. But with +`CELERY_TASK_ALWAYS_EAGER=true` (tests, and `.delay()` called from inside an +async FastAPI route handler), the task body runs synchronously *inside* the +caller's already-running event loop, and `asyncio.run()` refuses to nest. +`run_async_task` handles both: it uses `asyncio.run()` directly when no loop +is running, and falls back to a dedicated thread with its own loop when one is. +""" + +from __future__ import annotations + +import asyncio +import contextvars +from collections.abc import Coroutine +from concurrent.futures import ThreadPoolExecutor + + +def run_async_task[T](coro: Coroutine[object, object, T]) -> T: + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(coro) + + # ThreadPoolExecutor does not copy contextvars into the worker thread by + # default, which would silently drop the structlog correlation id + # (request_id/run_id/task_id - see core/logging.py) bound by the caller. + # Capturing the current context explicitly and running the executor call + # through it keeps those log fields intact even on this fallback path. + ctx = contextvars.copy_context() + with ThreadPoolExecutor(max_workers=1) as executor: + return executor.submit(ctx.run, asyncio.run, coro).result() diff --git a/apps/api/app/tasks/celery_app.py b/apps/api/app/tasks/celery_app.py new file mode 100644 index 0000000..2cad646 --- /dev/null +++ b/apps/api/app/tasks/celery_app.py @@ -0,0 +1,60 @@ +"""Celery application: broker/backend config, queue routing, and the Beat +schedule. Task modules are plain sync functions that bridge into the async +service layer via `asyncio.run` (see tasks/base.py) - Celery's worker model +is synchronous/prefork, while the rest of the app is async SQLAlchemy. +""" + +from __future__ import annotations + +from celery import Celery +from celery.schedules import crontab + +from app.core.config import get_settings +from app.core.logging import configure_logging + +settings = get_settings() +# The worker process never went through main.py's bootstrap, so without +# this call every logger.* call here ran through structlog's unconfigured +# default - no secret redaction, no capture into the Settings-page log feed. +# This module is imported once per worker process (Celery's `include=` +# above), so this runs exactly where main.py's equivalent call does. +configure_logging(settings) + +celery_app = Celery( + "ci_agent", + broker=settings.redis_url, + backend=settings.redis_url, + include=[ + "app.tasks.collection", + "app.tasks.scheduler", + "app.tasks.maintenance", + "app.tasks.enrichment", + ], +) + +celery_app.conf.update( + task_always_eager=settings.celery_task_always_eager, + task_eager_propagates=settings.celery_task_always_eager, + task_default_queue="default", + task_routes={ + "app.tasks.collection.*": {"queue": "collection"}, + "app.tasks.scheduler.*": {"queue": "default"}, + "app.tasks.analysis.*": {"queue": "analysis"}, + "app.tasks.notifications.*": {"queue": "notifications"}, + "app.tasks.maintenance.*": {"queue": "maintenance"}, + "app.tasks.enrichment.*": {"queue": "enrichment"}, + }, + beat_schedule={ + "sync-monitoring-schedules": { + "task": "app.tasks.scheduler.sync_schedules", + "schedule": 60.0, + }, + "purge-expired-data": { + "task": "app.tasks.maintenance.purge_expired_data", + "schedule": crontab(hour=3, minute=0), + }, + }, + timezone="UTC", + worker_hijack_root_logger=False, + task_track_started=True, +) diff --git a/apps/api/app/tasks/collection.py b/apps/api/app/tasks/collection.py new file mode 100644 index 0000000..7b56cfe --- /dev/null +++ b/apps/api/app/tasks/collection.py @@ -0,0 +1,265 @@ +"""Celery task that executes one MonitoringRun: discovers sources on a +company's first run, collects every active source, and records progress on +the MonitoringRun row as it goes so the frontend can poll live status. +""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime + +from structlog.contextvars import bound_contextvars + +from app.analysis.llm.factory import get_llm_provider +from app.core.config import get_settings +from app.core.logging import get_logger +from app.db.session import get_sessionmaker +from app.models.enums import MonitoringRunStatus, MonitoringRunTrigger, ReportType, SourceStatus +from app.repositories.company_repository import CompanyRepository +from app.repositories.monitoring_run_repository import MonitoringRunRepository +from app.repositories.report_repository import ReportRepository +from app.repositories.source_repository import SnapshotRepository, SourceRepository +from app.services import ( + alert_service, + change_detection_service, + collection_service, + report_service, + user_api_key_service, +) +from app.services.scheduling import validate_and_compute_next_run +from app.tasks.base import run_async_task +from app.tasks.celery_app import celery_app + +logger = get_logger(__name__) + + +@celery_app.task(bind=True, name="app.tasks.collection.run_monitoring", max_retries=2) +def run_monitoring(self, run_id: str) -> None: + run_async_task(_run_monitoring_async(run_id, self.request.id)) + + +async def _run_monitoring_async(run_id: str, task_id: str | None) -> None: + with bound_contextvars(run_id=run_id, task_id=task_id): + await _run_monitoring(run_id, task_id) + + +async def _run_monitoring(run_id: str, task_id: str | None) -> None: + settings = get_settings() + session_factory = get_sessionmaker() + + async with session_factory() as db: + run_repo = MonitoringRunRepository(db) + run = await run_repo.get(uuid.UUID(run_id)) + if run is None: + logger.warning("monitoring_run_not_found", run_id=run_id) + return + + company_repo = CompanyRepository(db) + company = await company_repo.get_by_id(run.company_id) + if company is None: + await run_repo.mark_finished( + run, status=MonitoringRunStatus.FAILED, error_summary="Company no longer exists" + ) + return + + await run_repo.set_worker_task_id(run, task_id) + await run_repo.mark_running(run) + logger.info( + "monitoring_run_started", + run_id=run_id, + company_id=str(company.id), + trigger=run.trigger_type.value, + ) + + settings = await user_api_key_service.get_effective_settings(db, company.user_id, settings) + llm = get_llm_provider(settings) + source_repo = SourceRepository(db) + snapshot_repo = SnapshotRepository(db) + sources = await source_repo.list_for_company(company.id) + + if not sources: + try: + await collection_service.discover_sources_for_company(db, company, settings) + except Exception as exc: # pragma: no cover - defensive, discovery is best-effort + logger.warning( + "discovery_failed_during_run", company_id=str(company.id), error=str(exc) + ) + sources = await source_repo.list_for_company(company.id) + + now = datetime.now(UTC) + config = company.monitor_configuration + if run.trigger_type == MonitoringRunTrigger.SCHEDULED: + # Only the sources actually due right now - a source with a + # slower override (e.g. patents checked monthly) sits out a run + # that only a faster sibling source (e.g. news checked daily) + # triggered. See SourceRepository.list_due_for_company. + company_next_run = config.next_run if config is not None else None + active_sources = await source_repo.list_due_for_company( + company.id, now, company_next_run + ) + else: + # Manual "Run now" is always a full on-demand check of every + # active source, regardless of any source's individual cadence. + active_sources = [s for s in sources if s.active] + + items_collected = 0 + successful = 0 + failed = 0 + changes_detected = 0 + errors: list[str] = [] + + for source in active_sources: + try: + result = await collection_service.collect_source( + db, settings, source, company, monitoring_run_id=run.id + ) + items_collected += len(result.documents) + + # Advance this source's own clock only if it has an override + # - a source with no override has no next_check of its own + # (it rides the company's next_run instead, advanced below). + # Guarded separately from the collection result above: a + # scheduling computation issue must never get reported as a + # collection failure for a source that actually succeeded. + if source.frequency_type is not None: + try: + tz_name = config.timezone if config is not None else "UTC" + source.next_check = validate_and_compute_next_run( + frequency_type=source.frequency_type, + interval_minutes=source.interval_minutes, + cron_expression=source.cron_expression, + tz_name=tz_name, + minimum_interval_minutes=settings.minimum_monitoring_interval_minutes, + from_time=now, + ) + except Exception as exc: # pragma: no cover - defensive + logger.warning( + "source_next_check_computation_failed", + source_id=str(source.id), + error=str(exc), + ) + + if result.status == SourceStatus.ACTIVE: + successful += 1 + else: + failed += 1 + if result.error: + errors.append(f"{source.name}: {result.error}") + + if result.documents: + current_snapshot = await snapshot_repo.latest_for_source(source.id) + if current_snapshot is not None: + change = await change_detection_service.detect_change_for_source( + db, source, company, current_snapshot, run.id + ) + if change is not None: + changes_detected += 1 + logger.info( + "change_detected", + source_id=str(source.id), + change_type=change.change_type.value, + severity=change.severity.value, + significance=change.significance_score, + confidence=change.confidence_score, + ) + try: + alert = await alert_service.create_alert_for_change( + db, settings, llm, change, company + ) + if alert is not None: + logger.info( + "alert_created", + alert_id=str(alert.id), + severity=alert.severity.value, + ) + except Exception as exc: # pragma: no cover - defensive, alerting failure shouldn't fail the run + logger.error( + "alert_creation_failed", + change_id=str(change.id), + error=str(exc), + ) + except ( + Exception + ) as exc: # pragma: no cover - defensive, one bad source shouldn't kill the run + failed += 1 + errors.append(f"{source.name}: {exc}") + logger.error( + "source_collection_unexpected_error", source_id=str(source.id), error=str(exc) + ) + + await run_repo.update_progress( + run, + sources_attempted=successful + failed, + sources_successful=successful, + sources_failed=failed, + items_collected=items_collected, + changes_detected=changes_detected, + ) + + if not active_sources or failed == 0: + status = MonitoringRunStatus.SUCCESSFUL + elif successful == 0: + status = MonitoringRunStatus.FAILED + else: + status = MonitoringRunStatus.PARTIAL + + # Generate a baseline report the first time this company has any + # evidence at all, and an update report whenever a run actually + # detected something new - never regenerate on a no-op run (keeps + # LLM usage proportional to real activity, not to schedule cadence). + report_repo = ReportRepository(db) + existing_report_count = await report_repo.count_for_company(company.id) + report_type = None + if existing_report_count == 0 and items_collected > 0: + report_type = ReportType.BASELINE + elif changes_detected > 0: + report_type = ReportType.UPDATE + + if report_type is not None: + try: + await report_service.generate_and_persist_report( + db, settings, llm, company, report_type=report_type, monitoring_run_id=run.id + ) + logger.info( + "report_generated", company_id=str(company.id), report_type=report_type.value + ) + except ( + Exception + ) as exc: # pragma: no cover - defensive, report failure shouldn't fail the run + logger.error("report_generation_failed", company_id=str(company.id), error=str(exc)) + + # last_run reflects any check; next_run (the schedule cadence) only + # advances for runs the schedule itself fired - a manual "run now" + # must not disrupt the next scheduled run (see spec + ARCHITECTURE.md). + # Applied unconditionally when config exists - it also carries each + # source's own next_check advance from the collection loop above. + if config is not None: + config.last_run = datetime.now(UTC) + if run.trigger_type == MonitoringRunTrigger.SCHEDULED: + config.next_run = validate_and_compute_next_run( + frequency_type=config.frequency_type, + interval_minutes=config.interval_minutes, + cron_expression=config.cron_expression, + tz_name=config.timezone, + minimum_interval_minutes=settings.minimum_monitoring_interval_minutes, + ) + + # Marked finished last, and deliberately in the same commit as the + # report/config work above: the frontend polls run.status to decide + # when to stop showing "Running..." - flipping it to a terminal + # status any earlier would let that indicator (and the Latest + # report/Sources/Monitoring history/Snapshots/Overview tabs it + # gates) go stale while report generation and schedule bookkeeping + # are still in flight. + await run_repo.mark_finished( + run, status=status, error_summary="; ".join(errors[:5]) or None + ) + + logger.info( + "monitoring_run_finished", + run_id=run_id, + status=status.value, + sources_successful=successful, + sources_failed=failed, + items_collected=items_collected, + ) diff --git a/apps/api/app/tasks/enrichment.py b/apps/api/app/tasks/enrichment.py new file mode 100644 index 0000000..3ab6e94 --- /dev/null +++ b/apps/api/app/tasks/enrichment.py @@ -0,0 +1,55 @@ +"""Celery task that runs company enrichment exactly once, right after a +company is created (see company_service.create_company, which only ever +enqueues this when NINJAPEAR_API_KEY is configured). Generous time limits +since some NinjaPear endpoints are documented as taking up to 5 minutes, +and a single company can trigger the company-level calls plus several +capped per-leadership-member lookups in series. +""" + +from __future__ import annotations + +import uuid + +from structlog.contextvars import bound_contextvars + +from app.core.config import get_settings +from app.core.logging import get_logger +from app.db.session import get_sessionmaker +from app.enrichment.factory import get_enrichment_provider +from app.repositories.company_repository import CompanyRepository +from app.services import user_api_key_service +from app.services.enrichment_service import enrich_company as enrich_company_service +from app.tasks.base import run_async_task +from app.tasks.celery_app import celery_app + +logger = get_logger(__name__) + + +@celery_app.task( + bind=True, + name="app.tasks.enrichment.enrich_company", + max_retries=1, + soft_time_limit=1500, + time_limit=1600, +) +def enrich_company(self, company_id: str) -> None: + with bound_contextvars(task_id=self.request.id, company_id=company_id): + run_async_task(_enrich_company_async(company_id)) + + +async def _enrich_company_async(company_id: str) -> None: + settings = get_settings() + session_factory = get_sessionmaker() + + async with session_factory() as db: + company = await CompanyRepository(db).get_by_id(uuid.UUID(company_id)) + if company is None: + logger.warning("enrichment_company_not_found", company_id=company_id) + return + + settings = await user_api_key_service.get_effective_settings(db, company.user_id, settings) + provider = get_enrichment_provider(settings) + logger.info( + "company_enrichment_started", company_id=company_id, provider=provider.provider_name + ) + await enrich_company_service(db, settings, provider, company) diff --git a/apps/api/app/tasks/maintenance.py b/apps/api/app/tasks/maintenance.py new file mode 100644 index 0000000..690a608 --- /dev/null +++ b/apps/api/app/tasks/maintenance.py @@ -0,0 +1,43 @@ +"""Celery Beat-triggered housekeeping. Currently just the data-retention +purge (DATA_RETENTION_DAYS) - only SourceDocument rows are ever deleted +here; see SourceDocumentRepository.delete_older_than for why that's the +only safe target in the current FK graph. +""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +from structlog.contextvars import bound_contextvars + +from app.core.config import get_settings +from app.core.logging import get_logger +from app.db.session import get_sessionmaker +from app.repositories.source_repository import SourceDocumentRepository +from app.tasks.base import run_async_task +from app.tasks.celery_app import celery_app + +logger = get_logger(__name__) + + +@celery_app.task(bind=True, name="app.tasks.maintenance.purge_expired_data") +def purge_expired_data(self) -> None: + with bound_contextvars(task_id=self.request.id): + run_async_task(_purge_expired_data_async()) + + +async def _purge_expired_data_async() -> None: + settings = get_settings() + cutoff = datetime.now(UTC) - timedelta(days=settings.data_retention_days) + + session_factory = get_sessionmaker() + async with session_factory() as db: + deleted = await SourceDocumentRepository(db).delete_older_than(cutoff) + await db.commit() + + logger.info( + "data_retention_purge_completed", + deleted_source_documents=deleted, + retention_days=settings.data_retention_days, + cutoff=cutoff.isoformat(), + ) diff --git a/apps/api/app/tasks/scheduler.py b/apps/api/app/tasks/scheduler.py new file mode 100644 index 0000000..76a4dc9 --- /dev/null +++ b/apps/api/app/tasks/scheduler.py @@ -0,0 +1,73 @@ +"""Celery Beat-triggered task. Runs every minute (see celery_app.py's +beat_schedule) and dynamically discovers which companies are due for a +check by querying MonitorConfiguration/Source directly - adding a new +company never requires touching this task or any static schedule config. + +A company is due either because its own default MonitorConfiguration.next_run +has arrived, or because at least one of its sources has its own faster +frequency override that's independently due (see +SourceRepository.list_due_for_company) - e.g. a "check news daily" source +inside a company whose overall default cadence is weekly. Either way, +run_monitoring only actually collects the sources that are themselves due +for a SCHEDULED trigger - see tasks/collection.py. +""" + +from __future__ import annotations + +from datetime import UTC, datetime + +from structlog.contextvars import bound_contextvars + +from app.core.logging import get_logger +from app.db.session import get_sessionmaker +from app.models.enums import MonitoringRunTrigger +from app.repositories.company_repository import MonitorConfigurationRepository +from app.repositories.monitoring_run_repository import MonitoringRunRepository +from app.repositories.source_repository import SourceRepository +from app.tasks.base import run_async_task +from app.tasks.celery_app import celery_app + +logger = get_logger(__name__) + + +@celery_app.task(bind=True, name="app.tasks.scheduler.sync_schedules") +def sync_schedules(self) -> None: + with bound_contextvars(task_id=self.request.id): + run_async_task(_sync_schedules_async()) + + +async def _sync_schedules_async() -> None: + from app.tasks.collection import ( + run_monitoring, + ) # local import: avoids a circular import at module load + + session_factory = get_sessionmaker() + async with session_factory() as db: + monitor_repo = MonitorConfigurationRepository(db) + source_repo = SourceRepository(db) + run_repo = MonitoringRunRepository(db) + + now = datetime.now(UTC) + enabled_configs = await monitor_repo.list_enabled() + enqueued = 0 + + for config in enabled_configs: + is_due = await source_repo.company_has_due_work(config.company_id, now, config.next_run) + if not is_due: + continue + + # Idempotency: don't double-enqueue a company that's still + # mid-run from a previous tick or a manual "run now". + active_run = await run_repo.get_active_for_company(config.company_id) + if active_run is not None: + continue + + run = await run_repo.create( + company_id=config.company_id, trigger_type=MonitoringRunTrigger.SCHEDULED + ) + await db.commit() + run_monitoring.delay(str(run.id)) + enqueued += 1 + + if enqueued: + logger.info("schedule_sync_enqueued_runs", count=enqueued) diff --git a/apps/api/celerybeat-schedule b/apps/api/celerybeat-schedule new file mode 100644 index 0000000000000000000000000000000000000000..b9e9814be42d3500a4f195478e221e0a48034abf GIT binary patch literal 4096 zcmeHHO-~a+7@pECZGo*={6bL@1gYB2L8FPr#2EBo8i}zO6Ax^*+p)W$Oq<=Qv6v=3 zi6nDlxcDDTj6Xonc<~2-Cf6g<&;>xV zb0ciGvi-s~2LOgRY{o^5w?1iF!Gp^GdO(6v<*(`zj46MmSAv@I=c5w5RDLe5z`og; z3~1_@eJc~O$MWDm_ew}Ue^UMAIpu?xjIa29kMvi4UkSlhzE~yB`p%o&;pfp@_%eVn zaX~`Hg)FS`Gd7#Q+v6+CpT-U2G3rzK7P5GV_Y>2vPgRL!TwfROl()_D@b=ZzF)|0h zd9{y__$2Qeo5;C?yoOe1P~6A$oH90yhPLDjF^yb;m*GVQ$sCU=xdt?Re`^iuZ@hlq^swLzx+H0W_a%(V;Ov0$1Q(tcJ zTh-ViOW7ehVpb}UnBI&B{e>$cLM~jG None: + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +async def run_migrations_online() -> None: + connectable = async_engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + + await connectable.dispose() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + asyncio.run(run_migrations_online()) diff --git a/apps/api/migrations/script.py.mako b/apps/api/migrations/script.py.mako new file mode 100644 index 0000000..ed57b86 --- /dev/null +++ b/apps/api/migrations/script.py.mako @@ -0,0 +1,27 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +revision: str = ${repr(up_revision)} +down_revision: str | None = ${repr(down_revision)} +branch_labels: Sequence[str] | str | None = ${repr(branch_labels)} +depends_on: Sequence[str] | str | None = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/apps/api/migrations/versions/.gitkeep b/apps/api/migrations/versions/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/migrations/versions/05a3ccfe49b9_create_users_and_refresh_tokens.py b/apps/api/migrations/versions/05a3ccfe49b9_create_users_and_refresh_tokens.py new file mode 100644 index 0000000..07bbedc --- /dev/null +++ b/apps/api/migrations/versions/05a3ccfe49b9_create_users_and_refresh_tokens.py @@ -0,0 +1,64 @@ +"""create users and refresh tokens + +Revision ID: 05a3ccfe49b9 +Revises: +Create Date: 2026-07-31 18:14:41.012285 + +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "05a3ccfe49b9" +down_revision: str | None = None +branch_labels: Sequence[str] | str | None = None +depends_on: Sequence[str] | str | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "users", + sa.Column("email", sa.String(length=320), nullable=False), + sa.Column("password_hash", sa.String(length=255), nullable=True), + sa.Column("display_name", sa.String(length=120), nullable=False), + sa.Column("timezone", sa.String(length=64), nullable=False), + sa.Column("is_active", sa.Boolean(), nullable=False), + sa.Column("is_admin", sa.Boolean(), nullable=False), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("id"), + ) + op.create_index(op.f("ix_users_email"), "users", ["email"], unique=True) + op.create_table( + "refresh_tokens", + sa.Column("user_id", sa.Uuid(), nullable=False), + sa.Column("token_hash", sa.String(length=64), nullable=False), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("id"), + ) + op.create_index( + op.f("ix_refresh_tokens_token_hash"), "refresh_tokens", ["token_hash"], unique=True + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f("ix_refresh_tokens_token_hash"), table_name="refresh_tokens") + op.drop_table("refresh_tokens") + op.drop_index(op.f("ix_users_email"), table_name="users") + op.drop_table("users") + # ### end Alembic commands ### diff --git a/apps/api/migrations/versions/06e7f03cea36_add_per_source_scheduling_override_.py b/apps/api/migrations/versions/06e7f03cea36_add_per_source_scheduling_override_.py new file mode 100644 index 0000000..751a16c --- /dev/null +++ b/apps/api/migrations/versions/06e7f03cea36_add_per_source_scheduling_override_.py @@ -0,0 +1,57 @@ +"""add per-source scheduling override columns + +Revision ID: 06e7f03cea36 +Revises: 60a25ddfc6a3 +Create Date: 2026-08-03 17:22:37.278775 + +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "06e7f03cea36" +down_revision: str | None = "60a25ddfc6a3" +branch_labels: Sequence[str] | str | None = None +depends_on: Sequence[str] | str | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "sources", + sa.Column( + "frequency_type", + sa.Enum( + "HOURLY", + "EVERY_6_HOURS", + "EVERY_12_HOURS", + "DAILY", + "EVERY_2_DAYS", + "WEEKLY", + "EVERY_2_WEEKS", + "MONTHLY", + "CUSTOM", + name="monitoringfrequency", + native_enum=False, + length=20, + ), + nullable=True, + ), + ) + op.add_column("sources", sa.Column("interval_minutes", sa.Integer(), nullable=True)) + op.add_column("sources", sa.Column("cron_expression", sa.String(length=120), nullable=True)) + op.add_column("sources", sa.Column("next_check", sa.DateTime(timezone=True), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("sources", "next_check") + op.drop_column("sources", "cron_expression") + op.drop_column("sources", "interval_minutes") + op.drop_column("sources", "frequency_type") + # ### end Alembic commands ### diff --git a/apps/api/migrations/versions/484419ccd357_create_sources_source_documents_and_.py b/apps/api/migrations/versions/484419ccd357_create_sources_source_documents_and_.py new file mode 100644 index 0000000..b715dd0 --- /dev/null +++ b/apps/api/migrations/versions/484419ccd357_create_sources_source_documents_and_.py @@ -0,0 +1,155 @@ +"""create sources source_documents and snapshots + +Revision ID: 484419ccd357 +Revises: 8880af3b8aa1 +Create Date: 2026-07-31 18:55:01.926914 + +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "484419ccd357" +down_revision: str | None = "8880af3b8aa1" +branch_labels: Sequence[str] | str | None = None +depends_on: Sequence[str] | str | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "sources", + sa.Column("company_id", sa.Uuid(), nullable=False), + sa.Column( + "source_type", + sa.Enum( + "WEBSITE", + "RSS", + "CUSTOM_URL", + "SEC_EDGAR", + "GITHUB", + "JOB_POSTING", + "PATENT", + "REVIEW", + name="sourcetype", + native_enum=False, + length=20, + ), + nullable=False, + ), + sa.Column("name", sa.String(length=200), nullable=False), + sa.Column("base_url", sa.String(length=500), nullable=True), + sa.Column("active", sa.Boolean(), nullable=False), + sa.Column( + "status", + sa.Enum( + "ACTIVE", + "DISABLED", + "RATE_LIMITED", + "AUTH_REQUIRED", + "BLOCKED_BY_POLICY", + "FAILED", + name="sourcestatus", + native_enum=False, + length=20, + ), + nullable=False, + ), + sa.Column("trust_score", sa.Float(), nullable=False), + sa.Column("last_checked", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_successful_check", sa.DateTime(timezone=True), nullable=True), + sa.Column("failure_count", sa.Integer(), nullable=False), + sa.Column("configuration_metadata", sa.JSON(), nullable=False), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["company_id"], ["companies.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("id"), + ) + op.create_index(op.f("ix_sources_company_id"), "sources", ["company_id"], unique=False) + op.create_table( + "snapshots", + sa.Column("company_id", sa.Uuid(), nullable=False), + sa.Column("source_id", sa.Uuid(), nullable=False), + sa.Column("snapshot_type", sa.String(length=50), nullable=False), + sa.Column("hash", sa.String(length=64), nullable=False), + sa.Column("structured_summary", sa.JSON(), nullable=False), + sa.Column("text_summary", sa.Text(), nullable=True), + sa.Column("monitoring_run_id", sa.Uuid(), nullable=True), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["company_id"], ["companies.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["source_id"], ["sources.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("id"), + ) + op.create_index(op.f("ix_snapshots_company_id"), "snapshots", ["company_id"], unique=False) + op.create_index(op.f("ix_snapshots_hash"), "snapshots", ["hash"], unique=False) + op.create_index( + op.f("ix_snapshots_monitoring_run_id"), "snapshots", ["monitoring_run_id"], unique=False + ) + op.create_index(op.f("ix_snapshots_source_id"), "snapshots", ["source_id"], unique=False) + op.create_table( + "source_documents", + sa.Column("source_id", sa.Uuid(), nullable=False), + sa.Column("company_id", sa.Uuid(), nullable=False), + sa.Column("url", sa.String(length=1000), nullable=False), + sa.Column("canonical_url", sa.String(length=1000), nullable=False), + sa.Column("title", sa.String(length=500), nullable=True), + sa.Column("author", sa.String(length=200), nullable=True), + sa.Column("publication_date", sa.DateTime(timezone=True), nullable=True), + sa.Column("retrieved_date", sa.DateTime(timezone=True), nullable=False), + sa.Column("content_text", sa.Text(), nullable=False), + sa.Column("content_hash", sa.String(length=64), nullable=False), + sa.Column("metadata_json", sa.JSON(), nullable=False), + sa.Column("language", sa.String(length=16), nullable=True), + sa.Column("http_status", sa.Integer(), nullable=True), + sa.Column("extraction_method", sa.String(length=50), nullable=False), + sa.Column("trust_score", sa.Float(), nullable=False), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["company_id"], ["companies.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["source_id"], ["sources.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("id"), + ) + op.create_index( + op.f("ix_source_documents_canonical_url"), + "source_documents", + ["canonical_url"], + unique=False, + ) + op.create_index( + op.f("ix_source_documents_company_id"), "source_documents", ["company_id"], unique=False + ) + op.create_index( + op.f("ix_source_documents_content_hash"), "source_documents", ["content_hash"], unique=False + ) + op.create_index( + op.f("ix_source_documents_source_id"), "source_documents", ["source_id"], unique=False + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f("ix_source_documents_source_id"), table_name="source_documents") + op.drop_index(op.f("ix_source_documents_content_hash"), table_name="source_documents") + op.drop_index(op.f("ix_source_documents_company_id"), table_name="source_documents") + op.drop_index(op.f("ix_source_documents_canonical_url"), table_name="source_documents") + op.drop_table("source_documents") + op.drop_index(op.f("ix_snapshots_source_id"), table_name="snapshots") + op.drop_index(op.f("ix_snapshots_monitoring_run_id"), table_name="snapshots") + op.drop_index(op.f("ix_snapshots_hash"), table_name="snapshots") + op.drop_index(op.f("ix_snapshots_company_id"), table_name="snapshots") + op.drop_table("snapshots") + op.drop_index(op.f("ix_sources_company_id"), table_name="sources") + op.drop_table("sources") + # ### end Alembic commands ### diff --git a/apps/api/migrations/versions/60a25ddfc6a3_add_notification_destination_companies_.py b/apps/api/migrations/versions/60a25ddfc6a3_add_notification_destination_companies_.py new file mode 100644 index 0000000..b914306 --- /dev/null +++ b/apps/api/migrations/versions/60a25ddfc6a3_add_notification_destination_companies_.py @@ -0,0 +1,120 @@ +"""add notification_destination_companies join table + +Notification destinations were previously user-global with no concept of +"which company is this for" - every destination got every one of the user's +companies' alerts. This migration adds the join table and then, in the same +step, (1) backfills links so every existing destination is linked to every +company the same user currently has (this is a no-op behavior change - it's +exactly what already happened implicitly before this table existed), and (2) +deduplicates destinations that share the same (user, type, value) - the +wizard previously created a brand new row per company even when the same +email/phone was reused, which is the duplicate-rows bug this whole feature +was built to fix. The earliest-created row per duplicate group is kept; +later duplicates are deleted (cascading to their NotificationDelivery +history, which is acceptable one-time cleanup - see KNOWN_LIMITATIONS.md). + +Revision ID: 60a25ddfc6a3 +Revises: f01919a99ee9 +Create Date: 2026-08-02 11:47:04.760190 + +""" + +from __future__ import annotations + +from collections.abc import Sequence +from datetime import UTC, datetime + +import sqlalchemy as sa +from alembic import op + +revision: str = "60a25ddfc6a3" +down_revision: str | None = "f01919a99ee9" +branch_labels: Sequence[str] | str | None = None +depends_on: Sequence[str] | str | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "notification_destination_companies", + sa.Column("destination_id", sa.Uuid(), nullable=False), + sa.Column("company_id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["company_id"], ["companies.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["destination_id"], ["notification_destinations.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("destination_id", "company_id"), + ) + # ### end Alembic commands ### + + _backfill_links_and_dedupe_destinations() + + +def _backfill_links_and_dedupe_destinations() -> None: + bind = op.get_bind() + + companies_t = sa.table( + "companies", + sa.column("id", sa.Uuid()), + sa.column("user_id", sa.Uuid()), + ) + destinations_t = sa.table( + "notification_destinations", + sa.column("id", sa.Uuid()), + sa.column("user_id", sa.Uuid()), + sa.column("type", sa.String()), + sa.column("destination_value", sa.String()), + sa.column("created_at", sa.DateTime(timezone=True)), + ) + links_t = sa.table( + "notification_destination_companies", + sa.column("destination_id", sa.Uuid()), + sa.column("company_id", sa.Uuid()), + sa.column("created_at", sa.DateTime(timezone=True)), + sa.column("updated_at", sa.DateTime(timezone=True)), + ) + + all_destinations = bind.execute( + sa.select( + destinations_t.c.id, + destinations_t.c.user_id, + destinations_t.c.type, + destinations_t.c.destination_value, + ).order_by(destinations_t.c.user_id, destinations_t.c.created_at) + ).fetchall() + + companies_by_user: dict = {} + for company_id, user_id in bind.execute( + sa.select(companies_t.c.id, companies_t.c.user_id) + ).fetchall(): + companies_by_user.setdefault(user_id, []).append(company_id) + + now = datetime.now(UTC) + link_rows = [ + {"destination_id": dest_id, "company_id": company_id, "created_at": now, "updated_at": now} + for dest_id, user_id, _type, _value in all_destinations + for company_id in companies_by_user.get(user_id, []) + ] + if link_rows: + bind.execute(sa.insert(links_t), link_rows) + + seen: dict[tuple, object] = {} + duplicate_ids: list = [] + for dest_id, user_id, dtype, value in all_destinations: + normalized_value = value.strip().lower() if dtype == "email" else value.strip() + key = (user_id, dtype, normalized_value) + if key in seen: + duplicate_ids.append(dest_id) + else: + seen[key] = dest_id + + if duplicate_ids: + bind.execute(sa.delete(destinations_t).where(destinations_t.c.id.in_(duplicate_ids))) + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("notification_destination_companies") + # ### end Alembic commands ### diff --git a/apps/api/migrations/versions/6d296b533f7c_add_user_api_keys_table.py b/apps/api/migrations/versions/6d296b533f7c_add_user_api_keys_table.py new file mode 100644 index 0000000..c06cf0a --- /dev/null +++ b/apps/api/migrations/versions/6d296b533f7c_add_user_api_keys_table.py @@ -0,0 +1,47 @@ +"""add user_api_keys table + +Revision ID: 6d296b533f7c +Revises: 7e0adee63b3b +Create Date: 2026-08-05 10:40:15.989218 + +""" +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa + + +revision: str = '6d296b533f7c' +down_revision: str | None = '7e0adee63b3b' +branch_labels: Sequence[str] | str | None = None +depends_on: Sequence[str] | str | None = None + + +def upgrade() -> None: + op.create_table( + 'user_api_keys', + sa.Column('user_id', sa.Uuid(), nullable=False), + sa.Column( + 'provider', + sa.Enum( + 'ANTHROPIC', 'BRAVE_SEARCH', 'NINJAPEAR', 'USPTO', + name='apikeyprovider', native_enum=False, length=16, + ), + nullable=False, + ), + sa.Column('encrypted_key', sa.Text(), nullable=False), + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('user_id', 'provider', name='uq_user_api_keys_user_provider'), + ) + op.create_index(op.f('ix_user_api_keys_user_id'), 'user_api_keys', ['user_id'], unique=False) + + +def downgrade() -> None: + op.drop_index(op.f('ix_user_api_keys_user_id'), table_name='user_api_keys') + op.drop_table('user_api_keys') diff --git a/apps/api/migrations/versions/79e3aa041131_create_company_enrichments_table.py b/apps/api/migrations/versions/79e3aa041131_create_company_enrichments_table.py new file mode 100644 index 0000000..70e6279 --- /dev/null +++ b/apps/api/migrations/versions/79e3aa041131_create_company_enrichments_table.py @@ -0,0 +1,64 @@ +"""create company enrichments table + +Revision ID: 79e3aa041131 +Revises: 06e7f03cea36 +Create Date: 2026-08-03 18:43:27.311271 + +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "79e3aa041131" +down_revision: str | None = "06e7f03cea36" +branch_labels: Sequence[str] | str | None = None +depends_on: Sequence[str] | str | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "company_enrichments", + sa.Column("company_id", sa.Uuid(), nullable=False), + sa.Column( + "status", + sa.Enum( + "PENDING", + "PARTIAL", + "COMPLETE", + "FAILED", + name="enrichmentstatus", + native_enum=False, + length=20, + ), + nullable=False, + ), + sa.Column("data", sa.JSON(), nullable=False), + sa.Column("errors", sa.JSON(), nullable=False), + sa.Column("credits_spent", sa.Integer(), nullable=True), + sa.Column("fetched_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["company_id"], ["companies.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("id"), + ) + op.create_index( + op.f("ix_company_enrichments_company_id"), + "company_enrichments", + ["company_id"], + unique=True, + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f("ix_company_enrichments_company_id"), table_name="company_enrichments") + op.drop_table("company_enrichments") + # ### end Alembic commands ### diff --git a/apps/api/migrations/versions/7e0adee63b3b_add_password_history_entries_table.py b/apps/api/migrations/versions/7e0adee63b3b_add_password_history_entries_table.py new file mode 100644 index 0000000..ec78743 --- /dev/null +++ b/apps/api/migrations/versions/7e0adee63b3b_add_password_history_entries_table.py @@ -0,0 +1,45 @@ +"""add password_history_entries table + +Revision ID: 7e0adee63b3b +Revises: ef56f181dbb9 +Create Date: 2026-08-05 07:24:33.677085 + +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa + +revision: str = "7e0adee63b3b" +down_revision: str | None = "ef56f181dbb9" +branch_labels: Sequence[str] | str | None = None +depends_on: Sequence[str] | str | None = None + + +def upgrade() -> None: + op.create_table( + "password_history_entries", + sa.Column("user_id", sa.Uuid(), nullable=False), + sa.Column("password_hash", sa.String(length=255), nullable=False), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + op.f("ix_password_history_entries_user_id"), + "password_history_entries", + ["user_id"], + unique=False, + ) + + +def downgrade() -> None: + op.drop_index( + op.f("ix_password_history_entries_user_id"), table_name="password_history_entries" + ) + op.drop_table("password_history_entries") diff --git a/apps/api/migrations/versions/8880af3b8aa1_create_companies_monitoring_and_.py b/apps/api/migrations/versions/8880af3b8aa1_create_companies_monitoring_and_.py new file mode 100644 index 0000000..f87c721 --- /dev/null +++ b/apps/api/migrations/versions/8880af3b8aa1_create_companies_monitoring_and_.py @@ -0,0 +1,186 @@ +"""create companies monitoring and notification tables + +Revision ID: 8880af3b8aa1 +Revises: 05a3ccfe49b9 +Create Date: 2026-07-31 18:29:28.820437 + +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "8880af3b8aa1" +down_revision: str | None = "05a3ccfe49b9" +branch_labels: Sequence[str] | str | None = None +depends_on: Sequence[str] | str | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "companies", + sa.Column("user_id", sa.Uuid(), nullable=False), + sa.Column("name", sa.String(length=200), nullable=False), + sa.Column("slug", sa.String(length=220), nullable=False), + sa.Column("official_website", sa.String(length=500), nullable=True), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("monitoring_focus", sa.Text(), nullable=True), + sa.Column("industry", sa.String(length=120), nullable=True), + sa.Column("country", sa.String(length=120), nullable=True), + sa.Column("region", sa.String(length=120), nullable=True), + sa.Column( + "status", + sa.Enum("ACTIVE", "PAUSED", name="companystatus", native_enum=False, length=20), + nullable=False, + ), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("id"), + ) + op.create_index(op.f("ix_companies_slug"), "companies", ["slug"], unique=False) + op.create_index(op.f("ix_companies_user_id"), "companies", ["user_id"], unique=False) + op.create_table( + "notification_destinations", + sa.Column("user_id", sa.Uuid(), nullable=False), + sa.Column( + "type", + sa.Enum( + "EMAIL", "SMS", "CONSOLE", name="notificationtype", native_enum=False, length=20 + ), + nullable=False, + ), + sa.Column("destination_value", sa.String(length=320), nullable=False), + sa.Column("verified", sa.Boolean(), nullable=False), + sa.Column("enabled", sa.Boolean(), nullable=False), + sa.Column( + "minimum_severity", + sa.Enum( + "CRITICAL", + "HIGH", + "MEDIUM", + "LOW", + name="severitylevel", + native_enum=False, + length=20, + ), + nullable=False, + ), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("id"), + ) + op.create_index( + op.f("ix_notification_destinations_user_id"), + "notification_destinations", + ["user_id"], + unique=False, + ) + op.create_table( + "company_aliases", + sa.Column("company_id", sa.Uuid(), nullable=False), + sa.Column("alias", sa.String(length=200), nullable=False), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["company_id"], ["companies.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("id"), + ) + op.create_index( + op.f("ix_company_aliases_company_id"), "company_aliases", ["company_id"], unique=False + ) + op.create_table( + "competitors", + sa.Column("company_id", sa.Uuid(), nullable=False), + sa.Column("name", sa.String(length=200), nullable=False), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["company_id"], ["companies.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("id"), + ) + op.create_index(op.f("ix_competitors_company_id"), "competitors", ["company_id"], unique=False) + op.create_table( + "monitor_configurations", + sa.Column("company_id", sa.Uuid(), nullable=False), + sa.Column( + "frequency_type", + sa.Enum( + "HOURLY", + "EVERY_6_HOURS", + "EVERY_12_HOURS", + "DAILY", + "EVERY_2_DAYS", + "WEEKLY", + "EVERY_2_WEEKS", + "MONTHLY", + "CUSTOM", + name="monitoringfrequency", + native_enum=False, + length=30, + ), + nullable=False, + ), + sa.Column("interval_minutes", sa.Integer(), nullable=True), + sa.Column("cron_expression", sa.String(length=120), nullable=True), + sa.Column("timezone", sa.String(length=64), nullable=False), + sa.Column("enabled", sa.Boolean(), nullable=False), + sa.Column("next_run", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_run", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "severity_threshold", + sa.Enum( + "CRITICAL", + "HIGH", + "MEDIUM", + "LOW", + name="severitylevel", + native_enum=False, + length=20, + ), + nullable=False, + ), + sa.Column("source_configuration", sa.JSON(), nullable=False), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["company_id"], ["companies.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("id"), + ) + op.create_index( + op.f("ix_monitor_configurations_company_id"), + "monitor_configurations", + ["company_id"], + unique=True, + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f("ix_monitor_configurations_company_id"), table_name="monitor_configurations") + op.drop_table("monitor_configurations") + op.drop_index(op.f("ix_competitors_company_id"), table_name="competitors") + op.drop_table("competitors") + op.drop_index(op.f("ix_company_aliases_company_id"), table_name="company_aliases") + op.drop_table("company_aliases") + op.drop_index( + op.f("ix_notification_destinations_user_id"), table_name="notification_destinations" + ) + op.drop_table("notification_destinations") + op.drop_index(op.f("ix_companies_user_id"), table_name="companies") + op.drop_index(op.f("ix_companies_slug"), table_name="companies") + op.drop_table("companies") + # ### end Alembic commands ### diff --git a/apps/api/migrations/versions/8ceefbd0a6a5_create_monitoring_runs.py b/apps/api/migrations/versions/8ceefbd0a6a5_create_monitoring_runs.py new file mode 100644 index 0000000..b16cb54 --- /dev/null +++ b/apps/api/migrations/versions/8ceefbd0a6a5_create_monitoring_runs.py @@ -0,0 +1,96 @@ +"""create monitoring runs + +Revision ID: 8ceefbd0a6a5 +Revises: 484419ccd357 +Create Date: 2026-07-31 19:10:32.580895 + +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "8ceefbd0a6a5" +down_revision: str | None = "484419ccd357" +branch_labels: Sequence[str] | str | None = None +depends_on: Sequence[str] | str | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "monitoring_runs", + sa.Column("company_id", sa.Uuid(), nullable=False), + sa.Column( + "trigger_type", + sa.Enum( + "SCHEDULED", + "MANUAL", + "INITIAL", + "RETRY", + name="monitoringruntrigger", + native_enum=False, + length=20, + ), + nullable=False, + ), + sa.Column( + "status", + sa.Enum( + "QUEUED", + "RUNNING", + "SUCCESSFUL", + "PARTIAL", + "FAILED", + name="monitoringrunstatus", + native_enum=False, + length=20, + ), + nullable=False, + ), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("sources_attempted", sa.Integer(), nullable=False), + sa.Column("sources_successful", sa.Integer(), nullable=False), + sa.Column("sources_failed", sa.Integer(), nullable=False), + sa.Column("items_collected", sa.Integer(), nullable=False), + sa.Column("changes_detected", sa.Integer(), nullable=False), + sa.Column("error_summary", sa.Text(), nullable=True), + sa.Column("worker_task_id", sa.String(length=255), nullable=True), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["company_id"], ["companies.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("id"), + ) + op.create_index( + op.f("ix_monitoring_runs_company_id"), "monitoring_runs", ["company_id"], unique=False + ) + # ### end Alembic commands ### + + # snapshots.monitoring_run_id was added in the previous migration without + # a FK constraint, since monitoring_runs didn't exist yet - add it now + # that the table this phase creates exists. batch_alter_table so this + # also works on SQLite (which can't ALTER TABLE ADD CONSTRAINT directly). + with op.batch_alter_table("snapshots") as batch_op: + batch_op.create_foreign_key( + "fk_snapshots_monitoring_run_id", + "monitoring_runs", + ["monitoring_run_id"], + ["id"], + ondelete="SET NULL", + ) + + +def downgrade() -> None: + with op.batch_alter_table("snapshots") as batch_op: + batch_op.drop_constraint("fk_snapshots_monitoring_run_id", type_="foreignkey") + + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f("ix_monitoring_runs_company_id"), table_name="monitoring_runs") + op.drop_table("monitoring_runs") + # ### end Alembic commands ### diff --git a/apps/api/migrations/versions/8db5bd1c31f8_add_system_secrets_table.py b/apps/api/migrations/versions/8db5bd1c31f8_add_system_secrets_table.py new file mode 100644 index 0000000..6f15e13 --- /dev/null +++ b/apps/api/migrations/versions/8db5bd1c31f8_add_system_secrets_table.py @@ -0,0 +1,46 @@ +"""add system secrets table + +Revision ID: 8db5bd1c31f8 +Revises: 6d296b533f7c +Create Date: 2026-08-05 07:48:50.750288 + +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "8db5bd1c31f8" +down_revision: str | None = "6d296b533f7c" +branch_labels: Sequence[str] | str | None = None +depends_on: Sequence[str] | str | None = None + + +def upgrade() -> None: + op.create_table( + "system_secrets", + sa.Column( + "key", + sa.Enum( + "TURNSTILE_SITE_KEY", + "TURNSTILE_SECRET", + name="systemsecretkey", + native_enum=False, + length=32, + ), + nullable=False, + ), + sa.Column("encrypted_value", sa.Text(), nullable=False), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("key"), + ) + + +def downgrade() -> None: + op.drop_table("system_secrets") diff --git a/apps/api/migrations/versions/b5e0d93c89ac_create_alerts_and_notification_.py b/apps/api/migrations/versions/b5e0d93c89ac_create_alerts_and_notification_.py new file mode 100644 index 0000000..95ec98e --- /dev/null +++ b/apps/api/migrations/versions/b5e0d93c89ac_create_alerts_and_notification_.py @@ -0,0 +1,121 @@ +"""create alerts and notification deliveries + +Revision ID: b5e0d93c89ac +Revises: c06c845cdd10 +Create Date: 2026-07-31 22:15:07.310833 + +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "b5e0d93c89ac" +down_revision: str | None = "c06c845cdd10" +branch_labels: Sequence[str] | str | None = None +depends_on: Sequence[str] | str | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "alerts", + sa.Column("company_id", sa.Uuid(), nullable=False), + sa.Column("detected_change_id", sa.Uuid(), nullable=False), + sa.Column("user_id", sa.Uuid(), nullable=False), + sa.Column("title", sa.String(length=200), nullable=False), + sa.Column("summary", sa.Text(), nullable=False), + sa.Column("why_it_matters", sa.Text(), nullable=False), + sa.Column( + "severity", + sa.Enum( + "CRITICAL", + "HIGH", + "MEDIUM", + "LOW", + name="severitylevel", + native_enum=False, + length=20, + ), + nullable=False, + ), + sa.Column("confidence", sa.Float(), nullable=False), + sa.Column("read", sa.Boolean(), nullable=False), + sa.Column("resolved", sa.Boolean(), nullable=False), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["company_id"], ["companies.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["detected_change_id"], ["detected_changes.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("id"), + ) + op.create_index(op.f("ix_alerts_company_id"), "alerts", ["company_id"], unique=False) + op.create_index( + op.f("ix_alerts_detected_change_id"), "alerts", ["detected_change_id"], unique=False + ) + op.create_index(op.f("ix_alerts_user_id"), "alerts", ["user_id"], unique=False) + op.create_table( + "notification_deliveries", + sa.Column("alert_id", sa.Uuid(), nullable=False), + sa.Column("destination_id", sa.Uuid(), nullable=False), + sa.Column("provider", sa.String(length=50), nullable=False), + sa.Column( + "status", + sa.Enum( + "PENDING", + "SENT", + "FAILED", + name="notificationdeliverystatus", + native_enum=False, + length=20, + ), + nullable=False, + ), + sa.Column("attempt_count", sa.Integer(), nullable=False), + sa.Column("last_attempt", sa.DateTime(timezone=True), nullable=True), + sa.Column("external_message_id", sa.String(length=255), nullable=True), + sa.Column("error_message", sa.Text(), nullable=True), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["alert_id"], ["alerts.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["destination_id"], ["notification_destinations.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("id"), + ) + op.create_index( + op.f("ix_notification_deliveries_alert_id"), + "notification_deliveries", + ["alert_id"], + unique=False, + ) + op.create_index( + op.f("ix_notification_deliveries_destination_id"), + "notification_deliveries", + ["destination_id"], + unique=False, + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index( + op.f("ix_notification_deliveries_destination_id"), table_name="notification_deliveries" + ) + op.drop_index(op.f("ix_notification_deliveries_alert_id"), table_name="notification_deliveries") + op.drop_table("notification_deliveries") + op.drop_index(op.f("ix_alerts_user_id"), table_name="alerts") + op.drop_index(op.f("ix_alerts_detected_change_id"), table_name="alerts") + op.drop_index(op.f("ix_alerts_company_id"), table_name="alerts") + op.drop_table("alerts") + # ### end Alembic commands ### diff --git a/apps/api/migrations/versions/c06c845cdd10_create_reports.py b/apps/api/migrations/versions/c06c845cdd10_create_reports.py new file mode 100644 index 0000000..0e4b288 --- /dev/null +++ b/apps/api/migrations/versions/c06c845cdd10_create_reports.py @@ -0,0 +1,68 @@ +"""create reports + +Revision ID: c06c845cdd10 +Revises: cb61ba9570fb +Create Date: 2026-07-31 19:52:22.947563 + +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "c06c845cdd10" +down_revision: str | None = "cb61ba9570fb" +branch_labels: Sequence[str] | str | None = None +depends_on: Sequence[str] | str | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "reports", + sa.Column("company_id", sa.Uuid(), nullable=False), + sa.Column("monitoring_run_id", sa.Uuid(), nullable=True), + sa.Column( + "report_type", + sa.Enum( + "BASELINE", + "UPDATE", + "MONTHLY", + "MANUAL", + name="reporttype", + native_enum=False, + length=20, + ), + nullable=False, + ), + sa.Column("title", sa.String(length=300), nullable=False), + sa.Column("executive_summary", sa.Text(), nullable=False), + sa.Column("structured_report", sa.JSON(), nullable=False), + sa.Column("markdown_content", sa.Text(), nullable=False), + sa.Column("model_provider", sa.String(length=50), nullable=False), + sa.Column("model_name", sa.String(length=100), nullable=False), + sa.Column("prompt_version", sa.String(length=20), nullable=False), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["company_id"], ["companies.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["monitoring_run_id"], ["monitoring_runs.id"], ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("id"), + ) + op.create_index(op.f("ix_reports_company_id"), "reports", ["company_id"], unique=False) + op.create_index( + op.f("ix_reports_monitoring_run_id"), "reports", ["monitoring_run_id"], unique=False + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f("ix_reports_monitoring_run_id"), table_name="reports") + op.drop_index(op.f("ix_reports_company_id"), table_name="reports") + op.drop_table("reports") + # ### end Alembic commands ### diff --git a/apps/api/migrations/versions/c34c769afc07_add_user_known_ips_table.py b/apps/api/migrations/versions/c34c769afc07_add_user_known_ips_table.py new file mode 100644 index 0000000..3bf1918 --- /dev/null +++ b/apps/api/migrations/versions/c34c769afc07_add_user_known_ips_table.py @@ -0,0 +1,41 @@ +"""add user known ips table + +Revision ID: c34c769afc07 +Revises: 8db5bd1c31f8 +Create Date: 2026-08-05 09:13:20.293276 + +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "c34c769afc07" +down_revision: str | None = "8db5bd1c31f8" +branch_labels: Sequence[str] | str | None = None +depends_on: Sequence[str] | str | None = None + + +def upgrade() -> None: + op.create_table( + "user_known_ips", + sa.Column("user_id", sa.Uuid(), nullable=False), + sa.Column("ip_address", sa.String(length=45), nullable=False), + sa.Column("first_seen_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("user_id", "ip_address", name="uq_user_known_ips_user_ip"), + ) + op.create_index(op.f("ix_user_known_ips_user_id"), "user_known_ips", ["user_id"], unique=False) + + +def downgrade() -> None: + op.drop_index(op.f("ix_user_known_ips_user_id"), table_name="user_known_ips") + op.drop_table("user_known_ips") diff --git a/apps/api/migrations/versions/cb61ba9570fb_create_detected_changes.py b/apps/api/migrations/versions/cb61ba9570fb_create_detected_changes.py new file mode 100644 index 0000000..8c1e15a --- /dev/null +++ b/apps/api/migrations/versions/cb61ba9570fb_create_detected_changes.py @@ -0,0 +1,107 @@ +"""create detected changes + +Revision ID: cb61ba9570fb +Revises: 8ceefbd0a6a5 +Create Date: 2026-07-31 19:34:49.775190 + +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "cb61ba9570fb" +down_revision: str | None = "8ceefbd0a6a5" +branch_labels: Sequence[str] | str | None = None +depends_on: Sequence[str] | str | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "detected_changes", + sa.Column("company_id", sa.Uuid(), nullable=False), + sa.Column("source_id", sa.Uuid(), nullable=False), + sa.Column("monitoring_run_id", sa.Uuid(), nullable=False), + sa.Column("previous_snapshot_id", sa.Uuid(), nullable=True), + sa.Column("current_snapshot_id", sa.Uuid(), nullable=False), + sa.Column( + "change_type", + sa.Enum( + "NEW_DOCUMENT", + "REMOVED_DOCUMENT", + "CONTENT_MODIFIED", + "PRICE_CHANGE", + "LEADERSHIP_CHANGE", + "FILING_NEW", + name="changetype", + native_enum=False, + length=30, + ), + nullable=False, + ), + sa.Column("raw_diff", sa.JSON(), nullable=False), + sa.Column("significance_score", sa.Float(), nullable=False), + sa.Column("confidence_score", sa.Float(), nullable=False), + sa.Column( + "severity", + sa.Enum( + "CRITICAL", + "HIGH", + "MEDIUM", + "LOW", + name="severitylevel", + native_enum=False, + length=20, + ), + nullable=False, + ), + sa.Column( + "status", + sa.Enum( + "NEW", + "ACKNOWLEDGED", + "DISMISSED", + name="changestatus", + native_enum=False, + length=20, + ), + nullable=False, + ), + sa.Column("summary", sa.String(length=500), nullable=False), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["company_id"], ["companies.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["current_snapshot_id"], ["snapshots.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["monitoring_run_id"], ["monitoring_runs.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["previous_snapshot_id"], ["snapshots.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["source_id"], ["sources.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("id"), + ) + op.create_index( + op.f("ix_detected_changes_company_id"), "detected_changes", ["company_id"], unique=False + ) + op.create_index( + op.f("ix_detected_changes_monitoring_run_id"), + "detected_changes", + ["monitoring_run_id"], + unique=False, + ) + op.create_index( + op.f("ix_detected_changes_source_id"), "detected_changes", ["source_id"], unique=False + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f("ix_detected_changes_source_id"), table_name="detected_changes") + op.drop_index(op.f("ix_detected_changes_monitoring_run_id"), table_name="detected_changes") + op.drop_index(op.f("ix_detected_changes_company_id"), table_name="detected_changes") + op.drop_table("detected_changes") + # ### end Alembic commands ### diff --git a/apps/api/migrations/versions/ef56f181dbb9_add_email_verification_ip_throttle_ban_.py b/apps/api/migrations/versions/ef56f181dbb9_add_email_verification_ip_throttle_ban_.py new file mode 100644 index 0000000..3841ae2 --- /dev/null +++ b/apps/api/migrations/versions/ef56f181dbb9_add_email_verification_ip_throttle_ban_.py @@ -0,0 +1,159 @@ +"""add email verification, ip throttle/ban, unban requests, user security events + +Revision ID: ef56f181dbb9 +Revises: 79e3aa041131 +Create Date: 2026-08-05 00:46:28.371645 + +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa + +revision: str = "ef56f181dbb9" +down_revision: str | None = "79e3aa041131" +branch_labels: Sequence[str] | str | None = None +depends_on: Sequence[str] | str | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "ip_bans", + sa.Column("ip_address", sa.String(length=45), nullable=False), + sa.Column("banned_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("reason", sa.String(length=255), nullable=False), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("id"), + ) + op.create_index(op.f("ix_ip_bans_ip_address"), "ip_bans", ["ip_address"], unique=True) + op.create_table( + "ip_throttle_state", + sa.Column("ip_address", sa.String(length=45), nullable=False), + sa.Column( + "action", + sa.Enum( + "RESEND_VERIFICATION", + "RESEND_RESET", + "FAILED_LOGIN", + name="throttleaction", + native_enum=False, + length=24, + ), + nullable=False, + ), + sa.Column("attempt_count", sa.Integer(), nullable=False), + sa.Column("next_allowed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("timeout_until", sa.DateTime(timezone=True), nullable=True), + sa.Column("offense_count", sa.Integer(), nullable=False), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("id"), + sa.UniqueConstraint("ip_address", "action", name="uq_ip_throttle_state_ip_action"), + ) + op.create_index( + op.f("ix_ip_throttle_state_ip_address"), "ip_throttle_state", ["ip_address"], unique=False + ) + op.create_table( + "unban_requests", + sa.Column("ip_address", sa.String(length=45), nullable=False), + sa.Column("message", sa.Text(), nullable=True), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("id"), + ) + op.create_index( + op.f("ix_unban_requests_ip_address"), "unban_requests", ["ip_address"], unique=False + ) + op.create_table( + "email_codes", + sa.Column("user_id", sa.Uuid(), nullable=False), + sa.Column( + "purpose", + sa.Enum( + "VERIFY_EMAIL", + "PASSWORD_RESET", + name="emailcodepurpose", + native_enum=False, + length=20, + ), + nullable=False, + ), + sa.Column("code_hash", sa.String(length=64), nullable=False), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("used_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("id"), + ) + op.create_index(op.f("ix_email_codes_code_hash"), "email_codes", ["code_hash"], unique=False) + op.create_table( + "user_security_events", + sa.Column("user_id", sa.Uuid(), nullable=False), + sa.Column( + "event_type", + sa.Enum( + "LOGIN_SUCCESS", + "LOGIN_FAILED", + "ACCOUNT_LOCKED", + "PASSWORD_RESET_REQUESTED", + "PASSWORD_RESET_COMPLETED", + "EMAIL_VERIFICATION_SENT", + "EMAIL_VERIFIED", + name="securityeventtype", + native_enum=False, + length=32, + ), + nullable=False, + ), + sa.Column("ip_address", sa.String(length=45), nullable=False), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("id"), + ) + op.create_index( + op.f("ix_user_security_events_user_id"), "user_security_events", ["user_id"], unique=False + ) + op.add_column( + "users", + sa.Column("email_verified", sa.Boolean(), nullable=False, server_default=sa.false()), + ) + op.add_column( + "users", sa.Column("failed_login_count", sa.Integer(), nullable=False, server_default="0") + ) + op.add_column("users", sa.Column("locked_at", sa.DateTime(timezone=True), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("users", "locked_at") + op.drop_column("users", "failed_login_count") + op.drop_column("users", "email_verified") + op.drop_index(op.f("ix_user_security_events_user_id"), table_name="user_security_events") + op.drop_table("user_security_events") + op.drop_index(op.f("ix_email_codes_code_hash"), table_name="email_codes") + op.drop_table("email_codes") + op.drop_index(op.f("ix_unban_requests_ip_address"), table_name="unban_requests") + op.drop_table("unban_requests") + op.drop_index(op.f("ix_ip_throttle_state_ip_address"), table_name="ip_throttle_state") + op.drop_table("ip_throttle_state") + op.drop_index(op.f("ix_ip_bans_ip_address"), table_name="ip_bans") + op.drop_table("ip_bans") + # ### end Alembic commands ### diff --git a/apps/api/migrations/versions/f01919a99ee9_add_company_headquarters_and_public_.py b/apps/api/migrations/versions/f01919a99ee9_add_company_headquarters_and_public_.py new file mode 100644 index 0000000..5688053 --- /dev/null +++ b/apps/api/migrations/versions/f01919a99ee9_add_company_headquarters_and_public_.py @@ -0,0 +1,39 @@ +"""add company headquarters and public_identifiers + +Revision ID: f01919a99ee9 +Revises: b5e0d93c89ac +Create Date: 2026-08-01 15:28:37.144461 + +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "f01919a99ee9" +down_revision: str | None = "b5e0d93c89ac" +branch_labels: Sequence[str] | str | None = None +depends_on: Sequence[str] | str | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("companies", sa.Column("headquarters", sa.String(length=200), nullable=True)) + # server_default backfills existing rows; NOT NULL alone would fail + # against a non-empty companies table (Postgres rejects adding a NOT + # NULL column with no default when rows already exist). + op.add_column( + "companies", + sa.Column("public_identifiers", sa.JSON(), nullable=False, server_default="{}"), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("companies", "public_identifiers") + op.drop_column("companies", "headquarters") + # ### end Alembic commands ### diff --git a/apps/api/pyproject.toml b/apps/api/pyproject.toml new file mode 100644 index 0000000..81d2b45 --- /dev/null +++ b/apps/api/pyproject.toml @@ -0,0 +1,77 @@ +[project] +name = "ci-agent-api" +version = "0.1.0" +description = "CI Agent backend API" +requires-python = ">=3.12" +dependencies = [ + "fastapi>=0.115", + "uvicorn[standard]>=0.32", + "pydantic>=2.9", + "pydantic-settings>=2.5", + "sqlalchemy>=2.0.35", + "alembic>=1.13", + "psycopg[binary,pool]>=3.2", + "aiosqlite>=0.20", + "greenlet>=3.1", + "celery>=5.4", + "redis>=5.1", + "httpx>=0.27", + "beautifulsoup4>=4.12", + "lxml>=5.3", + "trafilatura>=1.12", + "feedparser>=6.0", + "argon2-cffi>=23.1", + "cryptography>=43.0", + "pyjwt>=2.9", + "slowapi>=0.1.9", + "anthropic>=0.39", + "google-genai>=1.0", + "python-multipart>=0.0.12", + "tenacity>=9.0", + "structlog>=24.4", + "python-dateutil>=2.9", + "email-validator>=2.2", + "markdown-it-py>=3.0", + "croniter>=3.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.3", + "pytest-asyncio>=0.24", + "pytest-cov>=5.0", + "ruff>=0.7", + "black>=24.10", + "mypy>=1.13", + "faker>=30.8", + "respx>=0.21", +] + +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["app*"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] + +[tool.ruff] +line-length = 100 +target-version = "py312" + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B"] +# E501: line length handled by black. B008: FastAPI's Depends() pattern requires +# calling Depends(...) in argument defaults - that's idiomatic here, not a bug. +ignore = ["E501", "B008"] + +[tool.black] +line-length = 100 +target-version = ["py312"] + +[tool.mypy] +python_version = "3.12" +ignore_missing_imports = true diff --git a/apps/api/tests/__init__.py b/apps/api/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/tests/collectors/__init__.py b/apps/api/tests/collectors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/tests/collectors/conftest.py b/apps/api/tests/collectors/conftest.py new file mode 100644 index 0000000..65204d2 --- /dev/null +++ b/apps/api/tests/collectors/conftest.py @@ -0,0 +1,24 @@ +"""Collector tests never touch the real network or real DNS - every +hostname resolves to a fixed public IP, and respx intercepts the actual +HTTP layer per test.""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from app.collectors import robots as robots_module + + +@pytest.fixture(autouse=True) +def _no_real_dns(): + with patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]): + yield + + +@pytest.fixture(autouse=True) +def _reset_robots_cache(): + robots_module.clear_cache() + yield + robots_module.clear_cache() diff --git a/apps/api/tests/collectors/test_custom_url_collector.py b/apps/api/tests/collectors/test_custom_url_collector.py new file mode 100644 index 0000000..02d28a6 --- /dev/null +++ b/apps/api/tests/collectors/test_custom_url_collector.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import httpx +import pytest +import respx + +from app.collectors.base import CompanyContext, SourceConfig +from app.collectors.custom_url import CustomUrlCollector +from app.models.enums import SourceStatus, SourceType + +COMPANY = CompanyContext(id="c1", name="Acme Corp", official_website=None, monitoring_focus=None) + + +@pytest.mark.asyncio +async def test_collect_extracts_single_page(): + source = SourceConfig( + id="s1", + source_type=SourceType.CUSTOM_URL, + name="Pricing page", + base_url="https://example.com/pricing", + ) + with respx.mock: + respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404)) + respx.get("https://example.com/pricing").mock( + return_value=httpx.Response( + 200, + html=( + "Pricing" + "

Pricing

Plans start at $99/month for the base tier.

" + "
" + ), + ) + ) + result = await CustomUrlCollector().collect(source, COMPANY) + + assert result.status == SourceStatus.ACTIVE + assert len(result.documents) == 1 + assert "$99/month" in result.documents[0].content_text + + +@pytest.mark.asyncio +async def test_collect_reports_auth_required_on_403(): + source = SourceConfig( + id="s1", + source_type=SourceType.CUSTOM_URL, + name="Gated page", + base_url="https://example.com/gated", + ) + with respx.mock: + respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404)) + respx.get("https://example.com/gated").mock(return_value=httpx.Response(403)) + result = await CustomUrlCollector().collect(source, COMPANY) + + assert result.status == SourceStatus.AUTH_REQUIRED + + +@pytest.mark.asyncio +async def test_collect_respects_robots_disallow(): + source = SourceConfig( + id="s1", + source_type=SourceType.CUSTOM_URL, + name="Disallowed page", + base_url="https://example.com/private", + ) + with respx.mock: + respx.get("https://example.com/robots.txt").mock( + return_value=httpx.Response(200, text="User-agent: *\nDisallow: /private\n") + ) + result = await CustomUrlCollector().collect(source, COMPANY) + + assert result.status == SourceStatus.BLOCKED_BY_POLICY + assert result.documents == [] diff --git a/apps/api/tests/collectors/test_fixture_collectors.py b/apps/api/tests/collectors/test_fixture_collectors.py new file mode 100644 index 0000000..6545e47 --- /dev/null +++ b/apps/api/tests/collectors/test_fixture_collectors.py @@ -0,0 +1,75 @@ +"""Patent/review collectors: documented fixture adapters that must never +fabricate data when no provider or fixture is configured.""" + +from __future__ import annotations + +import pytest + +from app.collectors.base import CompanyContext, SourceConfig +from app.collectors.patents import PatentSourceCollector +from app.collectors.reviews import ReviewSourceCollector +from app.models.enums import SourceStatus, SourceType + +COMPANY = CompanyContext( + id="c1", name="Acme Mobility Systems", official_website=None, monitoring_focus=None +) + + +@pytest.mark.asyncio +async def test_patent_collector_disabled_without_fixture_key(): + source = SourceConfig(id="s1", source_type=SourceType.PATENT, name="Patents", base_url=None) + result = await PatentSourceCollector().collect(source, COMPANY) + assert result.status == SourceStatus.DISABLED + assert result.documents == [] + assert "does not fabricate" in result.error + + +@pytest.mark.asyncio +async def test_patent_collector_disabled_for_unknown_fixture_key(): + source = SourceConfig( + id="s1", + source_type=SourceType.PATENT, + name="Patents", + base_url=None, + configuration_metadata={"fixture_key": "does_not_exist"}, + ) + result = await PatentSourceCollector().collect(source, COMPANY) + assert result.status == SourceStatus.DISABLED + + +@pytest.mark.asyncio +async def test_patent_collector_loads_fixture_when_configured(): + source = SourceConfig( + id="s1", + source_type=SourceType.PATENT, + name="Patents", + base_url=None, + configuration_metadata={"fixture_key": "acme_mobility"}, + ) + result = await PatentSourceCollector().collect(source, COMPANY) + assert result.status == SourceStatus.ACTIVE + assert len(result.documents) == 1 + assert result.documents[0].metadata["is_fixture"] is True + + +@pytest.mark.asyncio +async def test_review_collector_disabled_without_fixture_key(): + source = SourceConfig(id="s1", source_type=SourceType.REVIEW, name="Reviews", base_url=None) + result = await ReviewSourceCollector().collect(source, COMPANY) + assert result.status == SourceStatus.DISABLED + assert result.documents == [] + + +@pytest.mark.asyncio +async def test_review_collector_loads_fixture_when_configured(): + source = SourceConfig( + id="s1", + source_type=SourceType.REVIEW, + name="Reviews", + base_url=None, + configuration_metadata={"fixture_key": "acme_mobility"}, + ) + result = await ReviewSourceCollector().collect(source, COMPANY) + assert result.status == SourceStatus.ACTIVE + assert len(result.documents) == 2 + assert all(d.metadata["is_fixture"] for d in result.documents) diff --git a/apps/api/tests/collectors/test_github_collector.py b/apps/api/tests/collectors/test_github_collector.py new file mode 100644 index 0000000..3d673f3 --- /dev/null +++ b/apps/api/tests/collectors/test_github_collector.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import json + +import httpx +import pytest +import respx + +from app.collectors.base import CompanyContext, SourceConfig +from app.collectors.github import GithubCollector +from app.models.enums import SourceStatus, SourceType + +COMPANY = CompanyContext(id="c1", name="Acme Corp", official_website=None, monitoring_focus=None) + + +@pytest.mark.asyncio +async def test_discover_finds_org_login(): + with respx.mock: + respx.get("https://api.github.com/search/users").mock( + return_value=httpx.Response(200, text=json.dumps({"items": [{"login": "acme-corp"}]})) + ) + discovered = await GithubCollector().discover(COMPANY) + + assert discovered[0].configuration_metadata["org"] == "acme-corp" + + +@pytest.mark.asyncio +async def test_collect_returns_repo_documents(): + source = SourceConfig( + id="s1", + source_type=SourceType.GITHUB, + name="Acme GitHub", + base_url="https://github.com/acme-corp", + configuration_metadata={"org": "acme-corp"}, + ) + repos = [ + { + "full_name": "acme-corp/battery-sdk", + "description": "SDK for battery management systems", + "language": "Python", + "stargazers_count": 42, + "pushed_at": "2026-05-01T12:00:00Z", + "html_url": "https://github.com/acme-corp/battery-sdk", + "archived": False, + } + ] + with respx.mock: + respx.get("https://api.github.com/orgs/acme-corp/repos").mock( + return_value=httpx.Response(200, text=json.dumps(repos)) + ) + result = await GithubCollector().collect(source, COMPANY) + + assert result.status == SourceStatus.ACTIVE + assert len(result.documents) == 1 + assert "battery management" in result.documents[0].content_text + + +@pytest.mark.asyncio +async def test_collect_reports_rate_limited_on_403(): + source = SourceConfig( + id="s1", + source_type=SourceType.GITHUB, + name="Acme GitHub", + base_url=None, + configuration_metadata={"org": "acme-corp"}, + ) + with respx.mock: + respx.get("https://api.github.com/orgs/acme-corp/repos").mock( + return_value=httpx.Response(403) + ) + result = await GithubCollector().collect(source, COMPANY) + + assert result.status == SourceStatus.RATE_LIMITED diff --git a/apps/api/tests/collectors/test_gov_contracts_collector.py b/apps/api/tests/collectors/test_gov_contracts_collector.py new file mode 100644 index 0000000..55258d8 --- /dev/null +++ b/apps/api/tests/collectors/test_gov_contracts_collector.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import httpx +import pytest +import respx + +from app.collectors.base import CompanyContext, SourceConfig +from app.collectors.gov_contracts import GovContractCollector +from app.models.enums import SourceStatus, SourceType + +COMPANY = CompanyContext(id="c1", name="Acme Corp", official_website=None, monitoring_focus=None) + + +@pytest.mark.asyncio +async def test_discover_always_returns_a_source_without_a_network_call(): + with respx.mock: + discovered = await GovContractCollector().discover(COMPANY) + + assert len(discovered) == 1 + assert discovered[0].source_type == SourceType.GOV_CONTRACT + assert discovered[0].configuration_metadata["recipient_search_text"] == "Acme Corp" + + +@pytest.mark.asyncio +async def test_collect_maps_awards_to_documents(): + source = SourceConfig( + id="s1", + source_type=SourceType.GOV_CONTRACT, + name="Acme Federal Contracts", + base_url=None, + configuration_metadata={"recipient_search_text": "Acme Corp"}, + ) + payload = { + "results": [ + { + "Award ID": "W91CRB-26-C-0001", + "Recipient Name": "Acme Corp", + "Award Amount": 4_500_000, + "Start Date": "2026-02-01", + "Awarding Agency": "Department of Defense", + "Description": "Supply of tactical equipment.", + } + ] + } + with respx.mock: + respx.post("https://api.usaspending.gov/api/v2/search/spending_by_award/").mock( + return_value=httpx.Response(200, json=payload) + ) + result = await GovContractCollector().collect(source, COMPANY) + + assert result.status == SourceStatus.ACTIVE + assert len(result.documents) == 1 + doc = result.documents[0] + assert doc.metadata["award_id"] == "W91CRB-26-C-0001" + assert "Department of Defense" in doc.title + assert "$4,500,000" in doc.title + + +@pytest.mark.asyncio +async def test_collect_zero_awards_is_not_a_failure(): + source = SourceConfig( + id="s1", + source_type=SourceType.GOV_CONTRACT, + name="Acme Federal Contracts", + base_url=None, + configuration_metadata={"recipient_search_text": "Acme Corp"}, + ) + with respx.mock: + respx.post("https://api.usaspending.gov/api/v2/search/spending_by_award/").mock( + return_value=httpx.Response(200, json={"results": []}) + ) + result = await GovContractCollector().collect(source, COMPANY) + + assert result.status == SourceStatus.ACTIVE + assert result.documents == [] + + +@pytest.mark.asyncio +async def test_collect_fails_gracefully_on_http_error(): + source = SourceConfig( + id="s1", + source_type=SourceType.GOV_CONTRACT, + name="Acme Federal Contracts", + base_url=None, + configuration_metadata={"recipient_search_text": "Acme Corp"}, + ) + with respx.mock: + respx.post("https://api.usaspending.gov/api/v2/search/spending_by_award/").mock( + return_value=httpx.Response(500, text="internal error") + ) + result = await GovContractCollector().collect(source, COMPANY) + + assert result.status == SourceStatus.FAILED + assert "500" in result.error diff --git a/apps/api/tests/collectors/test_job_posting_collector.py b/apps/api/tests/collectors/test_job_posting_collector.py new file mode 100644 index 0000000..a180ec6 --- /dev/null +++ b/apps/api/tests/collectors/test_job_posting_collector.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import httpx +import pytest +import respx + +from app.collectors.base import CompanyContext, SourceConfig +from app.collectors.jobs import JobPostingCollector +from app.models.enums import SourceStatus, SourceType + +CAREERS_HTML = """ + +

Careers at Acme

+ + +""" + +COMPANY = CompanyContext( + id="c1", name="Acme Corp", official_website="https://example.com", monitoring_focus=None +) + + +@pytest.mark.asyncio +async def test_discover_targets_careers_page(): + discovered = await JobPostingCollector().discover(COMPANY) + assert discovered[0].base_url == "https://example.com/careers" + + +@pytest.mark.asyncio +async def test_collect_extracts_job_listings_and_ignores_unrelated_links(): + source = SourceConfig( + id="s1", + source_type=SourceType.JOB_POSTING, + name="Acme Careers", + base_url="https://example.com/careers", + ) + with respx.mock: + respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404)) + respx.get("https://example.com/careers").mock( + return_value=httpx.Response(200, html=CAREERS_HTML) + ) + result = await JobPostingCollector().collect(source, COMPANY) + + assert result.status == SourceStatus.ACTIVE + titles = {d.title for d in result.documents} + assert titles == {"Senior Battery Engineer", "Battery Systems Technician"} + + +@pytest.mark.asyncio +async def test_collect_falls_back_to_whole_page_when_no_job_links_found(): + source = SourceConfig( + id="s1", + source_type=SourceType.JOB_POSTING, + name="Acme Careers", + base_url="https://example.com/careers", + ) + with respx.mock: + respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404)) + respx.get("https://example.com/careers").mock( + return_value=httpx.Response( + 200, + html="

Careers

We use an external ATS widget.

", + ) + ) + result = await JobPostingCollector().collect(source, COMPANY) + + assert result.status == SourceStatus.ACTIVE + assert len(result.documents) == 1 + assert result.documents[0].metadata["extraction"] == "fallback_whole_page" diff --git a/apps/api/tests/collectors/test_patents_collector.py b/apps/api/tests/collectors/test_patents_collector.py new file mode 100644 index 0000000..d377fb9 --- /dev/null +++ b/apps/api/tests/collectors/test_patents_collector.py @@ -0,0 +1,160 @@ +"""PatentSourceCollector's live USPTO Open Data Portal branch - only +active when `Settings.uspto_api_key` is configured. The no-key disabled/ +fixture path is covered separately in test_fixture_collectors.py and is +asserted here to stay unchanged. + +Live-verified (2026-08): USPTO's Patent Application Search has no +queryable assignee/company field at all, so `collect()` searches by each +of the company's known leadership names (from NinjaPear enrichment) +instead - see the module docstring in app/collectors/patents.py. +""" + +from __future__ import annotations + +import httpx +import pytest +import respx + +from app.collectors.base import CompanyContext, SourceConfig +from app.collectors.patents import PatentSourceCollector +from app.core.config import Settings +from app.models.enums import SourceStatus, SourceType + +COMPANY = CompanyContext( + id="c1", name="Acme Mobility", official_website=None, monitoring_focus=None +) +COMPANY_WITH_LEADERSHIP = CompanyContext( + id="c1", + name="Acme Mobility", + official_website=None, + monitoring_focus=None, + leadership_names=["Jane Doe", "John Smith"], +) + + +def _with_key(monkeypatch, key: str = "test-uspto-key") -> None: + monkeypatch.setattr("app.collectors.patents.get_settings", lambda: Settings(uspto_api_key=key)) + + +@pytest.mark.asyncio +async def test_discover_returns_empty_without_a_key(): + discovered = await PatentSourceCollector().discover(COMPANY) + assert discovered == [] + + +@pytest.mark.asyncio +async def test_discover_returns_a_source_when_a_key_is_configured(monkeypatch): + _with_key(monkeypatch) + discovered = await PatentSourceCollector().discover(COMPANY) + + assert len(discovered) == 1 + assert discovered[0].source_type == SourceType.PATENT + + +@pytest.mark.asyncio +async def test_collect_with_no_leadership_names_is_an_empty_result_with_no_network_call( + monkeypatch, +): + """No names to search USPTO's inventor index with (enrichment never + ran, still pending, or found no leadership) - an honest empty result, + not a wasted/failed call.""" + _with_key(monkeypatch) + source = SourceConfig( + id="s1", source_type=SourceType.PATENT, name="Acme Patents", base_url=None + ) + with respx.mock: + route = respx.post("https://api.uspto.gov/api/v1/patent/applications/search") + result = await PatentSourceCollector().collect(source, COMPANY) + + assert route.call_count == 0 + assert result.status == SourceStatus.ACTIVE + assert result.documents == [] + + +@pytest.mark.asyncio +async def test_collect_searches_by_each_leadership_name_and_dedupes(monkeypatch): + _with_key(monkeypatch) + source = SourceConfig( + id="s1", source_type=SourceType.PATENT, name="Acme Patents", base_url=None + ) + # title/filingDate/abstractText live under applicationMetaData, not at + # the entry's top level - matches the real API's confirmed live shape. + shared_patent = { + "applicationNumberText": "18/123456", + "applicationMetaData": { + "inventionTitle": "Battery Swapping System for Electric Vehicles", + "filingDate": "2026-01-15", + "abstractText": "A system for rapid battery exchange in electric scooters.", + }, + } + + with respx.mock: + route = respx.post("https://api.uspto.gov/api/v1/patent/applications/search").mock( + return_value=httpx.Response(200, json={"patentFileWrapperDataBag": [shared_patent]}) + ) + result = await PatentSourceCollector().collect(source, COMPANY_WITH_LEADERSHIP) + + # Both names searched (2 requests)... + assert route.call_count == 2 + assert route.calls[0].request.headers["x-api-key"] == "test-uspto-key" + for call in route.calls: + assert b"inventorNameText" in call.request.content + # ...but the same application number returned by both searches is + # only kept once. + assert result.status == SourceStatus.ACTIVE + assert len(result.documents) == 1 + doc = result.documents[0] + assert doc.metadata["application_number"] == "18/123456" + assert doc.metadata["match_type"] == "leadership_name_heuristic" + assert doc.trust_score == 0.5 + assert "Battery Swapping" in doc.title + assert "heuristic" in doc.content_text + + +@pytest.mark.asyncio +async def test_no_matching_records_for_a_name_is_not_a_failure(monkeypatch): + """USPTO returns 404 for "no matching records" rather than a 200 with + an empty array - a real, expected outcome for most names, not a + collection failure.""" + _with_key(monkeypatch) + source = SourceConfig( + id="s1", source_type=SourceType.PATENT, name="Acme Patents", base_url=None + ) + with respx.mock: + respx.post("https://api.uspto.gov/api/v1/patent/applications/search").mock( + return_value=httpx.Response(404, json={"message": "No matching records found"}) + ) + result = await PatentSourceCollector().collect(source, COMPANY_WITH_LEADERSHIP) + + assert result.status == SourceStatus.ACTIVE + assert result.documents == [] + + +@pytest.mark.asyncio +async def test_collect_surfaces_a_real_api_error_when_every_search_fails(monkeypatch): + """Once a key is configured, a real failure must be reported honestly - + not silently masked as the "no provider configured" disabled state, + which would look like the user's key was never even attempted.""" + _with_key(monkeypatch) + source = SourceConfig( + id="s1", source_type=SourceType.PATENT, name="Acme Patents", base_url=None + ) + with respx.mock: + respx.post("https://api.uspto.gov/api/v1/patent/applications/search").mock( + return_value=httpx.Response(401, text="Invalid API key") + ) + result = await PatentSourceCollector().collect(source, COMPANY_WITH_LEADERSHIP) + + assert result.status == SourceStatus.FAILED + assert "401" in result.error + + +@pytest.mark.asyncio +async def test_collect_without_a_key_is_unchanged_disabled_behavior(): + source = SourceConfig( + id="s1", source_type=SourceType.PATENT, name="Acme Patents", base_url=None + ) + result = await PatentSourceCollector().collect(source, COMPANY_WITH_LEADERSHIP) + + assert result.status == SourceStatus.DISABLED + assert "does not fabricate" in result.error diff --git a/apps/api/tests/collectors/test_rss_collector.py b/apps/api/tests/collectors/test_rss_collector.py new file mode 100644 index 0000000..43798d4 --- /dev/null +++ b/apps/api/tests/collectors/test_rss_collector.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import httpx +import pytest +import respx + +from app.collectors.base import CompanyContext, SourceConfig +from app.collectors.rss import RssCollector +from app.models.enums import SourceStatus, SourceType + +FEED_XML = """ + +Acme News + + Acme opens new facility + https://example.com/news/1 + Acme Corp opened a new manufacturing facility this week. + Mon, 01 Jun 2026 10:00:00 GMT + + + Acme hires new VP + https://example.com/news/2 + Acme Corp announced a new VP of Engineering. + + +""" + +COMPANY = CompanyContext(id="c1", name="Acme Corp", official_website=None, monitoring_focus=None) + + +@pytest.mark.asyncio +async def test_discover_builds_a_google_news_search_url(): + discovered = await RssCollector().discover(COMPANY) + + assert len(discovered) == 1 + assert discovered[0].source_type == SourceType.RSS + assert discovered[0].base_url.startswith("https://news.google.com/rss/search?q=") + assert "Acme%20Corp" in discovered[0].base_url + + +@pytest.mark.asyncio +async def test_discover_url_encodes_ampersands_in_the_name(): + company = CompanyContext(id="c1", name="AT&T", official_website=None, monitoring_focus=None) + discovered = await RssCollector().discover(company) + + # A raw "&" from the company name must not leak into the query string, + # since it would be parsed as a new URL parameter separator (splitting + # the query in two) rather than part of the search text. + query_part = discovered[0].base_url.split("q=", 1)[1].split("&hl=", 1)[0] + assert "&" not in query_part + + +@pytest.mark.asyncio +async def test_collect_parses_feed_items(): + source = SourceConfig( + id="s1", source_type=SourceType.RSS, name="Acme RSS", base_url="https://example.com/feed" + ) + with respx.mock: + respx.get("https://example.com/feed").mock(return_value=httpx.Response(200, text=FEED_XML)) + result = await RssCollector().collect(source, COMPANY) + + assert result.status == SourceStatus.ACTIVE + assert len(result.documents) == 2 + assert result.documents[0].title == "Acme opens new facility" + assert result.documents[0].publication_date is not None + + +@pytest.mark.asyncio +async def test_collect_fails_gracefully_on_http_error(): + source = SourceConfig( + id="s1", source_type=SourceType.RSS, name="Acme RSS", base_url="https://example.com/feed" + ) + with respx.mock: + respx.get("https://example.com/feed").mock(return_value=httpx.Response(500)) + result = await RssCollector().collect(source, COMPANY) + + assert result.status == SourceStatus.FAILED + assert result.documents == [] + + +@pytest.mark.asyncio +async def test_collect_without_base_url_fails_without_network_call(): + source = SourceConfig(id="s1", source_type=SourceType.RSS, name="Acme RSS", base_url=None) + result = await RssCollector().collect(source, COMPANY) + assert result.status == SourceStatus.FAILED diff --git a/apps/api/tests/collectors/test_sec_edgar_collector.py b/apps/api/tests/collectors/test_sec_edgar_collector.py new file mode 100644 index 0000000..4563fa8 --- /dev/null +++ b/apps/api/tests/collectors/test_sec_edgar_collector.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import json + +import httpx +import pytest +import respx + +from app.collectors.base import CompanyContext, SourceConfig +from app.collectors.sec_edgar import SecEdgarCollector +from app.models.enums import SourceStatus, SourceType + +ATOM_RESPONSE = """ + + + ACME CORP + CIK=0000320193&ScoreKeyed=true + + +""" + +COMPANY = CompanyContext(id="c1", name="Acme Corp", official_website=None, monitoring_focus=None) + + +@pytest.mark.asyncio +async def test_discover_finds_cik_from_atom_search(): + with respx.mock: + respx.get("https://www.sec.gov/cgi-bin/browse-edgar").mock( + return_value=httpx.Response(200, text=ATOM_RESPONSE) + ) + discovered = await SecEdgarCollector().discover(COMPANY) + + assert len(discovered) == 1 + assert discovered[0].configuration_metadata["cik"] == "0000320193" + + +@pytest.mark.asyncio +async def test_discover_returns_empty_when_no_match(): + with respx.mock: + respx.get("https://www.sec.gov/cgi-bin/browse-edgar").mock( + return_value=httpx.Response( + 200, text='' + ) + ) + discovered = await SecEdgarCollector().discover(COMPANY) + + assert discovered == [] + + +@pytest.mark.asyncio +async def test_collect_extracts_relevant_filings(): + source = SourceConfig( + id="s1", + source_type=SourceType.SEC_EDGAR, + name="Acme EDGAR", + base_url=None, + configuration_metadata={"cik": "0000320193"}, + ) + submissions = { + "name": "Acme Corp", + "filings": { + "recent": { + "form": ["10-K", "S-1", "8-K"], + "filingDate": ["2026-02-01", "2026-01-15", "2026-03-01"], + "accessionNumber": [ + "0000320193-26-000001", + "0000320193-26-000002", + "0000320193-26-000003", + ], + "primaryDocument": ["10k.htm", "s1.htm", "8k.htm"], + } + }, + } + with respx.mock: + respx.get("https://data.sec.gov/submissions/CIK0000320193.json").mock( + return_value=httpx.Response(200, text=json.dumps(submissions)) + ) + result = await SecEdgarCollector().collect(source, COMPANY) + + assert result.status == SourceStatus.ACTIVE + forms = {d.metadata["form"] for d in result.documents} + assert forms == {"10-K", "8-K"} # S-1 filtered out - not in _RELEVANT_FORMS + + +@pytest.mark.asyncio +async def test_collect_without_cik_fails_without_network_call(): + source = SourceConfig( + id="s1", source_type=SourceType.SEC_EDGAR, name="Acme EDGAR", base_url=None + ) + result = await SecEdgarCollector().collect(source, COMPANY) + assert result.status == SourceStatus.FAILED diff --git a/apps/api/tests/collectors/test_website_collector.py b/apps/api/tests/collectors/test_website_collector.py new file mode 100644 index 0000000..c6441f1 --- /dev/null +++ b/apps/api/tests/collectors/test_website_collector.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +import httpx +import pytest +import respx + +from app.collectors.base import CompanyContext, SourceConfig +from app.collectors.website import WebsiteCollector +from app.models.enums import SourceStatus, SourceType + +ARTICLE_HTML = """ + + +
+

{title}

+

{body}

+
+ +""" + +COMPANY = CompanyContext( + id="c1", name="Acme Corp", official_website="https://example.com", monitoring_focus=None +) + + +@pytest.mark.asyncio +async def test_discover_uses_sitemap_when_available(): + with respx.mock: + respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404)) + respx.get("https://example.com/sitemap.xml").mock( + return_value=httpx.Response( + 200, + content=( + '' + '' + "https://example.com/about" + "https://example.com/products" + "" + ), + ) + ) + collector = WebsiteCollector() + discovered = await collector.discover(COMPANY) + + assert len(discovered) == 1 + pages = discovered[0].configuration_metadata["pages"] + assert "https://example.com/about" in pages + assert "https://example.com/products" in pages + + +@pytest.mark.asyncio +async def test_discover_falls_back_to_heuristic_paths_without_sitemap(): + with respx.mock: + respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404)) + respx.get("https://example.com/sitemap.xml").mock(return_value=httpx.Response(404)) + collector = WebsiteCollector() + discovered = await collector.discover(COMPANY) + + pages = discovered[0].configuration_metadata["pages"] + assert any(p.endswith("/about") for p in pages) + assert any(p.endswith("/careers") for p in pages) + + +@pytest.mark.asyncio +async def test_collect_extracts_documents_from_configured_pages(): + source = SourceConfig( + id="s1", + source_type=SourceType.WEBSITE, + name="Acme website", + base_url="https://example.com", + configuration_metadata={ + "pages": ["https://example.com/about", "https://example.com/products"] + }, + ) + with respx.mock: + respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404)) + respx.get("https://example.com/about").mock( + return_value=httpx.Response( + 200, + html=ARTICLE_HTML.format( + title="About Acme", body="Acme Corp builds electric utility vehicles." + ), + ) + ) + respx.get("https://example.com/products").mock( + return_value=httpx.Response( + 200, + html=ARTICLE_HTML.format( + title="Products", body="Our product line includes trucks and vans." + ), + ) + ) + collector = WebsiteCollector() + result = await collector.collect(source, COMPANY) + + assert result.status == SourceStatus.ACTIVE + assert len(result.documents) == 2 + assert any("electric utility vehicles" in d.content_text for d in result.documents) + + +@pytest.mark.asyncio +async def test_collect_skips_pages_disallowed_by_robots_txt(): + source = SourceConfig( + id="s1", + source_type=SourceType.WEBSITE, + name="Acme website", + base_url="https://example.com", + configuration_metadata={"pages": ["https://example.com/private"]}, + ) + with respx.mock: + respx.get("https://example.com/robots.txt").mock( + return_value=httpx.Response(200, text="User-agent: *\nDisallow: /private\n") + ) + collector = WebsiteCollector() + result = await collector.collect(source, COMPANY) + + assert result.documents == [] + + +@pytest.mark.asyncio +async def test_collect_deduplicates_identical_content_across_pages(): + source = SourceConfig( + id="s1", + source_type=SourceType.WEBSITE, + name="Acme website", + base_url="https://example.com", + configuration_metadata={"pages": ["https://example.com/a", "https://example.com/a-mirror"]}, + ) + same_html = ARTICLE_HTML.format(title="Same", body="Identical content on both URLs.") + with respx.mock: + respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404)) + respx.get("https://example.com/a").mock(return_value=httpx.Response(200, html=same_html)) + respx.get("https://example.com/a-mirror").mock( + return_value=httpx.Response(200, html=same_html) + ) + collector = WebsiteCollector() + result = await collector.collect(source, COMPANY) + + assert len(result.documents) == 1 diff --git a/apps/api/tests/conftest.py b/apps/api/tests/conftest.py new file mode 100644 index 0000000..13a0d0e --- /dev/null +++ b/apps/api/tests/conftest.py @@ -0,0 +1,87 @@ +"""Shared pytest fixtures. + +Tests always run with LLM_PROVIDER=mock, SEARCH_PROVIDER=mock, a throwaway +SQLite DB, and Celery in eager mode - never against a paid provider or a +real network target. +""" + +from __future__ import annotations + +import os + +os.environ.setdefault("APP_ENV", "test") +os.environ.setdefault("AUTH_MODE", "jwt") +os.environ.setdefault("DATABASE_URL", "sqlite+aiosqlite:///./test_ciagent.db") +os.environ.setdefault("REDIS_URL", "redis://localhost:6379/15") +os.environ.setdefault("LLM_PROVIDER", "mock") +os.environ.setdefault("SEARCH_PROVIDER", "mock") +os.environ.setdefault("CELERY_TASK_ALWAYS_EAGER", "true") +os.environ.setdefault("JWT_SECRET", "test-secret-please-change-32-characters") +# No artificial per-domain delay in tests - the collector tests hit many +# distinct mocked hostnames and shouldn't pay the real-world crawl-politeness cost. +os.environ.setdefault("SCRAPER_DOMAIN_DELAY_SECONDS", "0") + +import asyncio +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from app.core.config import Settings, get_settings +from app.db.base import Base +from app.db.session import get_engine, get_sessionmaker +from app.main import app + + +@pytest.fixture(scope="session", autouse=True) +def _setup_database(): + # The DB is a throwaway file-based SQLite DB (aiosqlite doesn't support + # ":memory:" across the multiple connections a test session opens), so it + # must be deleted up front - otherwise companies/runs left behind by a + # previous test session accumulate and pollute count-based assertions + # (e.g. the scheduler "how many companies got enqueued" tests). + db_path = Path("test_ciagent.db") + db_path.unlink(missing_ok=True) + + async def _create() -> None: + engine = get_engine() + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_create()) + yield + + +@pytest.fixture(scope="session") +def settings() -> Settings: + return get_settings() + + +@pytest.fixture() +def client() -> TestClient: + with TestClient(app) as c: + yield c + + +@pytest.fixture() +async def db_session(): + """Raw AsyncSession for tests that exercise services/repositories + directly rather than through the HTTP API.""" + session_factory = get_sessionmaker() + async with session_factory() as session: + yield session + + +@pytest.fixture() +def local_mode_client(settings: Settings) -> TestClient: + """A client with AUTH_MODE forced to "local" via dependency override, + independent of whatever AUTH_MODE the rest of the suite runs under - and + with its TCP peer set to loopback, since the local-dev-user bypass + (app.auth.dependencies.get_current_user) only fires for a genuinely + local request, not merely AUTH_MODE=local. Starlette's TestClient + defaults to a fake ("testclient", 50000) peer otherwise.""" + local_settings = settings.model_copy(update={"auth_mode": "local"}) + app.dependency_overrides[get_settings] = lambda: local_settings + with TestClient(app, client=("127.0.0.1", 51234)) as c: + yield c + app.dependency_overrides.pop(get_settings, None) diff --git a/apps/api/tests/enrichment/__init__.py b/apps/api/tests/enrichment/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/tests/enrichment/test_mock_provider.py b/apps/api/tests/enrichment/test_mock_provider.py new file mode 100644 index 0000000..599875b --- /dev/null +++ b/apps/api/tests/enrichment/test_mock_provider.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import pytest + +from app.enrichment.mock import MockEnrichmentProvider + + +@pytest.mark.asyncio +async def test_mock_provider_returns_honest_empties_for_everything(): + provider = MockEnrichmentProvider() + + details = await provider.get_company_details("Acme Corp", "https://acme.example.com") + assert details.description is None + assert details.leadership_team == [] + + funding = await provider.get_funding("Acme Corp", None) + assert funding.total_raised is None + assert funding.rounds == [] + + assert await provider.get_updates("Acme Corp", None) == [] + assert await provider.get_competitors("Acme Corp", None) == [] + assert await provider.get_products("Acme Corp", None) == [] + assert await provider.get_customers("Acme Corp", None) == [] + assert await provider.get_work_email("Jane Doe", "https://acme.example.com") is None + assert await provider.get_person_profile("Jane Doe", "https://acme.example.com") == (None, None) diff --git a/apps/api/tests/enrichment/test_ninjapear_provider.py b/apps/api/tests/enrichment/test_ninjapear_provider.py new file mode 100644 index 0000000..a5bc2d4 --- /dev/null +++ b/apps/api/tests/enrichment/test_ninjapear_provider.py @@ -0,0 +1,186 @@ +"""NinjaPearProvider against the real API shape documented at +nubela.co/llms-full.txt: website-only company identification, and the +vendor's actual response field names (executives/employee_count, +total_funds_raised/funding_rounds, competitors keyed by website, three +separately-categorized customer/investor/partner arrays, first_name+domain +for work-email, x_profile_url for the person-profile endpoint).""" + +from __future__ import annotations + +import httpx +import pytest +import respx + +from app.core.config import Settings +from app.enrichment.ninjapear import NinjaPearProvider + +_SETTINGS = Settings(ninjapear_api_key="test-key") +_WEBSITE = "https://acme.example.com" + + +@pytest.mark.asyncio +async def test_get_company_details_parses_executives_and_employee_count(): + payload = { + "description": "A payments company.", + "industry": "Fintech", + "founded_year": 2010, + "specialties": ["Payments", "APIs"], + "employee_count": "1001-5000", + "executives": [ + {"name": "Jane Doe", "title": "CEO"}, + {"name": "", "title": "Ignored - no name"}, + ], + } + with respx.mock: + respx.get("https://nubela.co/api/v1/company/details").mock( + return_value=httpx.Response(200, json=payload) + ) + details = await NinjaPearProvider(_SETTINGS).get_company_details("Acme Corp", _WEBSITE) + + assert details.industry == "Fintech" + assert details.employee_count_range == "1001-5000" + assert len(details.leadership_team) == 1 + assert details.leadership_team[0].name == "Jane Doe" + + +@pytest.mark.asyncio +async def test_company_details_raises_without_a_website(): + with pytest.raises(ValueError, match="website"): + await NinjaPearProvider(_SETTINGS).get_company_details("Acme Corp", None) + + +@pytest.mark.asyncio +async def test_get_funding_maps_total_funds_raised_and_funding_rounds(): + # total_funds_raised/amount_usd are raw numbers, and each investor is + # an object (not a plain string) - matches the real API's shape. + payload = { + "total_funds_raised_usd": 500_000_000, + "funding_rounds": [ + { + "round_type": "Series C", + "amount_usd": 200_000_000, + "date": "2024-01-01", + "investors": [{"name": "VC Co", "type": "company", "website": None}], + } + ], + } + with respx.mock: + respx.get("https://nubela.co/api/v1/company/funding").mock( + return_value=httpx.Response(200, json=payload) + ) + funding = await NinjaPearProvider(_SETTINGS).get_funding("Acme Corp", _WEBSITE) + + assert funding.total_raised == "500000000" + assert funding.rounds[0].round_name == "Series C" + assert funding.rounds[0].amount == "200000000" + assert funding.rounds[0].investors == ["VC Co"] + + +@pytest.mark.asyncio +async def test_get_competitors_falls_back_to_website_when_no_name_given(): + payload = { + "competitors": [{"website": "rival.example.com", "competition_reason": "Same market"}] + } + with respx.mock: + respx.get("https://nubela.co/api/v1/competitor/listing").mock( + return_value=httpx.Response(200, json=payload) + ) + competitors = await NinjaPearProvider(_SETTINGS).get_competitors("Acme Corp", _WEBSITE) + + assert competitors[0].name == "rival.example.com" + assert competitors[0].reason == "Same market" + + +@pytest.mark.asyncio +async def test_get_customers_merges_the_three_categorized_arrays(): + payload = { + "customers": [{"name": "BigCo"}], + "investors": [{"name": "VC Fund"}], + "partner_platforms": [{"name": "PlatformCo"}], + } + with respx.mock: + respx.get("https://nubela.co/api/v1/customer/listing").mock( + return_value=httpx.Response(200, json=payload) + ) + customers = await NinjaPearProvider(_SETTINGS).get_customers("Acme Corp", _WEBSITE) + + by_relationship = {c.relationship: c.name for c in customers} + assert by_relationship == {"customer": "BigCo", "investor": "VC Fund", "partner": "PlatformCo"} + + +@pytest.mark.asyncio +async def test_get_products_joins_categories(): + payload = { + "products": [ + {"name": "Widget", "description": "A widget", "categories": ["Hardware", "IoT"]} + ] + } + with respx.mock: + respx.get("https://nubela.co/api/v1/product/listing").mock( + return_value=httpx.Response(200, json=payload) + ) + products = await NinjaPearProvider(_SETTINGS).get_products("Acme Corp", _WEBSITE) + + assert products[0].name == "Widget" + assert products[0].category == "Hardware, IoT" + + +@pytest.mark.asyncio +async def test_get_updates_maps_title_and_source(): + payload = { + "updates": [ + { + "title": "We shipped a feature", + "source": "blog", + "url": "https://x", + "timestamp": "2026-01-01", + } + ] + } + with respx.mock: + respx.get("https://nubela.co/api/v1/company/updates").mock( + return_value=httpx.Response(200, json=payload) + ) + updates = await NinjaPearProvider(_SETTINGS).get_updates("Acme Corp", _WEBSITE) + + assert updates[0].text == "We shipped a feature" + assert updates[0].type == "blog" + + +@pytest.mark.asyncio +async def test_get_work_email_sends_first_last_name_and_domain(): + with respx.mock: + route = respx.get("https://nubela.co/api/v1/employee/work-email").mock( + return_value=httpx.Response(200, json={"work_email": "jane@acme.example.com"}) + ) + email = await NinjaPearProvider(_SETTINGS).get_work_email("Jane Doe", _WEBSITE) + + assert email == "jane@acme.example.com" + sent_params = dict(route.calls[0].request.url.params) + assert sent_params == {"first_name": "Jane", "last_name": "Doe", "domain": "acme.example.com"} + + +@pytest.mark.asyncio +async def test_get_person_profile_hits_the_v2_endpoint_and_maps_x_profile_url(): + with respx.mock: + respx.get("https://nubela.co/api/v2/employee/profile").mock( + return_value=httpx.Response( + 200, json={"x_profile_url": "https://x.com/janedoe", "bio": "A bio."} + ) + ) + profile_url, bio = await NinjaPearProvider(_SETTINGS).get_person_profile( + "Jane Doe", _WEBSITE + ) + + assert profile_url == "https://x.com/janedoe" + assert bio == "A bio." + + +@pytest.mark.asyncio +async def test_a_provider_error_response_raises_rather_than_being_swallowed(): + with respx.mock: + respx.get("https://nubela.co/api/v1/company/details").mock( + return_value=httpx.Response(429, text="rate limited") + ) + with pytest.raises(httpx.HTTPStatusError): + await NinjaPearProvider(_SETTINGS).get_company_details("Acme Corp", _WEBSITE) diff --git a/apps/api/tests/fixtures/acme_mobility/v1/about.html b/apps/api/tests/fixtures/acme_mobility/v1/about.html new file mode 100644 index 0000000..c5a2b4e --- /dev/null +++ b/apps/api/tests/fixtures/acme_mobility/v1/about.html @@ -0,0 +1,13 @@ + +About Acme Mobility Systems + +
+

About Us

+

Acme Mobility Systems is a mobility technology startup based in Austin, Texas, founded in 2019. +We build fleet management software for micromobility operators running e-scooter and e-bike fleets +across North America.

+

Our CEO is Maria Chen, who co-founded the company after a decade in urban transportation planning.

+

Acme Mobility Systems is headquartered in Austin with a satellite engineering office in Denver.

+
+ + diff --git a/apps/api/tests/fixtures/acme_mobility/v1/careers.html b/apps/api/tests/fixtures/acme_mobility/v1/careers.html new file mode 100644 index 0000000..d09ef23 --- /dev/null +++ b/apps/api/tests/fixtures/acme_mobility/v1/careers.html @@ -0,0 +1,12 @@ + +Careers - Acme Mobility Systems + + + + diff --git a/apps/api/tests/fixtures/acme_mobility/v1/press.html b/apps/api/tests/fixtures/acme_mobility/v1/press.html new file mode 100644 index 0000000..2526e81 --- /dev/null +++ b/apps/api/tests/fixtures/acme_mobility/v1/press.html @@ -0,0 +1,12 @@ + +Press - Acme Mobility Systems + +
+

Press & News

+

March 2024 - Acme Mobility raises $8M Series A

+

Acme Mobility Systems today announced an $8M Series A financing round led by Northside Ventures, +with participation from existing seed investors. The funding will be used to expand FleetOS into +new metro markets.

+
+ + diff --git a/apps/api/tests/fixtures/acme_mobility/v1/pricing.html b/apps/api/tests/fixtures/acme_mobility/v1/pricing.html new file mode 100644 index 0000000..bd710dd --- /dev/null +++ b/apps/api/tests/fixtures/acme_mobility/v1/pricing.html @@ -0,0 +1,11 @@ + +Pricing - Acme Mobility Systems + +
+

Pricing

+

FleetOS Starter plan: $499/month for up to 200 vehicles. Includes telemetry, rebalancing +routing, and email support.

+

FleetOS Enterprise: custom pricing for fleets over 1,000 vehicles.

+
+ + diff --git a/apps/api/tests/fixtures/acme_mobility/v1/products.html b/apps/api/tests/fixtures/acme_mobility/v1/products.html new file mode 100644 index 0000000..d31de76 --- /dev/null +++ b/apps/api/tests/fixtures/acme_mobility/v1/products.html @@ -0,0 +1,11 @@ + +Products - Acme Mobility Systems + +
+

Products

+

Our flagship product, FleetOS, helps micromobility operators manage e-scooter and e-bike fleets: +vehicle telemetry, battery-swap routing, and dynamic rebalancing.

+

FleetOS integrates with most major vehicle hardware vendors via an open telemetry API.

+
+ + diff --git a/apps/api/tests/fixtures/acme_mobility/v2/about.html b/apps/api/tests/fixtures/acme_mobility/v2/about.html new file mode 100644 index 0000000..d89f841 --- /dev/null +++ b/apps/api/tests/fixtures/acme_mobility/v2/about.html @@ -0,0 +1,14 @@ + +About Acme Mobility Systems + +
+

About Us

+

Acme Mobility Systems is a mobility technology startup based in Austin, Texas, founded in 2019. +We build fleet management software for micromobility operators running e-scooter and e-bike fleets +across North America.

+

In June 2026, James Okafor was named the company's new Chief Executive Officer, succeeding +co-founder Maria Chen, who will remain on the board as Chair.

+

Acme Mobility Systems is headquartered in Austin with a satellite engineering office in Denver.

+
+ + diff --git a/apps/api/tests/fixtures/acme_mobility/v2/careers.html b/apps/api/tests/fixtures/acme_mobility/v2/careers.html new file mode 100644 index 0000000..19dc153 --- /dev/null +++ b/apps/api/tests/fixtures/acme_mobility/v2/careers.html @@ -0,0 +1,13 @@ + +Careers - Acme Mobility Systems + + + + diff --git a/apps/api/tests/fixtures/acme_mobility/v2/press.html b/apps/api/tests/fixtures/acme_mobility/v2/press.html new file mode 100644 index 0000000..20a0ff9 --- /dev/null +++ b/apps/api/tests/fixtures/acme_mobility/v2/press.html @@ -0,0 +1,15 @@ + +Press - Acme Mobility Systems + +
+

Press & News

+

June 2026 - Acme Mobility announces expansion into 12 new cities

+

Acme Mobility Systems today announced it is expanding FleetOS operations into 12 new metro +markets across the Southeast and Midwest by the end of the year, doubling its operating footprint.

+

March 2024 - Acme Mobility raises $8M Series A

+

Acme Mobility Systems today announced an $8M Series A financing round led by Northside Ventures, +with participation from existing seed investors. The funding will be used to expand FleetOS into +new metro markets.

+
+ + diff --git a/apps/api/tests/fixtures/acme_mobility/v2/pricing.html b/apps/api/tests/fixtures/acme_mobility/v2/pricing.html new file mode 100644 index 0000000..cc6c2ae --- /dev/null +++ b/apps/api/tests/fixtures/acme_mobility/v2/pricing.html @@ -0,0 +1,11 @@ + +Pricing - Acme Mobility Systems + +
+

Pricing

+

FleetOS Starter plan: $999/month for up to 200 vehicles. Includes telemetry, rebalancing +routing, and email support.

+

FleetOS Enterprise: custom pricing for fleets over 1,000 vehicles.

+
+ + diff --git a/apps/api/tests/fixtures/acme_mobility/v2/products.html b/apps/api/tests/fixtures/acme_mobility/v2/products.html new file mode 100644 index 0000000..d31de76 --- /dev/null +++ b/apps/api/tests/fixtures/acme_mobility/v2/products.html @@ -0,0 +1,11 @@ + +Products - Acme Mobility Systems + +
+

Products

+

Our flagship product, FleetOS, helps micromobility operators manage e-scooter and e-bike fleets: +vehicle telemetry, battery-swap routing, and dynamic rebalancing.

+

FleetOS integrates with most major vehicle hardware vendors via an open telemetry API.

+
+ + diff --git a/apps/api/tests/fixtures/patents/acme_mobility.json b/apps/api/tests/fixtures/patents/acme_mobility.json new file mode 100644 index 0000000..c75a2bf --- /dev/null +++ b/apps/api/tests/fixtures/patents/acme_mobility.json @@ -0,0 +1,11 @@ +{ + "patents": [ + { + "title": "Battery thermal management system for electric drivetrains", + "assignee": "Acme Mobility Systems", + "abstract": "A thermal management system for regulating temperature in high-density battery packs used in electric vehicle drivetrains, improving charge cycle longevity.", + "filed_date": "2026-03-14", + "url": "https://patents.example/acme-mobility/US-2026-000123" + } + ] +} diff --git a/apps/api/tests/fixtures/reviews/acme_mobility.json b/apps/api/tests/fixtures/reviews/acme_mobility.json new file mode 100644 index 0000000..595d9d3 --- /dev/null +++ b/apps/api/tests/fixtures/reviews/acme_mobility.json @@ -0,0 +1,20 @@ +{ + "reviews": [ + { + "title": "Reliable fleet vehicles", + "author": "Fleet Operator", + "rating": 4, + "date": "2026-05-02", + "body": "We've run Acme's gasoline utility trucks for two years with minimal downtime. Parts availability is good.", + "url": "https://reviews.example/acme-mobility/review-1" + }, + { + "title": "Support could be faster", + "author": "Small Business Owner", + "rating": 3, + "date": "2026-04-18", + "body": "Vehicles are solid but the support line has long wait times during peak season.", + "url": "https://reviews.example/acme-mobility/review-2" + } + ] +} diff --git a/apps/api/tests/integration/__init__.py b/apps/api/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/tests/integration/test_acme_fixture_demo.py b/apps/api/tests/integration/test_acme_fixture_demo.py new file mode 100644 index 0000000..3896580 --- /dev/null +++ b/apps/api/tests/integration/test_acme_fixture_demo.py @@ -0,0 +1,117 @@ +"""Regression test for the collection + change-detection pipeline using the +Acme Mobility Systems v1/v2 fixture HTML under tests/fixtures/acme_mobility/, +read straight off disk and run through the real pipeline end-to-end - so a +change to those fixtures, or a regression in the pipeline, breaks a test +here rather than going unnoticed. No live network: respx mocks every HTTP +call for a fictitious acme-mobility.example host.""" + +from __future__ import annotations + +import uuid +from pathlib import Path +from unittest.mock import patch + +import httpx +import pytest +import respx +from sqlalchemy import select +from sqlalchemy.orm import selectinload + +from app.models.company import Company +from app.models.enums import ChangeType, SourceType +from app.models.monitor_configuration import MonitorConfiguration +from app.repositories.source_repository import SnapshotRepository, SourceRepository +from app.services import collection_service +from app.services.change_detection_service import detect_change_for_source + +_FIXTURE_ROOT = Path(__file__).resolve().parent.parent / "fixtures" / "acme_mobility" +_HOST = "https://acme-mobility.example" +_PAGES = ["about", "products", "careers", "press", "pricing"] + + +def _page_html(version: str, page: str) -> str: + return (_FIXTURE_ROOT / version / f"{page}.html").read_text(encoding="utf-8") + + +@pytest.fixture(autouse=True) +def _no_real_dns(): + with patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]): + yield + + +async def _make_company(db_session) -> Company: + company = Company( + id=uuid.uuid4(), + user_id=uuid.uuid4(), + name="Acme Mobility Systems (Demo)", + slug=f"acme-mobility-demo-{uuid.uuid4().hex[:6]}", + official_website=f"{_HOST}/about", + monitoring_focus="leadership changes, pricing, hiring, expansion", + ) + db_session.add(company) + db_session.add(MonitorConfiguration(company_id=company.id)) + await db_session.commit() + result = await db_session.execute( + select(Company) + .where(Company.id == company.id) + .options( + selectinload(Company.aliases), + selectinload(Company.competitors), + selectinload(Company.enrichment), + ) + ) + return result.scalar_one() + + +def _mock_version(version: str): + respx.get(f"{_HOST}/robots.txt").mock(return_value=httpx.Response(404)) + for page in _PAGES: + respx.get(f"{_HOST}/{page}").mock( + return_value=httpx.Response(200, html=_page_html(version, page)) + ) + + +@pytest.mark.asyncio +async def test_acme_v1_to_v2_produces_the_expected_change_types(db_session, settings): + company = await _make_company(db_session) + repo = SourceRepository(db_session) + sources = {} + for page in _PAGES: + sources[page] = await repo.create( + company_id=company.id, + source_type=SourceType.CUSTOM_URL, + name=f"Acme Mobility - {page.title()}", + base_url=f"{_HOST}/{page}", + ) + await db_session.commit() + + # Baseline run (v1): every source collects successfully, no prior + # snapshot to diff against yet. + with respx.mock: + _mock_version("v1") + for page in _PAGES: + result = await collection_service.collect_source( + db_session, settings, sources[page], company + ) + assert result.status.value == "active", f"{page} baseline collection failed" + + # Second run (v2): about/careers/press/pricing changed, products did not. + with respx.mock: + _mock_version("v2") + for page in _PAGES: + await collection_service.collect_source(db_session, settings, sources[page], company) + + snapshot_repo = SnapshotRepository(db_session) + changes: dict[str, ChangeType | None] = {} + for page in _PAGES: + current = await snapshot_repo.latest_for_source(sources[page].id) + change = await detect_change_for_source( + db_session, sources[page], company, current, uuid.uuid4() + ) + changes[page] = change.change_type if change else None + + assert changes["about"] == ChangeType.LEADERSHIP_CHANGE + assert changes["pricing"] == ChangeType.PRICE_CHANGE + assert changes["products"] is None # identical content -> hash short-circuit, no change + assert changes["careers"] is not None # new job listing -> detected as a real change + assert changes["press"] is not None # new press entry -> detected as a real change diff --git a/apps/api/tests/integration/test_alert_service.py b/apps/api/tests/integration/test_alert_service.py new file mode 100644 index 0000000..d441d63 --- /dev/null +++ b/apps/api/tests/integration/test_alert_service.py @@ -0,0 +1,326 @@ +"""Alert creation end-to-end against a real (SQLite) DB: a DetectedChange +above the company's threshold becomes an Alert (via the mock LLM's Task F), +gets dispatched to every enabled destination that also meets its own +threshold, and a NotificationDelivery is recorded per attempt. Two +independent thresholds by design - see alert_service.py docstring.""" + +from __future__ import annotations + +import uuid + +import pytest +from sqlalchemy import select + +from app.analysis.llm.mock import MockLLMProvider +from app.core.config import Settings +from app.models.company import Company +from app.models.detected_change import DetectedChange +from app.models.enums import ( + ChangeType, + MonitoringRunTrigger, + NotificationDeliveryStatus, + NotificationType, + SeverityLevel, + SourceType, +) +from app.models.monitor_configuration import MonitorConfiguration +from app.models.monitoring_run import MonitoringRun +from app.models.notification_delivery import NotificationDelivery +from app.models.notification_destination import ( + NotificationDestination, + NotificationDestinationCompany, +) +from app.models.snapshot import Snapshot +from app.models.source import Source +from app.repositories.company_repository import CompanyRepository +from app.services.alert_service import create_alert_for_change, send_test_notification + + +async def _make_company( + db_session, *, severity_threshold: SeverityLevel = SeverityLevel.MEDIUM +) -> Company: + company = Company( + id=uuid.uuid4(), + user_id=uuid.uuid4(), + name="Acme Mobility Systems", + slug=f"acme-mobility-{uuid.uuid4().hex[:6]}", + ) + db_session.add(company) + await db_session.flush() + + db_session.add( + MonitorConfiguration(company_id=company.id, severity_threshold=severity_threshold) + ) + await db_session.commit() + + return await CompanyRepository(db_session).get_for_user(company.id, company.user_id) + + +async def _make_change(db_session, company: Company, *, severity: SeverityLevel) -> DetectedChange: + source = Source( + company_id=company.id, + source_type=SourceType.WEBSITE, + name="Website", + base_url="https://acme.example", + ) + db_session.add(source) + await db_session.flush() + + run = MonitoringRun(company_id=company.id, trigger_type=MonitoringRunTrigger.MANUAL) + db_session.add(run) + await db_session.flush() + + snapshot = Snapshot( + company_id=company.id, + source_id=source.id, + snapshot_type=source.source_type.value, + hash="hash1", + structured_summary={}, + text_summary="", + monitoring_run_id=run.id, + ) + db_session.add(snapshot) + await db_session.flush() + + change = DetectedChange( + company_id=company.id, + source_id=source.id, + monitoring_run_id=run.id, + current_snapshot_id=snapshot.id, + change_type=ChangeType.NEW_DOCUMENT, + raw_diff={ + "text_added_lines": ["We're hiring a new VP of Engineering."], + "structured_added": ["/careers/vp-engineering"], + }, + significance_score=0.7, + confidence_score=0.8, + severity=severity, + summary="New leadership hire posting detected", + ) + db_session.add(change) + await db_session.commit() + await db_session.refresh(change) + return change + + +@pytest.mark.asyncio +async def test_change_below_company_threshold_creates_no_alert(db_session, settings: Settings): + company = await _make_company(db_session, severity_threshold=SeverityLevel.HIGH) + change = await _make_change(db_session, company, severity=SeverityLevel.LOW) + + alert = await create_alert_for_change(db_session, settings, MockLLMProvider(), change, company) + + assert alert is None + + +@pytest.mark.asyncio +async def test_change_at_threshold_creates_alert_with_llm_summary(db_session, settings: Settings): + company = await _make_company(db_session, severity_threshold=SeverityLevel.MEDIUM) + change = await _make_change(db_session, company, severity=SeverityLevel.HIGH) + + alert = await create_alert_for_change(db_session, settings, MockLLMProvider(), change, company) + + assert alert is not None + assert alert.company_id == company.id + assert alert.detected_change_id == change.id + assert alert.severity == SeverityLevel.HIGH + assert alert.confidence == change.confidence_score + assert alert.title + assert alert.summary + assert alert.why_it_matters + assert alert.read is False + assert alert.resolved is False + + +@pytest.mark.asyncio +async def test_alert_dispatches_only_to_destinations_meeting_their_own_threshold( + db_session, settings: Settings, monkeypatch +): + # EMAIL delivery goes through the real SmtpEmailProvider - mock the + # socket-level smtplib call rather than depending on a live SMTP + # relay (e.g. Mailpit) actually being reachable in the test environment. + class FakeSmtp: + def __init__(self, host, port, timeout=10): + pass + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def starttls(self): + pass + + def login(self, username, password): + pass + + def sendmail(self, from_addr, to_addrs, message): + pass + + monkeypatch.setattr("app.notifications.smtp_email.smtplib.SMTP", FakeSmtp) + + company = await _make_company(db_session, severity_threshold=SeverityLevel.LOW) + change = await _make_change(db_session, company, severity=SeverityLevel.MEDIUM) + + low_bar = NotificationDestination( + user_id=company.user_id, + type=NotificationType.EMAIL, + destination_value="gets-everything@example.com", + minimum_severity=SeverityLevel.LOW, + enabled=True, + ) + high_bar = NotificationDestination( + user_id=company.user_id, + type=NotificationType.EMAIL, + destination_value="critical-only@example.com", + minimum_severity=SeverityLevel.CRITICAL, + enabled=True, + ) + disabled = NotificationDestination( + user_id=company.user_id, + type=NotificationType.EMAIL, + destination_value="disabled@example.com", + minimum_severity=SeverityLevel.LOW, + enabled=False, + ) + db_session.add_all([low_bar, high_bar, disabled]) + await db_session.commit() + db_session.add_all( + NotificationDestinationCompany(destination_id=d.id, company_id=company.id) + for d in (low_bar, high_bar, disabled) + ) + await db_session.commit() + + alert = await create_alert_for_change(db_session, settings, MockLLMProvider(), change, company) + assert alert is not None + + deliveries = ( + ( + await db_session.execute( + select(NotificationDelivery).where(NotificationDelivery.alert_id == alert.id) + ) + ) + .scalars() + .all() + ) + + assert len(deliveries) == 1 + assert deliveries[0].destination_id == low_bar.id + assert deliveries[0].status == NotificationDeliveryStatus.SENT + assert deliveries[0].provider == "smtp" + + +@pytest.mark.asyncio +async def test_sms_destination_skipped_when_sms_disabled(db_session, settings: Settings): + company = await _make_company(db_session, severity_threshold=SeverityLevel.LOW) + change = await _make_change(db_session, company, severity=SeverityLevel.MEDIUM) + + sms_destination = NotificationDestination( + user_id=company.user_id, + type=NotificationType.SMS, + destination_value="+15551234567", + minimum_severity=SeverityLevel.LOW, + enabled=True, + ) + db_session.add(sms_destination) + await db_session.commit() + + disabled_sms_settings = settings.model_copy(update={"notification_sms_enabled": False}) + alert = await create_alert_for_change( + db_session, disabled_sms_settings, MockLLMProvider(), change, company + ) + assert alert is not None + + deliveries = ( + ( + await db_session.execute( + select(NotificationDelivery).where(NotificationDelivery.alert_id == alert.id) + ) + ) + .scalars() + .all() + ) + assert deliveries == [] + + +@pytest.mark.asyncio +async def test_failed_delivery_is_recorded_with_error(db_session, settings: Settings): + company = await _make_company(db_session, severity_threshold=SeverityLevel.LOW) + change = await _make_change(db_session, company, severity=SeverityLevel.MEDIUM) + + unconfigured_sms_settings = settings.model_copy( + update={ + "notification_sms_enabled": True, + "twilio_account_sid": "", + "twilio_auth_token": "", + "twilio_from_number": "", + } + ) + sms_destination = NotificationDestination( + user_id=company.user_id, + type=NotificationType.SMS, + destination_value="+15551234567", + minimum_severity=SeverityLevel.LOW, + enabled=True, + ) + db_session.add(sms_destination) + await db_session.commit() + db_session.add( + NotificationDestinationCompany(destination_id=sms_destination.id, company_id=company.id) + ) + await db_session.commit() + + alert = await create_alert_for_change( + db_session, unconfigured_sms_settings, MockLLMProvider(), change, company + ) + assert alert is not None + + deliveries = ( + ( + await db_session.execute( + select(NotificationDelivery).where(NotificationDelivery.alert_id == alert.id) + ) + ) + .scalars() + .all() + ) + assert len(deliveries) == 1 + assert deliveries[0].status == NotificationDeliveryStatus.FAILED + assert "not configured" in deliveries[0].error_message + + +@pytest.mark.asyncio +async def test_send_test_notification_short_circuits_sms_when_disabled( + db_session, settings: Settings +): + """The "Send test notification" button must respect NOTIFICATION_SMS_ENABLED + the same way real alert dispatch does - it must never place a real API + call to an SMS vendor while SMS delivery is switched off, even though a + Twilio/Telnyx-configured provider would otherwise happily send one.""" + user_id = uuid.uuid4() + sms_destination = NotificationDestination( + user_id=user_id, + type=NotificationType.SMS, + destination_value="+15551234567", + minimum_severity=SeverityLevel.LOW, + enabled=True, + ) + db_session.add(sms_destination) + await db_session.commit() + await db_session.refresh(sms_destination) + + disabled_sms_settings = settings.model_copy( + update={ + "notification_sms_enabled": False, + "sms_provider": "telnyx", + "telnyx_api_key": "would-be-a-real-key", + "telnyx_from_number": "+15559990000", + } + ) + result = await send_test_notification( + db_session, disabled_sms_settings, user_id, sms_destination.id + ) + + assert result.success is False + assert "disabled" in result.error diff --git a/apps/api/tests/integration/test_analytics_service.py b/apps/api/tests/integration/test_analytics_service.py new file mode 100644 index 0000000..7878505 --- /dev/null +++ b/apps/api/tests/integration/test_analytics_service.py @@ -0,0 +1,169 @@ +"""Dashboard analytics: aggregate counts scoped correctly to the requesting +user (never leaking another user's data), zero-filled for enum members with +no data, and recent signals ordered newest-first.""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime + +import pytest + +from app.models.alert import Alert +from app.models.company import Company +from app.models.detected_change import DetectedChange +from app.models.enums import ( + ChangeType, + MonitoringRunStatus, + MonitoringRunTrigger, + SeverityLevel, + SourceType, +) +from app.models.monitoring_run import MonitoringRun +from app.models.snapshot import Snapshot +from app.models.source import Source +from app.services.analytics_service import get_dashboard_analytics + + +async def _make_company_with_activity(db_session, user_id: uuid.UUID) -> Company: + company = Company( + id=uuid.uuid4(), user_id=user_id, name="Acme Corp", slug=f"acme-{uuid.uuid4().hex[:6]}" + ) + db_session.add(company) + await db_session.flush() + + source = Source( + company_id=company.id, + source_type=SourceType.WEBSITE, + name="Site", + base_url="https://acme.example", + ) + db_session.add(source) + await db_session.flush() + + run = MonitoringRun( + company_id=company.id, + trigger_type=MonitoringRunTrigger.MANUAL, + status=MonitoringRunStatus.SUCCESSFUL, + ) + db_session.add(run) + await db_session.flush() + + snapshot = Snapshot( + company_id=company.id, + source_id=source.id, + snapshot_type=source.source_type.value, + hash="h1", + structured_summary={}, + text_summary="", + monitoring_run_id=run.id, + ) + db_session.add(snapshot) + await db_session.flush() + + change = DetectedChange( + company_id=company.id, + source_id=source.id, + monitoring_run_id=run.id, + current_snapshot_id=snapshot.id, + change_type=ChangeType.LEADERSHIP_CHANGE, + raw_diff={}, + significance_score=0.6, + confidence_score=0.8, + severity=SeverityLevel.HIGH, + summary="New CEO announced", + ) + db_session.add(change) + await db_session.flush() + + alert = Alert( + company_id=company.id, + detected_change_id=change.id, + user_id=user_id, + title="Leadership change", + summary="New CEO announced", + why_it_matters="Signals a strategy shift", + severity=SeverityLevel.HIGH, + confidence=0.8, + ) + db_session.add(alert) + await db_session.commit() + return company + + +@pytest.mark.asyncio +async def test_analytics_aggregates_counts_for_the_requesting_user(db_session): + user_id = uuid.uuid4() + await _make_company_with_activity(db_session, user_id) + + analytics = await get_dashboard_analytics(db_session, user_id) + + assert analytics.changes_by_type["leadership_change"] == 1 + assert analytics.changes_by_type["price_change"] == 0 + assert analytics.alerts_by_severity["high"] == 1 + assert analytics.alerts_by_severity["critical"] == 0 + assert analytics.sources_by_status["active"] == 1 + assert len(analytics.recent_signals) == 1 + assert analytics.recent_signals[0].company_name == "Acme Corp" + assert analytics.recent_signals[0].change_type == "leadership_change" + + +@pytest.mark.asyncio +async def test_analytics_does_not_leak_another_users_data(db_session): + user_id = uuid.uuid4() + other_user_id = uuid.uuid4() + await _make_company_with_activity(db_session, other_user_id) + + analytics = await get_dashboard_analytics(db_session, user_id) + + assert all(count == 0 for count in analytics.changes_by_type.values()) + assert all(count == 0 for count in analytics.alerts_by_severity.values()) + assert analytics.recent_signals == [] + + +@pytest.mark.asyncio +async def test_analytics_recent_signals_ordered_newest_first(db_session): + user_id = uuid.uuid4() + company = await _make_company_with_activity(db_session, user_id) + + source = Source( + company_id=company.id, + source_type=SourceType.WEBSITE, + name="Site 2", + base_url="https://acme.example/2", + ) + db_session.add(source) + await db_session.flush() + run = MonitoringRun(company_id=company.id, trigger_type=MonitoringRunTrigger.MANUAL) + db_session.add(run) + await db_session.flush() + snapshot = Snapshot( + company_id=company.id, + source_id=source.id, + snapshot_type=source.source_type.value, + hash="h2", + structured_summary={}, + text_summary="", + monitoring_run_id=run.id, + ) + db_session.add(snapshot) + await db_session.flush() + newer_change = DetectedChange( + company_id=company.id, + source_id=source.id, + monitoring_run_id=run.id, + current_snapshot_id=snapshot.id, + change_type=ChangeType.PRICE_CHANGE, + raw_diff={}, + significance_score=0.5, + confidence_score=0.7, + severity=SeverityLevel.MEDIUM, + summary="Price increased", + created_at=datetime.now(UTC), + ) + db_session.add(newer_change) + await db_session.commit() + + analytics = await get_dashboard_analytics(db_session, user_id) + + assert analytics.recent_signals[0].change_type == "price_change" diff --git a/apps/api/tests/integration/test_change_detection_service.py b/apps/api/tests/integration/test_change_detection_service.py new file mode 100644 index 0000000..64a642d --- /dev/null +++ b/apps/api/tests/integration/test_change_detection_service.py @@ -0,0 +1,372 @@ +"""End-to-end (DB-backed) tests for change_detection_service: two real +Snapshot rows in, a DetectedChange (or None) out. Collectors themselves are +already covered elsewhere; this exercises the diff/scoring/persistence +pipeline directly against SQLite.""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime, timedelta + +import pytest +from sqlalchemy import select +from sqlalchemy.orm import selectinload + +from app.models.company import Company +from app.models.detected_change import DetectedChange +from app.models.enums import ChangeStatus, ChangeType, SourceType +from app.models.snapshot import Snapshot +from app.models.source import Source +from app.repositories.source_repository import SnapshotRepository, SourceRepository +from app.services.change_detection_service import detect_change_for_source + + +async def _make_company(db_session, *, monitoring_focus: str | None = None) -> Company: + company = Company( + id=uuid.uuid4(), + user_id=uuid.uuid4(), + name="Acme Corp", + slug=f"acme-corp-{uuid.uuid4().hex[:6]}", + monitoring_focus=monitoring_focus, + ) + db_session.add(company) + await db_session.commit() + result = await db_session.execute( + select(Company).where(Company.id == company.id).options(selectinload(Company.aliases)) + ) + return result.scalar_one() + + +async def _make_source( + db_session, company: Company, *, source_type: SourceType = SourceType.WEBSITE, trust_score=0.8 +) -> Source: + return await SourceRepository(db_session).create( + company_id=company.id, + source_type=source_type, + name="Test Source", + base_url="https://example.com", + trust_score=trust_score, + ) + + +async def _make_snapshot( + db_session, + company: Company, + source: Source, + *, + content_hash: str, + urls: list[str], + text_summary: str, + created_at: datetime | None = None, +) -> Snapshot: + snapshot = await SnapshotRepository(db_session).create( + company_id=company.id, + source_id=source.id, + snapshot_type=source.source_type.value, + hash=content_hash, + structured_summary={"urls": urls, "titles": [], "document_count": len(urls)}, + text_summary=text_summary, + ) + if created_at is not None: + snapshot.created_at = created_at + await db_session.commit() + return snapshot + + +@pytest.mark.asyncio +async def test_no_previous_snapshot_produces_no_change(db_session): + company = await _make_company(db_session) + source = await _make_source(db_session, company) + current = await _make_snapshot( + db_session, company, source, content_hash="h1", urls=["/a"], text_summary="A" + ) + + result = await detect_change_for_source(db_session, source, company, current, uuid.uuid4()) + assert result is None + + +@pytest.mark.asyncio +async def test_identical_hash_produces_no_change(db_session): + company = await _make_company(db_session) + source = await _make_source(db_session, company) + await _make_snapshot( + db_session, + company, + source, + content_hash="same-hash", + urls=["/a"], + text_summary="A", + created_at=datetime.now(UTC) - timedelta(hours=1), + ) + current = await _make_snapshot( + db_session, company, source, content_hash="same-hash", urls=["/a"], text_summary="A" + ) + + result = await detect_change_for_source(db_session, source, company, current, uuid.uuid4()) + assert result is None + + +@pytest.mark.asyncio +async def test_new_item_detected_as_new_document(db_session): + company = await _make_company(db_session) + source = await _make_source(db_session, company) + await _make_snapshot( + db_session, + company, + source, + content_hash="h1", + urls=["/careers/job-a"], + text_summary="### Job A\nExisting posting.", + created_at=datetime.now(UTC) - timedelta(hours=1), + ) + current = await _make_snapshot( + db_session, + company, + source, + content_hash="h2", + urls=["/careers/job-a", "/careers/job-b"], + text_summary="### Job A\nExisting posting.\n\n### Job B\nNew battery engineer role.", + ) + + change = await detect_change_for_source(db_session, source, company, current, uuid.uuid4()) + + assert change is not None + assert change.change_type == ChangeType.NEW_DOCUMENT + assert change.status == ChangeStatus.NEW + assert "/careers/job-b" in change.raw_diff["structured_added"] + + +@pytest.mark.asyncio +async def test_leadership_mention_takes_priority_over_new_document(db_session): + company = await _make_company(db_session) + source = await _make_source(db_session, company) + await _make_snapshot( + db_session, + company, + source, + content_hash="h1", + urls=["/press/1"], + text_summary="### Old release\nRoutine update.", + created_at=datetime.now(UTC) - timedelta(hours=1), + ) + current = await _make_snapshot( + db_session, + company, + source, + content_hash="h2", + urls=["/press/1", "/press/2"], + text_summary=( + "### Old release\nRoutine update.\n\n" + "### New release\nJane Smith has been named the company's new CEO effective immediately." + ), + ) + + change = await detect_change_for_source(db_session, source, company, current, uuid.uuid4()) + + # This test is about classification priority (leadership beats the + # new-document signal that's also present), not about the resulting + # severity number - see test_scoring.py for severity-formula coverage. + assert change is not None + assert change.change_type == ChangeType.LEADERSHIP_CHANGE + + +@pytest.mark.asyncio +async def test_price_mention_change_detected(db_session): + company = await _make_company(db_session) + source = await _make_source(db_session, company) + await _make_snapshot( + db_session, + company, + source, + content_hash="h1", + urls=["/pricing"], + text_summary="### Pricing\nThe base plan is $49/month.", + created_at=datetime.now(UTC) - timedelta(hours=1), + ) + current = await _make_snapshot( + db_session, + company, + source, + content_hash="h2", + urls=["/pricing"], + text_summary="### Pricing\nThe base plan is $59/month.", + ) + + change = await detect_change_for_source(db_session, source, company, current, uuid.uuid4()) + + assert change is not None + assert change.change_type == ChangeType.PRICE_CHANGE + + +@pytest.mark.asyncio +async def test_sec_edgar_new_filing_detected(db_session): + company = await _make_company(db_session) + source = await _make_source(db_session, company, source_type=SourceType.SEC_EDGAR) + await _make_snapshot( + db_session, + company, + source, + content_hash="h1", + urls=["/filings/10-K-2025"], + text_summary="### 10-K\nAnnual report.", + created_at=datetime.now(UTC) - timedelta(hours=1), + ) + current = await _make_snapshot( + db_session, + company, + source, + content_hash="h2", + urls=["/filings/10-K-2025", "/filings/8-K-2026"], + text_summary="### 10-K\nAnnual report.\n\n### 8-K\nCurrent report.", + ) + + change = await detect_change_for_source(db_session, source, company, current, uuid.uuid4()) + + assert change is not None + assert change.change_type == ChangeType.FILING_NEW + + +@pytest.mark.asyncio +async def test_content_modified_below_threshold_is_not_reported(db_session): + company = await _make_company(db_session) + source = await _make_source(db_session, company) + # Bounded text diff operates line-by-line - many lines so that changing + # one of them is a small fraction of the whole, not "the whole line + # differs" (which is what a single long line would produce instead). + lines = [f"line {i} says something routine about the company" for i in range(50)] + previous_text = "\n".join(lines) + await _make_snapshot( + db_session, + company, + source, + content_hash="h1", + urls=["/about"], + text_summary=previous_text, + created_at=datetime.now(UTC) - timedelta(hours=1), + ) + # Change just one line out of 50 - below the content-modified threshold. + lines[25] = "line 25 says something slightly different about the company" + current = await _make_snapshot( + db_session, + company, + source, + content_hash="h2", + urls=["/about"], + text_summary="\n".join(lines), + ) + + change = await detect_change_for_source(db_session, source, company, current, uuid.uuid4()) + assert change is None + + +@pytest.mark.asyncio +async def test_focus_match_is_recorded_via_higher_significance(db_session): + matching_company = await _make_company( + db_session, monitoring_focus="Watch for battery manufacturing expansion" + ) + other_company = await _make_company(db_session, monitoring_focus="Watch for pricing changes") + + matching_source = await _make_source(db_session, matching_company) + other_source = await _make_source(db_session, other_company) + + old_text = "### Careers\nExisting listing." + new_text = ( + "### Careers\nExisting listing.\n\n### New role\nBattery manufacturing engineer wanted." + ) + + for company, source in ((matching_company, matching_source), (other_company, other_source)): + await _make_snapshot( + db_session, + company, + source, + content_hash="h1", + urls=["/careers/a"], + text_summary=old_text, + created_at=datetime.now(UTC) - timedelta(hours=1), + ) + + matching_current = await _make_snapshot( + db_session, + matching_company, + matching_source, + content_hash="h2", + urls=["/careers/a", "/careers/b"], + text_summary=new_text, + ) + other_current = await _make_snapshot( + db_session, + other_company, + other_source, + content_hash="h2", + urls=["/careers/a", "/careers/b"], + text_summary=new_text, + ) + + matched_change = await detect_change_for_source( + db_session, matching_source, matching_company, matching_current, uuid.uuid4() + ) + unmatched_change = await detect_change_for_source( + db_session, other_source, other_company, other_current, uuid.uuid4() + ) + + assert matched_change.significance_score > unmatched_change.significance_score + + +@pytest.mark.asyncio +async def test_exact_repeat_within_cooldown_is_suppressed(db_session): + company = await _make_company(db_session) + source = await _make_source(db_session, company) + + await _make_snapshot( + db_session, + company, + source, + content_hash="h1", + urls=["/a"], + text_summary="A", + created_at=datetime.now(UTC) - timedelta(hours=2), + ) + snap2 = await _make_snapshot( + db_session, + company, + source, + content_hash="h2", + urls=["/a", "/b"], + text_summary="A\nB", + created_at=datetime.now(UTC) - timedelta(hours=1), + ) + first_change = await detect_change_for_source(db_session, source, company, snap2, uuid.uuid4()) + assert first_change is not None + + # A third snapshot reverts to hash h1's item set momentarily, then a + # fourth snapshot reproduces the *exact same* added-item diff as before - + # this should be suppressed as a repeat within the cooldown window. + await _make_snapshot( + db_session, + company, + source, + content_hash="h1-again", + urls=["/a"], + text_summary="A", + ) + snap4 = await _make_snapshot( + db_session, + company, + source, + content_hash="h2-again", + urls=["/a", "/b"], + text_summary="A\nB", + ) + second_change = await detect_change_for_source(db_session, source, company, snap4, uuid.uuid4()) + + assert second_change is None + all_changes = ( + ( + await db_session.execute( + select(DetectedChange).where(DetectedChange.source_id == source.id) + ) + ) + .scalars() + .all() + ) + assert len(all_changes) == 1 diff --git a/apps/api/tests/integration/test_collection_service.py b/apps/api/tests/integration/test_collection_service.py new file mode 100644 index 0000000..5d1aab2 --- /dev/null +++ b/apps/api/tests/integration/test_collection_service.py @@ -0,0 +1,221 @@ +"""Integration tests: collection_service persisting collector output through +real repositories into a real (SQLite) database. No live network - respx +mocks every HTTP call and DNS resolution is patched to a fixed public IP.""" + +from __future__ import annotations + +import json +import uuid +from unittest.mock import patch + +import httpx +import pytest +import respx +from sqlalchemy import select +from sqlalchemy.orm import selectinload + +from app.models.company import Company +from app.models.enums import SourceStatus, SourceType +from app.models.monitor_configuration import MonitorConfiguration +from app.models.snapshot import Snapshot +from app.models.source import Source +from app.models.source_document import SourceDocument +from app.repositories.source_repository import SourceRepository +from app.services import collection_service + + +@pytest.fixture(autouse=True) +def _no_real_dns(): + with patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]): + yield + + +async def _make_company( + db_session, *, official_website: str | None = "https://example.com" +) -> Company: + company = Company( + id=uuid.uuid4(), + user_id=uuid.uuid4(), + name="Acme Corp", + slug=f"acme-corp-{uuid.uuid4().hex[:6]}", + official_website=official_website, + ) + db_session.add(company) + db_session.add(MonitorConfiguration(company_id=company.id)) + await db_session.commit() + result = await db_session.execute( + select(Company) + .where(Company.id == company.id) + .options( + selectinload(Company.aliases), + selectinload(Company.competitors), + selectinload(Company.enrichment), + ) + ) + return result.scalar_one() + + +@pytest.mark.asyncio +async def test_discover_sources_creates_website_and_job_posting_sources(db_session): + company = await _make_company(db_session) + + with respx.mock: + respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404)) + respx.get("https://example.com/sitemap.xml").mock(return_value=httpx.Response(404)) + respx.get("https://api.github.com/search/users").mock( + return_value=httpx.Response(200, text=json.dumps({"items": []})) + ) + respx.get("https://www.sec.gov/cgi-bin/browse-edgar").mock( + return_value=httpx.Response( + 200, text='' + ) + ) + created = await collection_service.discover_sources_for_company(db_session, company) + + types = {s.source_type for s in created} + assert SourceType.WEBSITE in types + assert SourceType.JOB_POSTING in types + assert SourceType.GITHUB not in types # no org match -> not created + assert SourceType.SEC_EDGAR not in types # no CIK match -> not created + + +@pytest.mark.asyncio +async def test_discover_sources_is_idempotent(db_session): + company = await _make_company(db_session) + + with respx.mock: + respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404)) + respx.get("https://example.com/sitemap.xml").mock(return_value=httpx.Response(404)) + respx.get("https://api.github.com/search/users").mock( + return_value=httpx.Response(200, text=json.dumps({"items": []})) + ) + respx.get("https://www.sec.gov/cgi-bin/browse-edgar").mock( + return_value=httpx.Response( + 200, text='' + ) + ) + first_batch = await collection_service.discover_sources_for_company(db_session, company) + second_batch = await collection_service.discover_sources_for_company(db_session, company) + + assert len(first_batch) > 0 + assert second_batch == [] # already-discovered sources aren't recreated + + repo = SourceRepository(db_session) + all_sources = await repo.list_for_company(company.id) + assert len(all_sources) == len(first_batch) + + +@pytest.mark.asyncio +async def test_collect_source_persists_documents_and_snapshot(db_session, settings): + company = await _make_company(db_session) + repo = SourceRepository(db_session) + source = await repo.create( + company_id=company.id, + source_type=SourceType.CUSTOM_URL, + name="Pricing page", + base_url="https://example.com/pricing", + ) + await db_session.commit() + + with respx.mock: + respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404)) + respx.get("https://example.com/pricing").mock( + return_value=httpx.Response( + 200, + html=( + "Pricing" + "

Pricing

Base plan is $49 per month.

" + "" + ), + ) + ) + result = await collection_service.collect_source(db_session, settings, source, company) + + assert result.status == SourceStatus.ACTIVE + + docs = ( + ( + await db_session.execute( + select(SourceDocument).where(SourceDocument.source_id == source.id) + ) + ) + .scalars() + .all() + ) + assert len(docs) == 1 + assert "$49 per month" in docs[0].content_text + + snapshots = ( + (await db_session.execute(select(Snapshot).where(Snapshot.source_id == source.id))) + .scalars() + .all() + ) + assert len(snapshots) == 1 + + refreshed_source = ( + await db_session.execute(select(Source).where(Source.id == source.id)) + ).scalar_one() + assert refreshed_source.status == SourceStatus.ACTIVE + assert refreshed_source.last_successful_check is not None + assert refreshed_source.failure_count == 0 + + +@pytest.mark.asyncio +async def test_collect_source_deduplicates_unchanged_content_across_runs(db_session, settings): + company = await _make_company(db_session) + repo = SourceRepository(db_session) + source = await repo.create( + company_id=company.id, + source_type=SourceType.CUSTOM_URL, + name="About page", + base_url="https://example.com/about", + ) + await db_session.commit() + + html = ( + "About" + "

About

We build electric trucks.

" + ) + + for _ in range(2): + with respx.mock: + respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404)) + respx.get("https://example.com/about").mock(return_value=httpx.Response(200, html=html)) + await collection_service.collect_source(db_session, settings, source, company) + + docs = ( + ( + await db_session.execute( + select(SourceDocument).where(SourceDocument.source_id == source.id) + ) + ) + .scalars() + .all() + ) + assert len(docs) == 1 # second run's identical content was deduplicated + + +@pytest.mark.asyncio +async def test_collect_source_marks_failure_and_increments_count(db_session, settings): + company = await _make_company(db_session) + repo = SourceRepository(db_session) + source = await repo.create( + company_id=company.id, + source_type=SourceType.CUSTOM_URL, + name="Broken page", + base_url="https://example.com/broken", + ) + await db_session.commit() + + with respx.mock: + respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404)) + respx.get("https://example.com/broken").mock(return_value=httpx.Response(500)) + result = await collection_service.collect_source(db_session, settings, source, company) + + assert result.status == SourceStatus.FAILED + + refreshed_source = ( + await db_session.execute(select(Source).where(Source.id == source.id)) + ).scalar_one() + assert refreshed_source.failure_count == 1 + assert refreshed_source.last_successful_check is None diff --git a/apps/api/tests/integration/test_discovery_service.py b/apps/api/tests/integration/test_discovery_service.py new file mode 100644 index 0000000..435b179 --- /dev/null +++ b/apps/api/tests/integration/test_discovery_service.py @@ -0,0 +1,189 @@ +"""Company discovery, end-to-end against the mock providers: name in, +DiscoveredCompanyProfile out, real evidence flow through search -> fetch -> +LLM extraction, with user hints always winning over discovered values. No +live network - respx mocks every HTTP call, DNS resolution is patched to a +fixed public IP (matching the pattern in test_collection_service.py).""" + +from __future__ import annotations + +import json +from unittest.mock import patch + +import httpx +import pytest +import respx + +from app.analysis.llm.mock import MockLLMProvider +from app.models.enums import SourceType +from app.search.base import SearchResult +from app.search.mock import MockSearchProvider +from app.services.discovery_service import _resolve_official_website, discover_company_profile + + +@pytest.fixture(autouse=True) +def _no_real_dns(): + with patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]): + yield + + +def _mock_empty_github_sec(): + respx.get("https://api.github.com/search/users").mock( + return_value=httpx.Response(200, text=json.dumps({"items": []})) + ) + respx.get("https://www.sec.gov/cgi-bin/browse-edgar").mock( + return_value=httpx.Response( + 200, text='' + ) + ) + + +@pytest.mark.asyncio +async def test_discover_resolves_website_and_extracts_profile_from_real_evidence(settings): + search = MockSearchProvider() + llm = MockLLMProvider() + + with respx.mock: + _mock_empty_github_sec() + respx.get("https://acmemobility.com/robots.txt").mock(return_value=httpx.Response(404)) + respx.get("https://acmemobility.com").mock( + return_value=httpx.Response( + 200, + html=( + "Acme Mobility" + "

About

Acme Mobility is headquartered in Austin, Texas. " + "Formerly known as Acme Scooters.

" + ), + ) + ) + profile = await discover_company_profile( + search, + llm, + settings, + name="Acme Mobility", + official_website=None, + monitoring_focus="pricing changes", + competitor_names=[], + alias_names=[], + ) + + assert profile.official_website == "https://acmemobility.com" + assert profile.headquarters == "Austin, Texas" + assert profile.aliases == ["Acme Scooters"] + assert profile.monitoring_focus == "pricing changes" + assert "https://acmemobility.com" in profile.sources_consulted + assert any(p.source_type == SourceType.WEBSITE for p in profile.potential_sources) + + +@pytest.mark.asyncio +async def test_discover_prefers_user_hints_over_discovered_values(settings): + search = MockSearchProvider() + llm = MockLLMProvider() + + with respx.mock: + _mock_empty_github_sec() + respx.get("https://acme.example/robots.txt").mock(return_value=httpx.Response(404)) + respx.get("https://acme.example").mock( + return_value=httpx.Response( + 200, html="

Acme, based in Denver, Colorado.

" + ) + ) + profile = await discover_company_profile( + search, + llm, + settings, + name="Acme", + official_website="https://acme.example", + monitoring_focus=None, + competitor_names=["Rival Corp"], + alias_names=["Acme Inc"], + ) + + # Hint website used as-is (no "official website" search performed for it) + assert profile.official_website == "https://acme.example" + assert profile.competitors == ["Rival Corp"] + assert profile.aliases == ["Acme Inc"] + # Still extracted from the real fetched page since that hint wasn't given + assert profile.headquarters == "Denver, Colorado" + + +@pytest.mark.asyncio +async def test_discover_handles_a_website_that_fails_to_resolve_gracefully(settings): + search = MockSearchProvider() + llm = MockLLMProvider() + + with respx.mock: + _mock_empty_github_sec() + respx.get("https://nowhereco.com/robots.txt").mock(return_value=httpx.Response(404)) + respx.get("https://nowhereco.com").mock(return_value=httpx.Response(500)) + + profile = await discover_company_profile( + search, + llm, + settings, + name="Nowhere Co", + official_website=None, + monitoring_focus=None, + competitor_names=[], + alias_names=[], + ) + + assert profile.official_website == "https://nowhereco.com" + assert profile.headquarters is None + assert profile.aliases == [] + + +class _StubSearchProvider: + """Returns a fixed result list regardless of query - lets a test control + exactly what "official website" search ranking looks like, independent + of MockSearchProvider's domain-guessing heuristic.""" + + provider_name = "stub" + + def __init__(self, results: list[SearchResult]) -> None: + self._results = results + + async def search(self, query: str, *, count: int = 5) -> list[SearchResult]: + return self._results[:count] + + +@pytest.mark.asyncio +async def test_resolve_official_website_skips_a_top_ranked_wikipedia_result(): + # Observed live against the real Brave API: "Stripe official website" + # ranked Stripe's Wikipedia article above stripe.com itself. + search = _StubSearchProvider( + [ + SearchResult( + title="Stripe, Inc. - Wikipedia", + url="https://en.wikipedia.org/wiki/Stripe,_Inc.", + snippet="Stripe, Inc. is an American financial services company.", + ), + SearchResult( + title="Stripe | Financial Infrastructure", + url="https://stripe.com", + snippet="Stripe powers online and in-person payment processing.", + ), + ] + ) + + url, consulted = await _resolve_official_website(search, "Stripe", None) + + assert url == "https://stripe.com" + assert consulted == ["https://stripe.com"] + + +@pytest.mark.asyncio +async def test_resolve_official_website_falls_back_to_top_result_when_all_are_reference_sites(): + search = _StubSearchProvider( + [ + SearchResult( + title="Acme - Wikipedia", + url="https://en.wikipedia.org/wiki/Acme", + snippet="An encyclopedia article.", + ), + ] + ) + + url, consulted = await _resolve_official_website(search, "Acme", None) + + assert url == "https://en.wikipedia.org/wiki/Acme" + assert consulted == ["https://en.wikipedia.org/wiki/Acme"] diff --git a/apps/api/tests/integration/test_enrichment_task.py b/apps/api/tests/integration/test_enrichment_task.py new file mode 100644 index 0000000..9a0ed21 --- /dev/null +++ b/apps/api/tests/integration/test_enrichment_task.py @@ -0,0 +1,105 @@ +"""app.tasks.enrichment.enrich_company: the Celery task end-to-end +(mocked HTTP, real DB) - confirms the CompanyEnrichment row persists with +the expected status after the task runs, and that a missing company is a +clean no-op rather than a crash.""" + +from __future__ import annotations + +import uuid + +import httpx +import pytest +import respx + +from app.core.config import Settings +from app.models.company import Company +from app.repositories.company_enrichment_repository import CompanyEnrichmentRepository +from app.tasks.enrichment import _enrich_company_async + + +async def _make_company(db_session) -> Company: + company = Company( + id=uuid.uuid4(), + user_id=uuid.uuid4(), + name="Acme Corp", + slug=f"acme-{uuid.uuid4().hex[:6]}", + official_website="https://acme.example.com", + ) + db_session.add(company) + await db_session.commit() + return company + + +def _mock_empty_endpoints() -> None: + for path in ( + "company/details", + "company/funding", + "company/updates", + "competitor/listing", + "product/listing", + "customer/listing", + ): + respx.get(f"https://nubela.co/api/v1/{path}").mock( + return_value=httpx.Response(200, json={}) + ) + + +@pytest.mark.asyncio +async def test_enrich_company_task_persists_a_complete_result(db_session, monkeypatch): + monkeypatch.setattr( + "app.tasks.enrichment.get_settings", lambda: Settings(ninjapear_api_key="test-key") + ) + company = await _make_company(db_session) + + with respx.mock: + _mock_empty_endpoints() + await _enrich_company_async(str(company.id)) + + enrichment = await CompanyEnrichmentRepository(db_session).get_for_company(company.id) + assert enrichment is not None + assert enrichment.status.value == "complete" + assert enrichment.fetched_at is not None + + +@pytest.mark.asyncio +async def test_enrich_company_task_records_partial_status_on_a_failed_section( + db_session, monkeypatch +): + monkeypatch.setattr( + "app.tasks.enrichment.get_settings", lambda: Settings(ninjapear_api_key="test-key") + ) + company = await _make_company(db_session) + + with respx.mock: + respx.get("https://nubela.co/api/v1/company/details").mock( + return_value=httpx.Response(200, json={}) + ) + respx.get("https://nubela.co/api/v1/company/funding").mock( + return_value=httpx.Response(500, text="boom") + ) + respx.get("https://nubela.co/api/v1/company/updates").mock( + return_value=httpx.Response(200, json={}) + ) + respx.get("https://nubela.co/api/v1/competitor/listing").mock( + return_value=httpx.Response(200, json={}) + ) + respx.get("https://nubela.co/api/v1/product/listing").mock( + return_value=httpx.Response(200, json={}) + ) + respx.get("https://nubela.co/api/v1/customer/listing").mock( + return_value=httpx.Response(200, json={}) + ) + await _enrich_company_async(str(company.id)) + + enrichment = await CompanyEnrichmentRepository(db_session).get_for_company(company.id) + assert enrichment.status.value == "partial" + assert "funding" in enrichment.errors + + +@pytest.mark.asyncio +async def test_enrich_company_task_is_a_no_op_for_a_missing_company(db_session, monkeypatch): + monkeypatch.setattr( + "app.tasks.enrichment.get_settings", lambda: Settings(ninjapear_api_key="test-key") + ) + # Should return cleanly rather than raising - no company, nothing to do. + await _enrich_company_async(str(uuid.uuid4())) diff --git a/apps/api/tests/integration/test_maintenance_task.py b/apps/api/tests/integration/test_maintenance_task.py new file mode 100644 index 0000000..d56db63 --- /dev/null +++ b/apps/api/tests/integration/test_maintenance_task.py @@ -0,0 +1,76 @@ +"""Data-retention purge task: only SourceDocument rows older than +DATA_RETENTION_DAYS are deleted, everything newer is untouched.""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime, timedelta + +import pytest +from sqlalchemy import select + +from app.models.company import Company +from app.models.enums import SourceType +from app.models.source import Source +from app.models.source_document import SourceDocument +from app.repositories.source_repository import SourceDocumentRepository +from app.tasks.maintenance import _purge_expired_data_async + + +async def _make_document( + db_session, company, source, *, retrieved_date: datetime +) -> SourceDocument: + return await SourceDocumentRepository(db_session).create( + source_id=source.id, + company_id=company.id, + url=f"https://acme.example/{uuid.uuid4().hex[:8]}", + canonical_url=f"https://acme.example/{uuid.uuid4().hex[:8]}", + title="Doc", + author=None, + publication_date=None, + retrieved_date=retrieved_date, + content_text="Some content", + content_hash=uuid.uuid4().hex, + metadata_json={}, + extraction_method="test", + trust_score=0.7, + ) + + +@pytest.mark.asyncio +async def test_purge_deletes_only_documents_older_than_retention_window(db_session, settings): + company = Company( + id=uuid.uuid4(), + user_id=uuid.uuid4(), + name="Retention Co", + slug=f"retention-co-{uuid.uuid4().hex[:6]}", + ) + db_session.add(company) + await db_session.flush() + + source = Source( + company_id=company.id, + source_type=SourceType.WEBSITE, + name="Site", + base_url="https://acme.example", + ) + db_session.add(source) + await db_session.flush() + + now = datetime.now(UTC) + old_doc = await _make_document( + db_session, + company, + source, + retrieved_date=now - timedelta(days=settings.data_retention_days + 30), + ) + recent_doc = await _make_document( + db_session, company, source, retrieved_date=now - timedelta(days=1) + ) + await db_session.commit() + + await _purge_expired_data_async() + + remaining_ids = set((await db_session.execute(select(SourceDocument.id))).scalars().all()) + assert old_doc.id not in remaining_ids + assert recent_doc.id in remaining_ids diff --git a/apps/api/tests/integration/test_per_source_scheduling.py b/apps/api/tests/integration/test_per_source_scheduling.py new file mode 100644 index 0000000..b7553aa --- /dev/null +++ b/apps/api/tests/integration/test_per_source_scheduling.py @@ -0,0 +1,126 @@ +"""run_monitoring's per-source due-ness filter: a SCHEDULED run only +collects sources that are actually due (an overdue fast-cadence source +alongside a not-yet-due default-cadence one), while a MANUAL "Run now" +always collects every active source regardless of individual cadence.""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime, timedelta +from unittest.mock import patch + +import httpx +import pytest +import respx + +from app.models.company import Company +from app.models.enums import MonitoringFrequency, MonitoringRunTrigger, SourceType +from app.models.monitor_configuration import MonitorConfiguration +from app.models.source import Source +from app.repositories.monitoring_run_repository import MonitoringRunRepository +from app.tasks.scheduler import _sync_schedules_async + +RSS_FEED = ( + 'News' + "Updatehttps://example.com/n" + "Something happened." +) + + +def _register_and_login(client) -> dict[str, str]: + email = f"user-{uuid.uuid4().hex[:12]}@example.com" + client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "Test User"}, + ) + tokens = client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ).json() + return {"Authorization": f"Bearer {tokens['access_token']}"} + + +async def _make_company_with_two_sources(db_session): + company = Company( + id=uuid.uuid4(), + user_id=uuid.uuid4(), + name="Scheduling Co", + slug=f"scheduling-co-{uuid.uuid4().hex[:6]}", + ) + db_session.add(company) + db_session.add( + MonitorConfiguration( + company_id=company.id, + frequency_type=MonitoringFrequency.WEEKLY, + enabled=True, + next_run=datetime.now(UTC) + timedelta(hours=1), # not due + ) + ) + due_source = Source( + company_id=company.id, + source_type=SourceType.RSS, + name="Daily News", + base_url="https://example.com/feed", + active=True, + frequency_type=MonitoringFrequency.DAILY, + next_check=datetime.now(UTC) - timedelta(minutes=5), # overdue + ) + not_due_source = Source( + company_id=company.id, + source_type=SourceType.RSS, + name="Default Cadence Feed", + base_url="https://example.com/other-feed", + active=True, + # frequency_type left None - inherits the company's weekly default, + # which per MonitorConfiguration.next_run above isn't due yet. + ) + db_session.add_all([due_source, not_due_source]) + await db_session.commit() + return company + + +@pytest.mark.asyncio +async def test_scheduled_run_only_collects_the_due_source(db_session): + company = await _make_company_with_two_sources(db_session) + + with respx.mock: + respx.get("https://example.com/feed").mock(return_value=httpx.Response(200, text=RSS_FEED)) + await _sync_schedules_async() + + runs = await MonitoringRunRepository(db_session).list_for_company(company.id) + assert len(runs) == 1 + run = runs[0] + assert run.trigger_type == MonitoringRunTrigger.SCHEDULED + assert run.sources_attempted == 1 + assert run.sources_successful == 1 + + +def test_manual_run_collects_every_active_source_regardless_of_cadence(client): + headers = _register_and_login(client) + company = client.post( + "/api/v1/companies", + json={"name": f"Manual Sched Co {uuid.uuid4().hex[:6]}", "frequency_type": "weekly"}, + headers=headers, + ).json() + + for name, url in [("A", "https://example.com/a"), ("B", "https://example.com/b")]: + client.post( + f"/api/v1/companies/{company['id']}/sources", + json={"source_type": "custom_url", "name": name, "base_url": url}, + headers=headers, + ) + + with patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]): + with respx.mock: + respx.get("https://example.com/a/robots.txt").mock(return_value=httpx.Response(404)) + respx.get("https://example.com/a").mock( + return_value=httpx.Response(200, html="A") + ) + respx.get("https://example.com/b/robots.txt").mock(return_value=httpx.Response(404)) + respx.get("https://example.com/b").mock( + return_value=httpx.Response(200, html="B") + ) + run = client.post(f"/api/v1/companies/{company['id']}/run", headers=headers).json() + + detail = client.get(f"/api/v1/runs/{run['id']}", headers=headers).json() + assert detail["sources_attempted"] == 2 + assert detail["sources_successful"] == 2 diff --git a/apps/api/tests/integration/test_report_service.py b/apps/api/tests/integration/test_report_service.py new file mode 100644 index 0000000..3d7dbbc --- /dev/null +++ b/apps/api/tests/integration/test_report_service.py @@ -0,0 +1,153 @@ +"""Report generation end-to-end against a real (SQLite) DB: real +SourceDocument/DetectedChange rows in, a persisted Report (JSON + Markdown) +out, via the mock LLM provider.""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime + +import pytest +from sqlalchemy import select +from sqlalchemy.orm import selectinload + +from app.analysis.llm.mock import MockLLMProvider +from app.models.company import Company +from app.models.detected_change import DetectedChange +from app.models.enums import ChangeType, MonitoringRunTrigger, ReportType, SeverityLevel, SourceType +from app.models.monitoring_run import MonitoringRun +from app.models.report import Report +from app.models.snapshot import Snapshot +from app.models.source import Source +from app.models.source_document import SourceDocument +from app.repositories.company_repository import CompanyRepository +from app.services.report_service import generate_and_persist_report + + +async def _make_company_with_evidence(db_session) -> Company: + company = Company( + id=uuid.uuid4(), + user_id=uuid.uuid4(), + name="Acme Mobility Systems", + slug=f"acme-mobility-{uuid.uuid4().hex[:6]}", + monitoring_focus="EV manufacturing expansion and battery technology", + ) + db_session.add(company) + await db_session.flush() + + source = Source( + company_id=company.id, + source_type=SourceType.JOB_POSTING, + name="Careers", + base_url="https://acme.example/careers", + ) + db_session.add(source) + await db_session.flush() + + db_session.add( + SourceDocument( + source_id=source.id, + company_id=company.id, + url="https://acme.example/careers/battery-engineer", + canonical_url="https://acme.example/careers/battery-engineer", + title="Senior Battery Engineer", + author=None, + publication_date=None, + retrieved_date=datetime.now(UTC), + content_text="We are hiring a senior battery engineer to lead our new EV platform.", + content_hash="hash1", + metadata_json={}, + extraction_method="job_link_heuristic", + trust_score=0.7, + ) + ) + + run = MonitoringRun(company_id=company.id, trigger_type=MonitoringRunTrigger.MANUAL) + db_session.add(run) + await db_session.flush() + + snapshot = Snapshot( + company_id=company.id, + source_id=source.id, + snapshot_type=source.source_type.value, + hash="hash1", + structured_summary={"urls": ["/careers/battery-engineer"]}, + text_summary="Senior Battery Engineer", + monitoring_run_id=run.id, + ) + db_session.add(snapshot) + await db_session.flush() + + db_session.add( + DetectedChange( + company_id=company.id, + source_id=source.id, + monitoring_run_id=run.id, + current_snapshot_id=snapshot.id, + change_type=ChangeType.NEW_DOCUMENT, + raw_diff={"structured_added": ["/careers/battery-engineer"], "structured_removed": []}, + significance_score=0.6, + confidence_score=0.75, + severity=SeverityLevel.HIGH, + summary="1 new item detected", + ) + ) + await db_session.commit() + + return await CompanyRepository(db_session).get_for_user(company.id, company.user_id) + + +@pytest.mark.asyncio +async def test_generate_report_persists_structured_and_markdown(db_session, settings): + company = await _make_company_with_evidence(db_session) + llm = MockLLMProvider() + + report = await generate_and_persist_report( + db_session, settings, llm, company, report_type=ReportType.BASELINE + ) + + assert report.id is not None + assert report.report_type == ReportType.BASELINE + assert report.model_provider == "mock" + assert "Acme Mobility Systems" in report.executive_summary + assert report.structured_report["executive_summary"] == report.executive_summary + assert len(report.structured_report["recent_developments"]) == 1 + assert len(report.structured_report["hiring_signals"]) == 1 + + assert "# Competitive Intelligence Report: Acme Mobility Systems" in report.markdown_content + assert "## 15. Sources" in report.markdown_content + assert "battery-engineer" in report.markdown_content + + persisted = ( + await db_session.execute(select(Report).where(Report.id == report.id)) + ).scalar_one() + assert persisted.company_id == company.id + + +@pytest.mark.asyncio +async def test_generate_report_with_no_evidence_is_still_honest(db_session, settings): + company = Company( + id=uuid.uuid4(), + user_id=uuid.uuid4(), + name="Empty Co", + slug=f"empty-co-{uuid.uuid4().hex[:6]}", + ) + db_session.add(company) + await db_session.commit() + result = await db_session.execute( + select(Company) + .where(Company.id == company.id) + .options( + selectinload(Company.aliases), + selectinload(Company.competitors), + selectinload(Company.enrichment), + ) + ) + company = result.scalar_one() + + report = await generate_and_persist_report( + db_session, settings, MockLLMProvider(), company, report_type=ReportType.BASELINE + ) + + assert "0 collected document" in report.executive_summary + assert report.structured_report["recent_developments"] == [] diff --git a/apps/api/tests/integration/test_scheduler_task.py b/apps/api/tests/integration/test_scheduler_task.py new file mode 100644 index 0000000..6954209 --- /dev/null +++ b/apps/api/tests/integration/test_scheduler_task.py @@ -0,0 +1,138 @@ +"""Celery Beat's sync_schedules task: dynamic due-schedule discovery, +idempotent skip of companies already mid-run, and delegation to +run_monitoring.delay for everything else.""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime, timedelta +from unittest.mock import patch + +import pytest +from sqlalchemy import select + +from app.models.company import Company +from app.models.enums import MonitoringFrequency, MonitoringRunTrigger, SourceType +from app.models.monitor_configuration import MonitorConfiguration +from app.models.monitoring_run import MonitoringRun +from app.models.source import Source +from app.repositories.monitoring_run_repository import MonitoringRunRepository +from app.tasks.scheduler import _sync_schedules_async + + +async def _make_due_company(db_session, *, next_run_offset_minutes: int, enabled: bool = True): + company = Company( + id=uuid.uuid4(), + user_id=uuid.uuid4(), + name="Scheduled Co", + slug=f"scheduled-co-{uuid.uuid4().hex[:6]}", + ) + db_session.add(company) + db_session.add( + MonitorConfiguration( + company_id=company.id, + frequency_type=MonitoringFrequency.WEEKLY, + enabled=enabled, + next_run=datetime.now(UTC) + timedelta(minutes=next_run_offset_minutes), + ) + ) + await db_session.commit() + return company + + +@pytest.mark.asyncio +async def test_sync_schedules_enqueues_due_companies(db_session): + company = await _make_due_company(db_session, next_run_offset_minutes=-5) # 5 min overdue + + with patch("app.tasks.collection.run_monitoring.delay") as mock_delay: + await _sync_schedules_async() + + mock_delay.assert_called_once() + + result = await db_session.execute( + select(MonitoringRun).where(MonitoringRun.company_id == company.id) + ) + run = result.scalar_one() + assert run.trigger_type == MonitoringRunTrigger.SCHEDULED + + +@pytest.mark.asyncio +async def test_sync_schedules_skips_not_yet_due_companies(db_session): + await _make_due_company(db_session, next_run_offset_minutes=60) # due in the future + + with patch("app.tasks.collection.run_monitoring.delay") as mock_delay: + await _sync_schedules_async() + + mock_delay.assert_not_called() + + +@pytest.mark.asyncio +async def test_sync_schedules_skips_disabled_configs(db_session): + await _make_due_company(db_session, next_run_offset_minutes=-5, enabled=False) + + with patch("app.tasks.collection.run_monitoring.delay") as mock_delay: + await _sync_schedules_async() + + mock_delay.assert_not_called() + + +@pytest.mark.asyncio +async def test_sync_schedules_does_not_double_enqueue_a_company_with_an_active_run(db_session): + company = await _make_due_company(db_session, next_run_offset_minutes=-5) + + run_repo = MonitoringRunRepository(db_session) + await run_repo.create(company_id=company.id, trigger_type=MonitoringRunTrigger.MANUAL) + await db_session.commit() + + with patch("app.tasks.collection.run_monitoring.delay") as mock_delay: + await _sync_schedules_async() + + mock_delay.assert_not_called() + + +@pytest.mark.asyncio +async def test_sync_schedules_enqueues_via_a_source_override_even_when_company_default_isnt_due( + db_session, +): + """A source with its own faster cadence (e.g. "check news daily") must + be able to trigger a run even while the company's own default schedule + (e.g. weekly) isn't due yet - this is the whole point of per-source + scheduling.""" + company = await _make_due_company(db_session, next_run_offset_minutes=60) # not due + db_session.add( + Source( + company_id=company.id, + source_type=SourceType.RSS, + name="Daily News", + active=True, + frequency_type=MonitoringFrequency.DAILY, + next_check=datetime.now(UTC) - timedelta(minutes=5), # overdue + ) + ) + await db_session.commit() + + with patch("app.tasks.collection.run_monitoring.delay") as mock_delay: + await _sync_schedules_async() + + mock_delay.assert_called_once() + + +@pytest.mark.asyncio +async def test_sync_schedules_skips_when_neither_company_default_nor_any_source_is_due(db_session): + company = await _make_due_company(db_session, next_run_offset_minutes=60) # not due + db_session.add( + Source( + company_id=company.id, + source_type=SourceType.RSS, + name="Daily News", + active=True, + frequency_type=MonitoringFrequency.DAILY, + next_check=datetime.now(UTC) + timedelta(hours=12), # not due yet + ) + ) + await db_session.commit() + + with patch("app.tasks.collection.run_monitoring.delay") as mock_delay: + await _sync_schedules_async() + + mock_delay.assert_not_called() diff --git a/apps/api/tests/unit/__init__.py b/apps/api/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/tests/unit/test_alerting_e2e.py b/apps/api/tests/unit/test_alerting_e2e.py new file mode 100644 index 0000000..ecdeafe --- /dev/null +++ b/apps/api/tests/unit/test_alerting_e2e.py @@ -0,0 +1,173 @@ +"""End-to-end proof that a real monitoring run wires all the way through to +a delivered notification: baseline run -> content changes -> second run's +DetectedChange crosses the company's (LOW) severity threshold -> Alert +created -> dispatched to the registered email destination -> smtplib +(mocked) actually gets called. Everything else is the real HTTP API + +Celery-eager pipeline, exactly as a user would trigger it from the +dashboard; only DNS and the outbound SMTP socket are mocked.""" + +from __future__ import annotations + +import json +import uuid +from unittest.mock import patch + +import httpx +import respx + + +def _register_and_login(client) -> dict[str, str]: + email = f"user-{uuid.uuid4().hex[:12]}@example.com" + client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "Test User"}, + ) + tokens = client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ).json() + return {"Authorization": f"Bearer {tokens['access_token']}"} + + +def _mock_empty_discovery(): + respx.get("https://api.github.com/search/users").mock( + return_value=httpx.Response(200, text=json.dumps({"items": []})) + ) + respx.get("https://www.sec.gov/cgi-bin/browse-edgar").mock( + return_value=httpx.Response( + 200, text='' + ) + ) + # RSS (Google News) and GOV_CONTRACT (USASpending) are always discovered + # unconditionally - see discovery_service.py's _PREVIEWABLE_TYPES - so + # their collect() calls need mocking here too. + respx.get(url__regex=r"https://news\.google\.com/rss/search.*").mock( + return_value=httpx.Response( + 200, + text=( + 'Google News' + "Testhttps://example.com/news" + "Test news item" + ), + ) + ) + respx.post("https://api.usaspending.gov/api/v2/search/spending_by_award/").mock( + return_value=httpx.Response(200, json={"results": []}) + ) + + +def test_monitoring_run_detecting_a_change_produces_alert_and_email(client, monkeypatch): + sent_emails = [] + + class FakeSmtp: + def __init__(self, host, port, timeout=10): + pass + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def starttls(self): + pass + + def login(self, username, password): + pass + + def sendmail(self, from_addr, to_addrs, message): + sent_emails.append({"to": to_addrs, "message": message}) + + monkeypatch.setattr("app.notifications.smtp_email.smtplib.SMTP", FakeSmtp) + + headers = _register_and_login(client) + + company = client.post( + "/api/v1/companies", + json={ + "name": f"Acme Mobility {uuid.uuid4().hex[:6]}", + "frequency_type": "weekly", + "severity_threshold": "low", + }, + headers=headers, + ).json() + + client.post( + f"/api/v1/companies/{company['id']}/sources", + json={ + "source_type": "custom_url", + "name": "Pricing", + "base_url": "https://example.com/pricing", + }, + headers=headers, + ) + + client.post( + "/api/v1/notification-destinations", + json={ + "type": "email", + "destination_value": "watcher@example.com", + "minimum_severity": "low", + "company_ids": [company["id"]], + }, + headers=headers, + ) + + baseline_html = ( + "Pricing" + "

Pricing

The base plan is $49/month.

" + "" + ) + changed_html = ( + "Pricing" + "

Pricing

The base plan is $99/month.

" + "" + ) + + with patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]): + with respx.mock: + _mock_empty_discovery() + respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404)) + respx.get("https://example.com/pricing").mock( + return_value=httpx.Response(200, html=baseline_html) + ) + baseline_run = client.post( + f"/api/v1/companies/{company['id']}/run", headers=headers + ).json() + + with respx.mock: + _mock_empty_discovery() + respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404)) + respx.get("https://example.com/pricing").mock( + return_value=httpx.Response(200, html=changed_html) + ) + second_run = client.post( + f"/api/v1/companies/{company['id']}/run", headers=headers + ).json() + + assert ( + client.get(f"/api/v1/runs/{baseline_run['id']}", headers=headers).json()["status"] + == "successful" + ) + assert ( + client.get(f"/api/v1/runs/{second_run['id']}", headers=headers).json()["status"] + == "successful" + ) + + alerts = client.get( + "/api/v1/alerts", headers=headers, params={"company_id": company["id"]} + ).json() + assert len(alerts) == 1 + alert = alerts[0] + assert alert["severity"] in {"low", "medium", "high", "critical"} + + detail = client.get(f"/api/v1/alerts/{alert['id']}", headers=headers).json() + assert len(detail["deliveries"]) == 1 + assert detail["deliveries"][0]["status"] == "sent" + + # Registration now also sends a verification-code email through this + # same SMTP path (no RESEND_API_KEY configured in tests, so + # security_email_service falls back to SmtpEmailProvider) - filter to + # the alert email specifically rather than assuming it's the only one. + alert_emails = [e for e in sent_emails if e["to"] == ["watcher@example.com"]] + assert len(alert_emails) == 1 + assert alert["title"] in alert_emails[0]["message"] diff --git a/apps/api/tests/unit/test_alerts_api.py b/apps/api/tests/unit/test_alerts_api.py new file mode 100644 index 0000000..0271f10 --- /dev/null +++ b/apps/api/tests/unit/test_alerts_api.py @@ -0,0 +1,200 @@ +"""Alerts API: ownership isolation, filters, and read/resolve mutations. +Alerts are only ever created internally by alert_service (never via a user +POST), so tests seed rows directly through db_session against the same +user_id the HTTP client is authenticated as (looked up via GET /auth/me).""" + +from __future__ import annotations + +import uuid + +import pytest + +from app.models.alert import Alert +from app.models.company import Company +from app.models.detected_change import DetectedChange +from app.models.enums import ChangeType, MonitoringRunTrigger, SeverityLevel, SourceType +from app.models.monitoring_run import MonitoringRun +from app.models.snapshot import Snapshot +from app.models.source import Source + + +def _register_and_login(client) -> tuple[dict[str, str], uuid.UUID]: + email = f"user-{uuid.uuid4().hex[:12]}@example.com" + client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "Test User"}, + ) + tokens = client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ).json() + headers = {"Authorization": f"Bearer {tokens['access_token']}"} + user_id = uuid.UUID(client.get("/api/v1/auth/me", headers=headers).json()["id"]) + return headers, user_id + + +async def _make_alert( + db_session, + user_id: uuid.UUID, + *, + severity: SeverityLevel = SeverityLevel.HIGH, + read: bool = False, + resolved: bool = False, +) -> Alert: + company = Company( + id=uuid.uuid4(), user_id=user_id, name="Acme Mobility", slug=f"acme-{uuid.uuid4().hex[:6]}" + ) + db_session.add(company) + await db_session.flush() + + source = Source( + company_id=company.id, + source_type=SourceType.WEBSITE, + name="Website", + base_url="https://acme.example", + ) + db_session.add(source) + await db_session.flush() + + run = MonitoringRun(company_id=company.id, trigger_type=MonitoringRunTrigger.MANUAL) + db_session.add(run) + await db_session.flush() + + snapshot = Snapshot( + company_id=company.id, + source_id=source.id, + snapshot_type=source.source_type.value, + hash="hash1", + structured_summary={}, + text_summary="", + monitoring_run_id=run.id, + ) + db_session.add(snapshot) + await db_session.flush() + + change = DetectedChange( + company_id=company.id, + source_id=source.id, + monitoring_run_id=run.id, + current_snapshot_id=snapshot.id, + change_type=ChangeType.NEW_DOCUMENT, + raw_diff={}, + significance_score=0.6, + confidence_score=0.75, + severity=severity, + summary="Change detected", + ) + db_session.add(change) + await db_session.flush() + + alert = Alert( + company_id=company.id, + detected_change_id=change.id, + user_id=user_id, + title="New hire announced", + summary="A new VP of Engineering was announced.", + why_it_matters="Signals a scaling push.", + severity=severity, + confidence=0.75, + read=read, + resolved=resolved, + ) + db_session.add(alert) + await db_session.commit() + await db_session.refresh(alert) + return alert + + +@pytest.mark.asyncio +async def test_list_alerts_scoped_to_owner(client, db_session): + owner_headers, owner_id = _register_and_login(client) + _other_headers, other_id = _register_and_login(client) + + await _make_alert(db_session, owner_id) + await _make_alert(db_session, other_id) + + resp = client.get("/api/v1/alerts", headers=owner_headers) + assert resp.status_code == 200 + body = resp.json() + assert len(body) == 1 + + +@pytest.mark.asyncio +async def test_list_alerts_filters_by_severity(client, db_session): + headers, user_id = _register_and_login(client) + await _make_alert(db_session, user_id, severity=SeverityLevel.CRITICAL) + await _make_alert(db_session, user_id, severity=SeverityLevel.LOW) + + resp = client.get("/api/v1/alerts", headers=headers, params={"severity": "critical"}) + assert resp.status_code == 200 + body = resp.json() + assert len(body) == 1 + assert body[0]["severity"] == "critical" + + +@pytest.mark.asyncio +async def test_list_alerts_filters_by_read_and_resolved(client, db_session): + headers, user_id = _register_and_login(client) + await _make_alert(db_session, user_id, read=True, resolved=False) + await _make_alert(db_session, user_id, read=False, resolved=False) + + resp = client.get("/api/v1/alerts", headers=headers, params={"read": "false"}) + assert resp.status_code == 200 + body = resp.json() + assert len(body) == 1 + assert body[0]["read"] is False + + +@pytest.mark.asyncio +async def test_get_alert_detail_includes_deliveries(client, db_session): + headers, user_id = _register_and_login(client) + alert = await _make_alert(db_session, user_id) + + resp = client.get(f"/api/v1/alerts/{alert.id}", headers=headers) + assert resp.status_code == 200 + body = resp.json() + assert body["id"] == str(alert.id) + assert body["deliveries"] == [] + + +@pytest.mark.asyncio +async def test_get_alert_not_owned_returns_404(client, db_session): + _owner_headers, owner_id = _register_and_login(client) + other_headers, _other_id = _register_and_login(client) + alert = await _make_alert(db_session, owner_id) + + resp = client.get(f"/api/v1/alerts/{alert.id}", headers=other_headers) + assert resp.status_code == 404 + + +@pytest.mark.asyncio +async def test_mark_alert_read(client, db_session): + headers, user_id = _register_and_login(client) + alert = await _make_alert(db_session, user_id, read=False) + + resp = client.post(f"/api/v1/alerts/{alert.id}/read", headers=headers) + assert resp.status_code == 200 + assert resp.json()["read"] is True + + +@pytest.mark.asyncio +async def test_resolve_alert(client, db_session): + headers, user_id = _register_and_login(client) + alert = await _make_alert(db_session, user_id, resolved=False) + + resp = client.post(f"/api/v1/alerts/{alert.id}/resolve", headers=headers) + assert resp.status_code == 200 + assert resp.json()["resolved"] is True + + +@pytest.mark.asyncio +async def test_patch_alert_updates_both_flags(client, db_session): + headers, user_id = _register_and_login(client) + alert = await _make_alert(db_session, user_id) + + resp = client.patch( + f"/api/v1/alerts/{alert.id}", json={"read": True, "resolved": True}, headers=headers + ) + assert resp.status_code == 200 + body = resp.json() + assert body["read"] is True + assert body["resolved"] is True diff --git a/apps/api/tests/unit/test_auth.py b/apps/api/tests/unit/test_auth.py new file mode 100644 index 0000000..a8d5465 --- /dev/null +++ b/apps/api/tests/unit/test_auth.py @@ -0,0 +1,254 @@ +"""Auth flow tests. Runs under AUTH_MODE=jwt (the suite default) except where +`local_mode_client` explicitly exercises the local-dev-user path.""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime, timedelta + +from fastapi.testclient import TestClient + +from app.core.config import get_settings +from app.main import app +from app.models.enums import SecurityEventType +from app.models.user import LOCAL_DEV_USER_ID +from app.repositories.user_security_event_repository import UserSecurityEventRepository + + +def _unique_email() -> str: + return f"user-{uuid.uuid4().hex[:12]}@example.com" + + +def test_register_then_login(client): + email = _unique_email() + register_resp = client.post( + "/api/v1/auth/register", + json={ + "email": email, + "password": "correct-horse-1", + "display_name": "Test User", + }, + ) + assert register_resp.status_code == 201 + body = register_resp.json() + assert body["email"] == email + assert "password" not in body + assert "password_hash" not in body + + login_resp = client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ) + assert login_resp.status_code == 200 + tokens = login_resp.json() + assert tokens["access_token"] + assert tokens["refresh_token"] + assert tokens["token_type"] == "bearer" + + +def test_register_duplicate_email_conflicts(client): + email = _unique_email() + payload = {"email": email, "password": "correct-horse-1", "display_name": "Test User"} + first = client.post("/api/v1/auth/register", json=payload) + assert first.status_code == 201 + + second = client.post("/api/v1/auth/register", json=payload) + assert second.status_code == 409 + + +def test_register_rejects_weak_password(client): + resp = client.post( + "/api/v1/auth/register", + json={"email": _unique_email(), "password": "allletters", "display_name": "Test User"}, + ) + assert resp.status_code == 422 + + +def test_login_wrong_password_rejected(client): + email = _unique_email() + client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "Test User"}, + ) + resp = client.post("/api/v1/auth/login", json={"email": email, "password": "wrong-password-1"}) + assert resp.status_code == 401 + + +def test_me_requires_bearer_token(client): + resp = client.get("/api/v1/auth/me") + assert resp.status_code == 401 + + +def test_me_returns_current_user(client): + email = _unique_email() + client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "Test User"}, + ) + login = client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ).json() + + resp = client.get( + "/api/v1/auth/me", headers={"Authorization": f"Bearer {login['access_token']}"} + ) + assert resp.status_code == 200 + body = resp.json() + assert body["email"] == email + assert body["auth_mode"] == "jwt" + + +def test_refresh_rotates_and_invalidates_old_token(client): + email = _unique_email() + client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "Test User"}, + ) + login = client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ).json() + + refresh_resp = client.post( + "/api/v1/auth/refresh", json={"refresh_token": login["refresh_token"]} + ) + assert refresh_resp.status_code == 200 + new_tokens = refresh_resp.json() + assert new_tokens["refresh_token"] != login["refresh_token"] + + # The old refresh token was rotated out and must not be reusable. + reuse_resp = client.post("/api/v1/auth/refresh", json={"refresh_token": login["refresh_token"]}) + assert reuse_resp.status_code == 401 + + +def test_logout_revokes_refresh_token(client): + email = _unique_email() + client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "Test User"}, + ) + login = client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ).json() + + logout_resp = client.post("/api/v1/auth/logout", json={"refresh_token": login["refresh_token"]}) + assert logout_resp.status_code == 204 + + reuse_resp = client.post("/api/v1/auth/refresh", json={"refresh_token": login["refresh_token"]}) + assert reuse_resp.status_code == 401 + + +def test_local_mode_me_returns_fixed_dev_user_without_token(local_mode_client): + resp = local_mode_client.get("/api/v1/auth/me") + assert resp.status_code == 200 + body = resp.json() + assert body["email"] == "local@ci-agent.local" + assert body["auth_mode"] == "local" + + +def test_register_login_available_even_in_local_mode(local_mode_client): + """Registering/logging in a real account must always be possible, + regardless of AUTH_MODE - the loopback convenience only affects whether + a request can skip auth entirely, not whether real accounts exist.""" + email = _unique_email() + register_resp = local_mode_client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "X"}, + ) + assert register_resp.status_code == 201 + + login_resp = local_mode_client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ) + assert login_resp.status_code == 200 + assert login_resp.json()["access_token"] + + +def test_security_events_requires_auth(client): + resp = client.get("/api/v1/auth/security-events") + assert resp.status_code == 401 + + +def _login_success_count(events: list[dict]) -> int: + return sum(1 for e in events if e["event_type"] == "login_success") + + +def test_local_dev_sign_in_is_logged_to_account_activity(local_mode_client): + """The local-dev bypass has no real login step - hitting any + authenticated endpoint (here /auth/me, the same check the frontend + performs on app load) must still show up as a sign-in.""" + local_mode_client.get("/api/v1/auth/me") + events = local_mode_client.get("/api/v1/auth/security-events").json() + assert any(e["event_type"] == "login_success" for e in events) + + +def test_local_dev_sign_in_is_not_logged_twice_within_cooldown(local_mode_client): + """Every authenticated request runs get_or_create_local_user - without + a cooldown, browsing the app at all would flood Account activity with + one login_success per request.""" + local_mode_client.get("/api/v1/auth/me") + before = _login_success_count(local_mode_client.get("/api/v1/auth/security-events").json()) + + local_mode_client.get("/api/v1/auth/me") + after = _login_success_count(local_mode_client.get("/api/v1/auth/security-events").json()) + + assert after == before + + +async def test_local_dev_sign_in_logs_again_after_cooldown_expires(local_mode_client, db_session): + local_mode_client.get("/api/v1/auth/me") + before = _login_success_count(local_mode_client.get("/api/v1/auth/security-events").json()) + + event_repo = UserSecurityEventRepository(db_session) + last_login = await event_repo.most_recent_of_type( + LOCAL_DEV_USER_ID, SecurityEventType.LOGIN_SUCCESS + ) + assert last_login is not None + last_login.created_at = datetime.now(UTC) - timedelta(minutes=31) + await db_session.commit() + + local_mode_client.get("/api/v1/auth/me") + after = _login_success_count(local_mode_client.get("/api/v1/auth/security-events").json()) + + assert after == before + 1 + + +def test_security_events_returns_own_login_events_only(client): + email_a = _unique_email() + client.post( + "/api/v1/auth/register", + json={"email": email_a, "password": "correct-horse-1", "display_name": "A"}, + ) + login_a = client.post( + "/api/v1/auth/login", json={"email": email_a, "password": "correct-horse-1"} + ).json() + + email_b = _unique_email() + client.post( + "/api/v1/auth/register", + json={"email": email_b, "password": "correct-horse-1", "display_name": "B"}, + ) + client.post("/api/v1/auth/login", json={"email": email_b, "password": "correct-horse-1"}) + + resp = client.get( + "/api/v1/auth/security-events", + headers={"Authorization": f"Bearer {login_a['access_token']}"}, + ) + assert resp.status_code == 200 + events = resp.json() + event_types = {e["event_type"] for e in events} + # registration writes email_verification_sent, login writes login_success - + # both belong to user A only, never user B's events. + assert event_types == {"email_verification_sent", "login_success"} + + +def test_local_mode_setting_alone_does_not_bypass_auth_for_non_loopback_callers(): + """AUTH_MODE=local is not a blanket switch - a request that isn't + actually from loopback (e.g. a LAN/WAN caller, or here Starlette's + TestClient default fake peer) still needs a real bearer token.""" + settings = get_settings().model_copy(update={"auth_mode": "local"}) + app.dependency_overrides[get_settings] = lambda: settings + try: + with TestClient(app) as non_loopback_client: # default peer: ("testclient", 50000) + resp = non_loopback_client.get("/api/v1/auth/me") + assert resp.status_code == 401 + finally: + app.dependency_overrides.pop(get_settings, None) diff --git a/apps/api/tests/unit/test_companies.py b/apps/api/tests/unit/test_companies.py new file mode 100644 index 0000000..ae51d97 --- /dev/null +++ b/apps/api/tests/unit/test_companies.py @@ -0,0 +1,254 @@ +"""Company CRUD, monitor configuration, and ownership isolation tests.""" + +from __future__ import annotations + +import uuid + + +def _register_and_login(client) -> dict[str, str]: + email = f"user-{uuid.uuid4().hex[:12]}@example.com" + client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "Test User"}, + ) + tokens = client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ).json() + return {"Authorization": f"Bearer {tokens['access_token']}"} + + +def _create_company(client, headers, **overrides): + payload = { + "name": "Acme Mobility Systems", + "official_website": "acme-mobility.example.com", + "monitoring_focus": "EV manufacturing expansion", + "competitor_names": ["Rival Motors"], + "alias_names": ["Acme"], + "frequency_type": "weekly", + **overrides, + } + return client.post("/api/v1/companies", json=payload, headers=headers) + + +def test_create_company_normalizes_website_and_sets_defaults(client): + headers = _register_and_login(client) + resp = _create_company(client, headers) + assert resp.status_code == 201 + body = resp.json() + assert body["official_website"] == "https://acme-mobility.example.com" + assert body["slug"] == "acme-mobility-systems" + assert body["status"] == "active" + assert body["aliases"] == ["Acme"] + assert body["competitors"] == ["Rival Motors"] + assert body["monitor_configuration"]["frequency_type"] == "weekly" + assert body["monitor_configuration"]["enabled"] is True + assert body["monitor_configuration"]["next_run"] is not None + + +def test_duplicate_company_names_get_distinct_slugs(client): + headers = _register_and_login(client) + first = _create_company(client, headers).json() + second = _create_company(client, headers).json() + assert first["slug"] != second["slug"] + + +def test_duplicate_company_names_get_a_unique_display_name(client): + headers = _register_and_login(client) + first = _create_company(client, headers).json() + second = _create_company(client, headers).json() + third = _create_company(client, headers).json() + + assert first["name"] == "Acme Mobility Systems" + assert second["name"] == "Acme Mobility Systems (2)" + assert third["name"] == "Acme Mobility Systems (3)" + + +def test_company_name_uniqueness_is_case_insensitive(client): + headers = _register_and_login(client) + first = _create_company(client, headers, name="Stripe").json() + second = _create_company(client, headers, name="STRIPE").json() + + assert first["name"] == "Stripe" + assert second["name"] == "STRIPE (2)" + + +def test_company_name_uniqueness_is_scoped_per_user(client): + owner_headers = _register_and_login(client) + other_headers = _register_and_login(client) + owner_company = _create_company(client, owner_headers).json() + other_company = _create_company(client, other_headers).json() + + # Two different users can each have a company with the exact same name - + # uniqueness is per-user, not global. + assert owner_company["name"] == other_company["name"] == "Acme Mobility Systems" + + +def test_company_list_and_detail_are_scoped_to_owner(client): + owner_headers = _register_and_login(client) + other_headers = _register_and_login(client) + + created = _create_company(client, owner_headers).json() + + owner_list = client.get("/api/v1/companies", headers=owner_headers).json() + assert any(c["id"] == created["id"] for c in owner_list) + + other_list = client.get("/api/v1/companies", headers=other_headers).json() + assert all(c["id"] != created["id"] for c in other_list) + + other_detail = client.get(f"/api/v1/companies/{created['id']}", headers=other_headers) + assert other_detail.status_code == 404 + + owner_detail = client.get(f"/api/v1/companies/{created['id']}", headers=owner_headers) + assert owner_detail.status_code == 200 + + +def test_pause_and_resume_company_toggles_monitor_enabled(client): + headers = _register_and_login(client) + company = _create_company(client, headers).json() + + paused = client.post(f"/api/v1/companies/{company['id']}/pause", headers=headers).json() + assert paused["status"] == "paused" + assert paused["monitor_configuration"]["enabled"] is False + + resumed = client.post(f"/api/v1/companies/{company['id']}/resume", headers=headers).json() + assert resumed["status"] == "active" + assert resumed["monitor_configuration"]["enabled"] is True + + +def test_update_company_replaces_aliases_and_competitors(client): + headers = _register_and_login(client) + company = _create_company(client, headers).json() + + resp = client.patch( + f"/api/v1/companies/{company['id']}", + json={"alias_names": ["New Alias"], "competitor_names": []}, + headers=headers, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["aliases"] == ["New Alias"] + assert body["competitors"] == [] + + +def test_delete_company_removes_it(client): + headers = _register_and_login(client) + company = _create_company(client, headers).json() + + resp = client.delete(f"/api/v1/companies/{company['id']}", headers=headers) + assert resp.status_code == 204 + + resp = client.get(f"/api/v1/companies/{company['id']}", headers=headers) + assert resp.status_code == 404 + + +def test_monitor_configuration_get_and_patch(client): + headers = _register_and_login(client) + company = _create_company(client, headers).json() + + resp = client.get(f"/api/v1/companies/{company['id']}/monitor", headers=headers) + assert resp.status_code == 200 + assert resp.json()["frequency_type"] == "weekly" + + patch_resp = client.patch( + f"/api/v1/companies/{company['id']}/monitor", + json={"frequency_type": "daily"}, + headers=headers, + ) + assert patch_resp.status_code == 200 + assert patch_resp.json()["frequency_type"] == "daily" + + +def test_custom_schedule_below_minimum_interval_is_rejected(client, settings): + headers = _register_and_login(client) + resp = _create_company( + client, + headers, + name="Too Frequent Co", + frequency_type="custom", + interval_minutes=settings.minimum_monitoring_interval_minutes - 1, + ) + assert resp.status_code == 400 + + +def test_custom_schedule_with_valid_interval_is_accepted(client, settings): + headers = _register_and_login(client) + resp = _create_company( + client, + headers, + name="Custom Interval Co", + frequency_type="custom", + interval_minutes=settings.minimum_monitoring_interval_minutes + 30, + ) + assert resp.status_code == 201 + assert resp.json()["monitor_configuration"]["interval_minutes"] == ( + settings.minimum_monitoring_interval_minutes + 30 + ) + + +def test_custom_schedule_requires_interval_or_cron(client): + headers = _register_and_login(client) + resp = _create_company(client, headers, name="No Schedule Co", frequency_type="custom") + assert resp.status_code == 400 + + +def test_company_creation_requires_authentication(client): + resp = client.post("/api/v1/companies", json={"name": "No Auth Co"}) + assert resp.status_code == 401 + + +def test_max_companies_per_user_enforced(client, settings, monkeypatch): + headers = _register_and_login(client) + from app.core import config as config_module + + limited_settings = settings.model_copy(update={"max_companies_per_user": 1}) + from app.main import app + + app.dependency_overrides[config_module.get_settings] = lambda: limited_settings + try: + first = _create_company(client, headers, name="Company One") + assert first.status_code == 201 + second = _create_company(client, headers, name="Company Two") + assert second.status_code == 409 + finally: + app.dependency_overrides.pop(config_module.get_settings, None) + + +def test_create_company_enqueues_enrichment_when_a_key_is_configured(client, settings): + """The real regression guard is every OTHER test in this suite: none of + them configure NINJAPEAR_API_KEY, so the whole rest of the suite proves + the enqueue is skipped by default (see the "without a key" test below + for the direct assertion).""" + from unittest.mock import patch + + from app.core import config as config_module + from app.main import app + + key_settings = settings.model_copy(update={"ninjapear_api_key": "test-key"}) + app.dependency_overrides[config_module.get_settings] = lambda: key_settings + try: + headers = _register_and_login(client) + with patch("app.tasks.enrichment.enrich_company.delay") as mock_delay: + resp = _create_company(client, headers, name="Enriched Co") + assert resp.status_code == 201 + body = resp.json() + mock_delay.assert_called_once_with(body["id"]) + assert body["enrichment"] == { + "status": "pending", + "data": {}, + "errors": {}, + "credits_spent": None, + "fetched_at": None, + } + finally: + app.dependency_overrides.pop(config_module.get_settings, None) + + +def test_create_company_does_not_enqueue_enrichment_without_a_key(client): + from unittest.mock import patch + + headers = _register_and_login(client) + with patch("app.tasks.enrichment.enrich_company.delay") as mock_delay: + resp = _create_company(client, headers, name="Plain Co") + assert resp.status_code == 201 + mock_delay.assert_not_called() + assert resp.json()["enrichment"] is None diff --git a/apps/api/tests/unit/test_correlation_id.py b/apps/api/tests/unit/test_correlation_id.py new file mode 100644 index 0000000..9d9d41d --- /dev/null +++ b/apps/api/tests/unit/test_correlation_id.py @@ -0,0 +1,22 @@ +"""Correlation-ID middleware: every response echoes an X-Request-ID, reusing +an inbound one from a gateway/client if present rather than always minting a +fresh one - see app/main.py::correlation_id_middleware.""" + +from __future__ import annotations + + +def test_response_includes_a_generated_request_id(client): + resp = client.get("/health") + assert "X-Request-ID" in resp.headers + assert len(resp.headers["X-Request-ID"]) > 0 + + +def test_response_reuses_inbound_request_id(client): + resp = client.get("/health", headers={"X-Request-ID": "test-correlation-abc123"}) + assert resp.headers["X-Request-ID"] == "test-correlation-abc123" + + +def test_each_request_gets_a_distinct_generated_id(client): + first = client.get("/health").headers["X-Request-ID"] + second = client.get("/health").headers["X-Request-ID"] + assert first != second diff --git a/apps/api/tests/unit/test_dashboard_api.py b/apps/api/tests/unit/test_dashboard_api.py new file mode 100644 index 0000000..a8cc89a --- /dev/null +++ b/apps/api/tests/unit/test_dashboard_api.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import uuid + + +def _register_and_login(client) -> dict[str, str]: + email = f"user-{uuid.uuid4().hex[:12]}@example.com" + client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "Test User"}, + ) + tokens = client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ).json() + return {"Authorization": f"Bearer {tokens['access_token']}"} + + +def test_dashboard_analytics_returns_zero_filled_buckets_for_a_new_user(client): + headers = _register_and_login(client) + + resp = client.get("/api/v1/dashboard/analytics", headers=headers) + assert resp.status_code == 200 + body = resp.json() + assert set(body["changes_by_type"].keys()) == { + "new_document", + "removed_document", + "content_modified", + "price_change", + "leadership_change", + "filing_new", + } + assert all(v == 0 for v in body["changes_by_type"].values()) + assert body["recent_signals"] == [] + + +def test_dashboard_analytics_requires_auth(client): + resp = client.get("/api/v1/dashboard/analytics") + assert resp.status_code in (401, 403) diff --git a/apps/api/tests/unit/test_discover_endpoint.py b/apps/api/tests/unit/test_discover_endpoint.py new file mode 100644 index 0000000..4e5074a --- /dev/null +++ b/apps/api/tests/unit/test_discover_endpoint.py @@ -0,0 +1,95 @@ +"""POST /companies/discover: persists nothing, returns a proposed profile, +rate-limited tightly since it costs a real search + LLM call.""" + +from __future__ import annotations + +import json +import uuid +from unittest.mock import patch + +import httpx +import respx + +from app.core.rate_limit import limiter + + +def _register_and_login(client) -> dict[str, str]: + email = f"user-{uuid.uuid4().hex[:12]}@example.com" + client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "Test User"}, + ) + tokens = client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ).json() + return {"Authorization": f"Bearer {tokens['access_token']}"} + + +def _mock_empty_github_sec(): + respx.get("https://api.github.com/search/users").mock( + return_value=httpx.Response(200, text=json.dumps({"items": []})) + ) + respx.get("https://www.sec.gov/cgi-bin/browse-edgar").mock( + return_value=httpx.Response( + 200, text='' + ) + ) + + +def test_discover_returns_a_profile_without_persisting_a_company(client): + headers = _register_and_login(client) + + with patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]): + with respx.mock: + _mock_empty_github_sec() + respx.get("https://acmewidgets.com/robots.txt").mock(return_value=httpx.Response(404)) + respx.get("https://acmewidgets.com").mock( + return_value=httpx.Response(200, html="Acme Widgets") + ) + resp = client.post( + "/api/v1/companies/discover", + json={"name": "Acme Widgets"}, + headers=headers, + ) + + assert resp.status_code == 200 + body = resp.json() + assert body["name"] == "Acme Widgets" + assert body["official_website"] == "https://acmewidgets.com" + assert "potential_sources" in body + assert "sources_consulted" in body + + companies = client.get("/api/v1/companies", headers=headers).json() + assert companies == [] + + +def test_discover_requires_auth(client): + resp = client.post("/api/v1/companies/discover", json={"name": "Acme"}) + assert resp.status_code in (401, 403) + + +def test_discover_enforces_rate_limit(client): + headers = _register_and_login(client) + + limiter.enabled = True + try: + with patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]): + with respx.mock: + _mock_empty_github_sec() + respx.route(host="acme0.com").mock(return_value=httpx.Response(404)) + respx.route(host="acme1.com").mock(return_value=httpx.Response(404)) + respx.route(host="acme2.com").mock(return_value=httpx.Response(404)) + respx.route(host="acme3.com").mock(return_value=httpx.Response(404)) + respx.route(host="acme4.com").mock(return_value=httpx.Response(404)) + respx.route(host="acme5.com").mock(return_value=httpx.Response(404)) + statuses = [ + client.post( + "/api/v1/companies/discover", + json={"name": f"Acme{i}"}, + headers=headers, + ).status_code + for i in range(6) + ] + assert 429 in statuses, f"Expected a 429 among {statuses} after 6 rapid discover calls" + finally: + limiter.enabled = False diff --git a/apps/api/tests/unit/test_enrichment_service.py b/apps/api/tests/unit/test_enrichment_service.py new file mode 100644 index 0000000..c890341 --- /dev/null +++ b/apps/api/tests/unit/test_enrichment_service.py @@ -0,0 +1,170 @@ +"""enrichment_service.enrich_company: per-section independent failure +handling, the leadership-lookup cap, and overall status derivation.""" + +from __future__ import annotations + +import uuid + +import pytest + +from app.enrichment.base import ( + CompanyDetails, + CompanyFunding, + LeadershipMember, +) +from app.models.company import Company +from app.models.enums import EnrichmentStatus +from app.repositories.company_enrichment_repository import CompanyEnrichmentRepository +from app.services.enrichment_service import enrich_company + + +class _FakeProvider: + provider_name = "fake" + + def __init__( + self, *, leadership: list[LeadershipMember] | None = None, fail: set[str] | None = None + ): + self._leadership = leadership or [] + self._fail = fail or set() + + async def get_company_details(self, name, website): + if "details" in self._fail: + raise RuntimeError("details boom") + return CompanyDetails(description="A company.", leadership_team=self._leadership) + + async def get_funding(self, name, website): + if "funding" in self._fail: + raise RuntimeError("funding boom") + return CompanyFunding(total_raised="$1M") + + async def get_updates(self, name, website): + if "updates" in self._fail: + raise RuntimeError("updates boom") + return [] + + async def get_competitors(self, name, website): + if "competitors" in self._fail: + raise RuntimeError("competitors boom") + return [] + + async def get_products(self, name, website): + if "products" in self._fail: + raise RuntimeError("products boom") + return [] + + async def get_customers(self, name, website): + if "customers" in self._fail: + raise RuntimeError("customers boom") + return [] + + async def get_work_email(self, person_name, company_website): + if "work_email" in self._fail: + raise RuntimeError("email boom") + return f"{person_name.split()[0].lower()}@example.com" + + async def get_person_profile(self, person_name, company_website): + if "person_profile" in self._fail: + raise RuntimeError("profile boom") + return f"https://example.com/{person_name}", "A bio." + + +async def _make_company(db_session, *, website: str | None = "https://acme.example.com") -> Company: + company = Company( + id=uuid.uuid4(), + user_id=uuid.uuid4(), + name="Acme Corp", + slug=f"acme-{uuid.uuid4().hex[:6]}", + official_website=website, + ) + db_session.add(company) + await db_session.commit() + return company + + +@pytest.mark.asyncio +async def test_all_sections_succeeding_yields_complete_status(db_session, settings): + company = await _make_company(db_session) + provider = _FakeProvider() + + enrichment = await enrich_company(db_session, settings, provider, company) + + assert enrichment.status == EnrichmentStatus.COMPLETE + assert enrichment.errors == {} + assert enrichment.data["funding"]["total_raised"] == "$1M" + assert enrichment.credits_spent is not None and enrichment.credits_spent > 0 + + +@pytest.mark.asyncio +async def test_one_failed_section_yields_partial_without_losing_the_rest(db_session, settings): + company = await _make_company(db_session) + provider = _FakeProvider(fail={"funding"}) + + enrichment = await enrich_company(db_session, settings, provider, company) + + assert enrichment.status == EnrichmentStatus.PARTIAL + assert "funding" in enrichment.errors + assert "funding" not in enrichment.data + assert enrichment.data["description"] == "A company." # the other sections still ran + + +@pytest.mark.asyncio +async def test_every_section_failing_yields_failed_status(db_session, settings): + company = await _make_company(db_session) + provider = _FakeProvider( + fail={"details", "funding", "updates", "competitors", "products", "customers"} + ) + + enrichment = await enrich_company(db_session, settings, provider, company) + + assert enrichment.status == EnrichmentStatus.FAILED + assert enrichment.data.get("leadership_team") == [] + + +@pytest.mark.asyncio +async def test_leadership_lookups_are_capped(db_session, settings): + company = await _make_company(db_session) + leadership = [LeadershipMember(name=f"Person {i}") for i in range(8)] + provider = _FakeProvider(leadership=leadership) + capped_settings = settings.model_copy(update={"ninjapear_max_leadership_lookups": 3}) + + enrichment = await enrich_company(db_session, capped_settings, provider, company) + + team = enrichment.data["leadership_team"] + assert len(team) == 8 # every discovered leader is kept... + with_email = [m for m in team if m.get("work_email")] + assert len(with_email) == 3 # ...but only the first 3 get person-level lookups + + +@pytest.mark.asyncio +async def test_no_website_fails_immediately_without_calling_the_provider(db_session, settings): + """NinjaPear identifies a company by website only - every call would + fail identically, so this must short-circuit to FAILED before spending + any credits, rather than attempting (and paying for) doomed calls.""" + company = await _make_company(db_session, website=None) + + class _ExplodingProvider: + provider_name = "exploding" + + async def get_company_details(self, *a, **k): + raise AssertionError("must not be called without a website") + + enrichment = await enrich_company(db_session, settings, _ExplodingProvider(), company) + + assert enrichment.status == EnrichmentStatus.FAILED + assert enrichment.credits_spent == 0 + assert "details" in enrichment.errors + + +@pytest.mark.asyncio +async def test_upsert_updates_the_same_row_on_a_second_call(db_session, settings): + company = await _make_company(db_session) + + first = await enrich_company(db_session, settings, _FakeProvider(), company) + second = await enrich_company(db_session, settings, _FakeProvider(fail={"funding"}), company) + + assert first.id == second.id + assert second.status == EnrichmentStatus.PARTIAL + + stored = await CompanyEnrichmentRepository(db_session).get_for_company(company.id) + assert stored.id == first.id + assert stored.status == EnrichmentStatus.PARTIAL diff --git a/apps/api/tests/unit/test_extraction.py b/apps/api/tests/unit/test_extraction.py new file mode 100644 index 0000000..677effb --- /dev/null +++ b/apps/api/tests/unit/test_extraction.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from app.collectors.extraction import ( + canonicalize_url, + compute_content_hash, + extract_readable_text, + extract_title, + normalize_whitespace, +) + + +def test_normalize_whitespace_collapses_blank_lines_and_trims(): + raw = " Title \n\n\n\nBody line one \n \nBody line two " + normalized = normalize_whitespace(raw) + assert normalized == "Title\n\nBody line one\n\nBody line two" + + +def test_compute_content_hash_is_stable_and_sensitive_to_change(): + a = compute_content_hash("hello world") + b = compute_content_hash("hello world") + c = compute_content_hash("hello world!") + assert a == b + assert a != c + + +def test_canonicalize_url_strips_tracking_params_and_trailing_slash(): + url = "HTTPS://Example.com/About/?utm_source=x&ref=y" + assert canonicalize_url(url) == "https://example.com/About?ref=y" + + +def test_canonicalize_url_normalizes_root_path(): + assert canonicalize_url("https://example.com") == "https://example.com/" + + +def test_extract_readable_text_prefers_trafilatura_for_article_html(): + html = """ + + +
+

Acme launches new product

+

Acme Corp announced a new electric vehicle platform today, expanding its lineup.

+

The company said manufacturing will begin next quarter at its main facility.

+
+
Copyright 2026
+ + """ + text, method = extract_readable_text(html, "https://example.com/news/1") + assert "Acme launches new product" in text + assert "Copyright 2026" not in text + assert method in ("trafilatura", "beautifulsoup_fallback") + + +def test_extract_readable_text_falls_back_when_trafilatura_finds_nothing(): + html = "" + text, method = extract_readable_text(html, "https://example.com/thin") + assert method == "beautifulsoup_fallback" + assert text == "" + + +def test_extract_title_prefers_title_tag(): + html = "Acme — About

About

" + assert extract_title(html) == "Acme — About" + + +def test_extract_title_falls_back_to_h1(): + html = "

Fallback Heading

" + assert extract_title(html) == "Fallback Heading" diff --git a/apps/api/tests/unit/test_extractors.py b/apps/api/tests/unit/test_extractors.py new file mode 100644 index 0000000..ed537f4 --- /dev/null +++ b/apps/api/tests/unit/test_extractors.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from app.change_detection.extractors import extract_prices, mentions_leadership_title + + +def test_extract_prices_finds_dollar_amounts(): + text = "The Pro plan is $49/month and the Enterprise plan is $199.99/month." + prices = extract_prices(text) + assert "$49/month" in prices + assert "$199.99/month" in prices + + +def test_extract_prices_empty_when_no_prices(): + assert extract_prices("No pricing information on this page.") == set() + + +def test_mentions_leadership_title_detects_ceo(): + assert mentions_leadership_title("Jane Smith has been appointed as the new CEO.") is True + + +def test_mentions_leadership_title_false_when_absent(): + assert mentions_leadership_title("We shipped a new feature this week.") is False diff --git a/apps/api/tests/unit/test_health.py b/apps/api/tests/unit/test_health.py new file mode 100644 index 0000000..2e7b8af --- /dev/null +++ b/apps/api/tests/unit/test_health.py @@ -0,0 +1,25 @@ +"""Basic liveness/readiness smoke tests.""" + +from __future__ import annotations + + +def test_root_health(client): + resp = client.get("/health") + assert resp.status_code == 200 + assert resp.json()["status"] == "ok" + + +def test_v1_health(client): + resp = client.get("/api/v1/health") + assert resp.status_code == 200 + body = resp.json() + assert body["status"] == "ok" + assert "app_name" in body + + +def test_system_status_reports_mock_providers(client): + resp = client.get("/api/v1/system/status") + assert resp.status_code == 200 + body = resp.json() + assert body["llm_provider"] == "mock" + assert body["search_provider"] == "mock" diff --git a/apps/api/tests/unit/test_ip_throttle_service.py b/apps/api/tests/unit/test_ip_throttle_service.py new file mode 100644 index 0000000..68a1af1 --- /dev/null +++ b/apps/api/tests/unit/test_ip_throttle_service.py @@ -0,0 +1,245 @@ +"""ip_throttle_service: the escalation engine backing resend-verification, +resend-password-reset, and failed-login throttling. Every stage is walked +by monkeypatching `_now()` forward - no real waiting, no manual +brute-forcing, per the explicit "test it smarter" requirement this feature +was built under.""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime, timedelta + +import pytest + +from app.db.base import ensure_aware_utc +from app.models.enums import ThrottleAction +from app.repositories.ip_throttle_repository import IpThrottleRepository +from app.services import ip_throttle_service +from app.services.ip_throttle_service import ( + LOGIN_BACKOFF_SECONDS, + RESEND_BACKOFF_SECONDS, + TIMEOUT_LADDER_SECONDS, + peek_throttle, + record_attempt, + reset_on_success, +) + + +def _unique_ip() -> str: + return f"10.{uuid.uuid4().int % 250}.{uuid.uuid4().int % 250}.{uuid.uuid4().int % 250}" + + +class _Clock: + """A monkeypatchable fake clock - advances only when told to, so tests + can jump straight past a 5-hour timeout without real time passing.""" + + def __init__(self) -> None: + self.now = datetime(2026, 1, 1, tzinfo=UTC) + + def __call__(self) -> datetime: + return self.now + + def advance(self, seconds: float) -> None: + self.now += timedelta(seconds=seconds) + + +@pytest.fixture() +def clock(monkeypatch): + fake_clock = _Clock() + monkeypatch.setattr(ip_throttle_service, "_now", fake_clock) + return fake_clock + + +async def _exhaust_stages_into_timeout(db_session, clock, ip: str, action: ThrottleAction) -> None: + """Drives one full stage array to exhaustion (advancing the clock past + each required wait, exactly as a real repeated-offender caller would), + landing the state in a fresh timeout. Always starts from a peek (so a + prior *completed* timeout cycle is lazily reset first, mirroring how + every real call site checks peek_throttle before record_attempt).""" + for _ in RESEND_BACKOFF_SECONDS: + peeked = await peek_throttle(db_session, ip, action) + assert peeked.allowed is True + await record_attempt(db_session, ip, action, RESEND_BACKOFF_SECONDS) + state = await IpThrottleRepository(db_session).get_state(ip, action) + if state.next_allowed_at is not None: + next_allowed_at = ensure_aware_utc(state.next_allowed_at) + clock.advance((next_allowed_at - clock.now).total_seconds()) + # One more attempt now that every stage is consumed - this is the one + # that exhausts the array and enters a timeout. + await record_attempt(db_session, ip, action, RESEND_BACKOFF_SECONDS) + + +async def test_first_attempt_is_always_allowed(db_session, clock): + ip = _unique_ip() + result = await peek_throttle(db_session, ip, ThrottleAction.RESEND_VERIFICATION) + assert result.allowed is True + assert result.banned is False + + +async def test_resend_backoff_stages_match_spec_exactly(db_session, clock): + ip = _unique_ip() + action = ThrottleAction.RESEND_VERIFICATION + + for expected_wait in RESEND_BACKOFF_SECONDS: + result = await peek_throttle(db_session, ip, action) + assert result.allowed is True + await record_attempt(db_session, ip, action, RESEND_BACKOFF_SECONDS) + + blocked = await peek_throttle(db_session, ip, action) + assert blocked.allowed is False + assert blocked.retry_after_seconds == expected_wait + + clock.advance(expected_wait) + + +async def test_resend_exhausting_stages_enters_first_timeout(db_session, clock): + ip = _unique_ip() + action = ThrottleAction.RESEND_RESET + + await _exhaust_stages_into_timeout(db_session, clock, ip, action) + + blocked = await peek_throttle(db_session, ip, action) + assert blocked.allowed is False + assert blocked.retry_after_seconds == TIMEOUT_LADDER_SECONDS[0] + + +async def test_timeout_expires_resets_attempts_but_keeps_offense_memory(db_session, clock): + ip = _unique_ip() + action = ThrottleAction.RESEND_RESET + + await _exhaust_stages_into_timeout(db_session, clock, ip, action) + + repo = IpThrottleRepository(db_session) + state = await repo.get_state(ip, action) + assert state.offense_count == 1 + + clock.advance(TIMEOUT_LADDER_SECONDS[0]) + result = await peek_throttle(db_session, ip, action) + assert result.allowed is True + + state = await repo.get_state(ip, action) + assert state.attempt_count == 0 + assert state.offense_count == 1 # memory kept, exactly as specified + + +async def test_repeat_offense_uses_next_longer_timeout(db_session, clock): + ip = _unique_ip() + action = ThrottleAction.RESEND_RESET + + await _exhaust_stages_into_timeout(db_session, clock, ip, action) # offense #1 -> 30min + clock.advance(TIMEOUT_LADDER_SECONDS[0]) + + await _exhaust_stages_into_timeout(db_session, clock, ip, action) # offense #2 -> 1h + blocked = await peek_throttle(db_session, ip, action) + assert blocked.retry_after_seconds == TIMEOUT_LADDER_SECONDS[1] + + +async def test_escalation_past_the_ladder_results_in_permanent_ban(db_session, clock): + ip = _unique_ip() + action = ThrottleAction.RESEND_RESET + + for i, timeout_seconds in enumerate(TIMEOUT_LADDER_SECONDS): + await _exhaust_stages_into_timeout(db_session, clock, ip, action) + clock.advance(timeout_seconds) + result = await peek_throttle(db_session, ip, action) + assert result.banned is False, f"should not be banned yet after offense {i + 1}" + + # One more full cycle exhausts past the ladder entirely -> permanent ban. + await _exhaust_stages_into_timeout(db_session, clock, ip, action) + result = await peek_throttle(db_session, ip, action) + assert result.banned is True + assert result.allowed is False + + +async def test_login_five_instant_attempts_then_escalating_delays(db_session, clock): + ip = _unique_ip() + action = ThrottleAction.FAILED_LOGIN + + for _ in range(5): + result = await peek_throttle(db_session, ip, action) + assert result.allowed is True + await record_attempt(db_session, ip, action, LOGIN_BACKOFF_SECONDS) + # All 5 were genuinely free - no wait was ever imposed before any of them. + + blocked = await peek_throttle(db_session, ip, action) + assert blocked.allowed is False + assert blocked.retry_after_seconds == 5 # first real delay stage, gating attempt 6 + + +async def test_login_all_delay_stages_match_spec_in_order(db_session, clock): + ip = _unique_ip() + action = ThrottleAction.FAILED_LOGIN + + for _ in range(5): + await record_attempt(db_session, ip, action, LOGIN_BACKOFF_SECONDS) + + expected_delays = [5, 15, 30, 60, 120, 300, 900] + for expected_wait in expected_delays: + blocked = await peek_throttle(db_session, ip, action) + assert blocked.retry_after_seconds == expected_wait + clock.advance(expected_wait) + await record_attempt(db_session, ip, action, LOGIN_BACKOFF_SECONDS) + + # That was the 12th recorded attempt (5 free + 7 delayed) - stages are + # now exhausted, so the IP itself enters its first timeout. + blocked = await peek_throttle(db_session, ip, action) + assert blocked.retry_after_seconds == TIMEOUT_LADDER_SECONDS[0] + + +async def test_ban_blocks_every_action_type_for_that_ip(db_session, clock): + ip = _unique_ip() + action = ThrottleAction.RESEND_RESET + + for timeout_seconds in TIMEOUT_LADDER_SECONDS: + await _exhaust_stages_into_timeout(db_session, clock, ip, action) + clock.advance(timeout_seconds) + await _exhaust_stages_into_timeout(db_session, clock, ip, action) # permanent ban + + # A totally different action from the same IP is also blocked - bans are + # global per-IP, not scoped to the action that triggered them. + login_result = await peek_throttle(db_session, ip, ThrottleAction.FAILED_LOGIN) + assert login_result.banned is True + + +async def test_reset_on_success_clears_stage_but_not_offense_count(db_session, clock): + ip = _unique_ip() + action = ThrottleAction.FAILED_LOGIN + + for _ in range(6): + await record_attempt(db_session, ip, action, LOGIN_BACKOFF_SECONDS) + + repo = IpThrottleRepository(db_session) + state = await repo.get_state(ip, action) + assert state.attempt_count == 6 + + await reset_on_success(db_session, ip, action) + state = await repo.get_state(ip, action) + assert state.attempt_count == 0 + assert state.next_allowed_at is None + + result = await peek_throttle(db_session, ip, action) + assert result.allowed is True + + +async def test_admin_unban_gives_a_clean_slate(db_session, clock): + ip = _unique_ip() + action = ThrottleAction.RESEND_RESET + + for timeout_seconds in TIMEOUT_LADDER_SECONDS: + await _exhaust_stages_into_timeout(db_session, clock, ip, action) + clock.advance(timeout_seconds) + await _exhaust_stages_into_timeout(db_session, clock, ip, action) # permanent ban + + result = await peek_throttle(db_session, ip, action) + assert result.banned is True + + repo = IpThrottleRepository(db_session) + cleared = await repo.clear_ban_and_state(ip) + assert cleared is True + + result = await peek_throttle(db_session, ip, action) + assert result.allowed is True + assert result.banned is False + + state = await repo.get_state(ip, action) + assert state.offense_count == 0 # a real pardon, not just lifting the ban diff --git a/apps/api/tests/unit/test_llm_providers.py b/apps/api/tests/unit/test_llm_providers.py new file mode 100644 index 0000000..458a1bd --- /dev/null +++ b/apps/api/tests/unit/test_llm_providers.py @@ -0,0 +1,179 @@ +"""Anthropic/Ollama providers, with the SDK/HTTP layer mocked - these never +run against a real paid API in the test suite. Verifies the structured- +output + repair-loop wiring actually works, not just that it imports.""" + +from __future__ import annotations + +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import httpx +import pytest +import respx +from pydantic import BaseModel + +from app.analysis.llm.base import LLMResponseError +from app.core.config import Settings + + +class _Toy(BaseModel): + answer: str + + +def _settings(**overrides) -> Settings: + defaults = { + "llm_provider": "anthropic", + "anthropic_api_key": "sk-test", + "anthropic_model": "claude-test", + "llm_max_retries": 1, + "llm_max_tokens_per_request": 100, + } + defaults.update(overrides) + return Settings(**defaults) + + +@pytest.mark.asyncio +async def test_anthropic_provider_parses_tool_use_response(): + from app.analysis.llm.anthropic_provider import AnthropicLLMProvider + + provider = AnthropicLLMProvider(_settings()) + fake_response = SimpleNamespace( + content=[SimpleNamespace(type="tool_use", input={"answer": "42"})] + ) + with patch.object(provider._client.messages, "create", AsyncMock(return_value=fake_response)): + result = await provider.generate_structured("system", "user", _Toy) + + assert result.answer == "42" + + +@pytest.mark.asyncio +async def test_anthropic_provider_retries_then_raises_on_missing_tool_use(): + from app.analysis.llm.anthropic_provider import AnthropicLLMProvider + + provider = AnthropicLLMProvider(_settings(llm_max_retries=1)) + fake_response = SimpleNamespace(content=[SimpleNamespace(type="text", text="oops")]) + with patch.object( + provider._client.messages, "create", AsyncMock(return_value=fake_response) + ) as mock_create: + with pytest.raises(LLMResponseError): + await provider.generate_structured("system", "user", _Toy) + + assert mock_create.call_count == 2 # initial attempt + 1 retry + + +@pytest.mark.asyncio +async def test_anthropic_provider_generate_text_joins_text_blocks(): + from app.analysis.llm.anthropic_provider import AnthropicLLMProvider + + provider = AnthropicLLMProvider(_settings()) + fake_response = SimpleNamespace( + content=[ + SimpleNamespace(type="text", text="Hello"), + SimpleNamespace(type="text", text="world"), + ] + ) + with patch.object(provider._client.messages, "create", AsyncMock(return_value=fake_response)): + text = await provider.generate_text("system", "user") + + assert text == "Hello\nworld" + + +@pytest.mark.asyncio +async def test_ollama_provider_parses_json_response(): + from app.analysis.llm.ollama_provider import OllamaLLMProvider + + settings = Settings( + llm_provider="ollama", + ollama_base_url="http://ollama.local:11434", + ollama_model="llama-test", + llm_max_retries=1, + ) + provider = OllamaLLMProvider(settings) + + with respx.mock: + respx.post("http://ollama.local:11434/api/chat").mock( + return_value=httpx.Response( + 200, json={"message": {"content": json.dumps({"answer": "42"})}} + ) + ) + result = await provider.generate_structured("system", "user", _Toy) + + assert result.answer == "42" + + +@pytest.mark.asyncio +async def test_ollama_provider_retries_then_raises_on_invalid_json(): + from app.analysis.llm.ollama_provider import OllamaLLMProvider + + settings = Settings( + llm_provider="ollama", + ollama_base_url="http://ollama.local:11434", + ollama_model="llama-test", + llm_max_retries=1, + ) + provider = OllamaLLMProvider(settings) + + with respx.mock: + route = respx.post("http://ollama.local:11434/api/chat").mock( + return_value=httpx.Response(200, json={"message": {"content": "not json"}}) + ) + with pytest.raises(LLMResponseError): + await provider.generate_structured("system", "user", _Toy) + + assert route.call_count == 2 + + +def _gemini_settings(**overrides) -> Settings: + defaults = { + "llm_provider": "gemini", + "gemini_api_key": "test-key", + "gemini_model": "gemini-test", + "llm_max_retries": 1, + "llm_max_tokens_per_request": 100, + } + defaults.update(overrides) + return Settings(**defaults) + + +@pytest.mark.asyncio +async def test_gemini_provider_parses_structured_response(): + from app.analysis.llm.gemini_provider import GeminiLLMProvider + + provider = GeminiLLMProvider(_gemini_settings()) + fake_response = SimpleNamespace(parsed=_Toy(answer="42")) + with patch.object( + provider._client.aio.models, "generate_content", AsyncMock(return_value=fake_response) + ): + result = await provider.generate_structured("system", "user", _Toy) + + assert result.answer == "42" + + +@pytest.mark.asyncio +async def test_gemini_provider_retries_then_raises_when_unparsed(): + from app.analysis.llm.gemini_provider import GeminiLLMProvider + + provider = GeminiLLMProvider(_gemini_settings(llm_max_retries=1)) + fake_response = SimpleNamespace(parsed=None) + with patch.object( + provider._client.aio.models, "generate_content", AsyncMock(return_value=fake_response) + ) as mock_generate: + with pytest.raises(LLMResponseError): + await provider.generate_structured("system", "user", _Toy) + + assert mock_generate.call_count == 2 # initial attempt + 1 retry + + +@pytest.mark.asyncio +async def test_gemini_provider_generate_text_returns_response_text(): + from app.analysis.llm.gemini_provider import GeminiLLMProvider + + provider = GeminiLLMProvider(_gemini_settings()) + fake_response = SimpleNamespace(text="Hello world") + with patch.object( + provider._client.aio.models, "generate_content", AsyncMock(return_value=fake_response) + ): + text = await provider.generate_text("system", "user") + + assert text == "Hello world" diff --git a/apps/api/tests/unit/test_mock_llm_provider.py b/apps/api/tests/unit/test_mock_llm_provider.py new file mode 100644 index 0000000..ab5ad74 --- /dev/null +++ b/apps/api/tests/unit/test_mock_llm_provider.py @@ -0,0 +1,160 @@ +"""MockLLMProvider: every analysis task's response schema must come back +valid and genuinely reflect the evidence passed in - never an empty +placeholder unrelated to the input.""" + +from __future__ import annotations + +import pytest + +from app.analysis.llm.mock import MockLLMProvider +from app.prompts.alert_summarization import AlertSummary +from app.prompts.base import build_user_prompt +from app.prompts.change_significance import ChangeSignificanceAssessment +from app.prompts.extraction import ExtractionResult +from app.prompts.relevance import RelevanceAssessment +from app.prompts.report_generation import ReportContent +from app.prompts.synthesis import SynthesisResult + +provider = MockLLMProvider() + + +@pytest.mark.asyncio +async def test_relevance_matches_focus_keyword(): + prompt = build_user_prompt( + "assess", + { + "monitoring_focus": "electric vehicle manufacturing expansion", + "document_text": "The company announced a new manufacturing facility for electric vehicles.", + }, + ) + result = await provider.generate_structured("system", prompt, RelevanceAssessment) + assert result.is_relevant is True + assert result.matches_focus is True + + +@pytest.mark.asyncio +async def test_extraction_pulls_first_sentence_as_signal(): + prompt = build_user_prompt( + "extract", {"document_text": "Acme Corp opened a new facility. It will employ 200 people."} + ) + result = await provider.generate_structured("system", prompt, ExtractionResult) + assert len(result.signals) == 1 + assert "Acme Corp opened a new facility" in result.signals[0].description + + +@pytest.mark.asyncio +async def test_synthesis_requires_multiple_signals(): + prompt_one = build_user_prompt("synthesize", {"signals": [{"description": "a"}]}) + result_one = await provider.generate_structured("system", prompt_one, SynthesisResult) + assert result_one.conclusions == [] + + prompt_two = build_user_prompt( + "synthesize", {"signals": [{"description": "a"}, {"description": "b"}]} + ) + result_two = await provider.generate_structured("system", prompt_two, SynthesisResult) + assert len(result_two.conclusions) == 1 + assert result_two.conclusions[0].source_count == 2 + + +@pytest.mark.asyncio +async def test_report_reflects_evidence_counts(): + prompt = build_user_prompt( + "report", + { + "company_profile": {"name": "Acme Corp"}, + "source_documents": [ + { + "id": "d1", + "title": "Job posting", + "url": "https://x.com/1", + "source_type": "job_posting", + "retrieved_date": "2026-01-01", + } + ], + "detected_changes": [ + { + "id": "c1", + "summary": "New job posting detected", + "change_type": "new_document", + "severity": "medium", + "confidence_score": 0.6, + "created_at": "2026-01-01", + } + ], + "sources_that_failed_to_collect": ["Broken Source"], + }, + ) + result = await provider.generate_structured("system", prompt, ReportContent) + assert "Acme Corp" in result.executive_summary + assert "1" in result.executive_summary # document/change counts mentioned + assert len(result.recent_developments) == 1 + assert len(result.hiring_signals) == 1 + assert any("Broken Source" in u for u in result.unknowns_and_missing_data) + + +@pytest.mark.asyncio +async def test_report_grounds_overview_in_discovered_profile_even_with_no_documents(): + """The bug this covers: a report generated before any monitoring run had + collected evidence used to come back near-empty even though the company's + discovered profile (from onboarding) had real data. That profile data + must now ground company_overview/market_positioning.""" + prompt = build_user_prompt( + "report", + { + "company_profile": { + "name": "Stripe", + "description": "Stripe builds economic infrastructure for the internet.", + "industry": "Financial infrastructure", + "headquarters": "South San Francisco, California", + "competitors": ["PayPal"], + "aliases": [], + }, + "source_documents": [], + "detected_changes": [], + "sources_that_failed_to_collect": [], + }, + ) + result = await provider.generate_structured("system", prompt, ReportContent) + assert "economic infrastructure" in result.company_overview + assert "South San Francisco" in result.company_overview + assert "PayPal" in result.market_positioning + + +@pytest.mark.asyncio +async def test_change_significance_reflects_deterministic_severity(): + prompt = build_user_prompt( + "assess", + { + "change_type": "leadership_change", + "deterministic_severity": "high", + "deterministic_confidence": 0.8, + }, + ) + result = await provider.generate_structured("system", prompt, ChangeSignificanceAssessment) + assert result.is_meaningful is True + assert result.should_notify is True + assert "high" in result.why_it_matters + + +@pytest.mark.asyncio +async def test_alert_summary_title_under_100_chars(): + prompt = build_user_prompt( + "summarize", + { + "company_name": "Acme Corp", + "change_type": "price_change", + "severity": "medium", + "confidence": 0.6, + }, + ) + result = await provider.generate_structured("system", prompt, AlertSummary) + assert len(result.title) <= 100 + assert "Acme Corp" in result.title + + +@pytest.mark.asyncio +async def test_generate_text_does_not_crash(): + prompt = build_user_prompt("do something", {"a": 1}) + text = await provider.generate_text("system", prompt) + assert isinstance(text, str) + assert len(text) > 0 diff --git a/apps/api/tests/unit/test_monitoring_smoke.py b/apps/api/tests/unit/test_monitoring_smoke.py new file mode 100644 index 0000000..5afa69d --- /dev/null +++ b/apps/api/tests/unit/test_monitoring_smoke.py @@ -0,0 +1,220 @@ +"""Monitoring run endpoints, exercised via HTTP with CELERY_TASK_ALWAYS_EAGER +so `.delay()` runs the task synchronously in-process (see app/tasks/base.py +for why that needs a threaded asyncio bridge to work from an async route +handler). No live network: every collector's discovery/collection call is +respx-mocked.""" + +from __future__ import annotations + +import json +import uuid +from unittest.mock import patch + +import httpx +import pytest +import respx + + +def _register_and_login(client) -> dict[str, str]: + email = f"user-{uuid.uuid4().hex[:12]}@example.com" + client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "Test User"}, + ) + tokens = client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ).json() + return {"Authorization": f"Bearer {tokens['access_token']}"} + + +def _create_company(client, headers, **overrides): + payload = { + "name": f"Run Test Co {uuid.uuid4().hex[:6]}", + "frequency_type": "weekly", + **overrides, + } + return client.post("/api/v1/companies", json=payload, headers=headers).json() + + +def _mock_empty_discovery(): + """Every discoverable collector type returns nothing (or a harmless + stub), so a first run doesn't need per-source mocks for whatever + discovery happens to find. RSS (Google News) and GOV_CONTRACT + (USASpending) are always discovered unconditionally - see + discovery_service.py's _PREVIEWABLE_TYPES - so their collect() calls + need mocking here too, unlike GitHub/SEC which discover.() itself.""" + respx.get("https://api.github.com/search/users").mock( + return_value=httpx.Response(200, text=json.dumps({"items": []})) + ) + respx.get("https://www.sec.gov/cgi-bin/browse-edgar").mock( + return_value=httpx.Response( + 200, text='' + ) + ) + respx.get(url__regex=r"https://news\.google\.com/rss/search.*").mock( + return_value=httpx.Response( + 200, + text=( + 'Google News' + "Testhttps://example.com/news" + "Test news item" + ), + ) + ) + respx.post("https://api.usaspending.gov/api/v2/search/spending_by_award/").mock( + return_value=httpx.Response(200, json={"results": []}) + ) + + +def test_run_now_executes_and_reports_final_status(client): + headers = _register_and_login(client) + company = _create_company(client, headers) + + with patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]): + with respx.mock: + _mock_empty_discovery() + resp = client.post(f"/api/v1/companies/{company['id']}/run", headers=headers) + + assert resp.status_code == 202 + run = resp.json() + assert run["trigger_type"] == "manual" + + # Eager mode means the task already finished by the time .delay() returns. + detail = client.get(f"/api/v1/runs/{run['id']}", headers=headers).json() + assert detail["status"] == "successful" + assert detail["completed_at"] is not None + + +async def test_run_now_is_idempotent_for_an_active_run(db_session, settings): + """A run-now call while one is already queued/running returns that same + run instead of enqueuing a duplicate. Exercised at the service layer + directly: under CELERY_TASK_ALWAYS_EAGER a `.delay()` call runs the task + to completion before returning, so an HTTP-level test can never observe + an in-flight run to dedupe against.""" + from app.models.company import Company + from app.models.enums import MonitoringRunTrigger + from app.models.monitor_configuration import MonitorConfiguration + from app.repositories.monitoring_run_repository import MonitoringRunRepository + from app.services import monitoring_service + + user_id = uuid.uuid4() + company = Company( + id=uuid.uuid4(), + user_id=user_id, + name="Idempotency Co", + slug=f"idempotency-co-{uuid.uuid4().hex[:6]}", + ) + db_session.add(company) + db_session.add(MonitorConfiguration(company_id=company.id)) + await db_session.commit() + + run_repo = MonitoringRunRepository(db_session) + existing = await run_repo.create( + company_id=company.id, trigger_type=MonitoringRunTrigger.SCHEDULED + ) + await db_session.commit() + + with patch("app.tasks.collection.run_monitoring.delay") as mock_delay: + result = await monitoring_service.enqueue_run_now(db_session, settings, user_id, company.id) + + assert result.id == existing.id + mock_delay.assert_not_called() + + +async def test_run_now_enforces_daily_manual_run_cap(db_session, settings): + from app.core.errors import RateLimitedError + from app.models.company import Company + from app.models.enums import MonitoringRunStatus, MonitoringRunTrigger + from app.models.monitor_configuration import MonitorConfiguration + from app.repositories.monitoring_run_repository import MonitoringRunRepository + from app.services import monitoring_service + + user_id = uuid.uuid4() + company = Company( + id=uuid.uuid4(), + user_id=user_id, + name="Rate Limited Co", + slug=f"rate-limited-co-{uuid.uuid4().hex[:6]}", + ) + db_session.add(company) + db_session.add(MonitorConfiguration(company_id=company.id)) + await db_session.commit() + + run_repo = MonitoringRunRepository(db_session) + for _ in range(2): + run = await run_repo.create(company_id=company.id, trigger_type=MonitoringRunTrigger.MANUAL) + await run_repo.mark_finished(run, status=MonitoringRunStatus.SUCCESSFUL, error_summary=None) + await db_session.commit() + + capped_settings = settings.model_copy(update={"max_manual_runs_per_day": 2}) + + with pytest.raises(RateLimitedError): + await monitoring_service.enqueue_run_now(db_session, capped_settings, user_id, company.id) + + +def test_run_history_and_single_run_scoped_to_owner(client): + owner_headers = _register_and_login(client) + other_headers = _register_and_login(client) + company = _create_company(client, owner_headers) + + with patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]): + with respx.mock: + _mock_empty_discovery() + run = client.post( + f"/api/v1/companies/{company['id']}/run", headers=owner_headers + ).json() + + assert ( + client.get(f"/api/v1/companies/{company['id']}/runs", headers=other_headers).status_code + == 404 + ) + assert client.get(f"/api/v1/runs/{run['id']}", headers=other_headers).status_code == 404 + assert client.get(f"/api/v1/runs/{run['id']}", headers=owner_headers).status_code == 200 + + +def test_successful_first_run_generates_a_baseline_report(client): + headers = _register_and_login(client) + company = _create_company(client, headers) + client.post( + f"/api/v1/companies/{company['id']}/sources", + json={ + "source_type": "custom_url", + "name": "Pricing", + "base_url": "https://example.com/pricing", + }, + headers=headers, + ) + + with patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]): + with respx.mock: + _mock_empty_discovery() + respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404)) + respx.get("https://example.com/pricing").mock( + return_value=httpx.Response( + 200, + html="Pricing" + "

Pricing

Plans start at $10/month.

" + "", + ) + ) + client.post(f"/api/v1/companies/{company['id']}/run", headers=headers) + + reports = client.get(f"/api/v1/companies/{company['id']}/reports", headers=headers).json() + assert len(reports) == 1 + assert reports[0]["report_type"] == "baseline" + assert reports[0]["model_provider"] == "mock" + + +def test_manual_run_does_not_disrupt_next_scheduled_run(client): + headers = _register_and_login(client) + company = _create_company(client, headers) + original_next_run = company["monitor_configuration"]["next_run"] + + with patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]): + with respx.mock: + _mock_empty_discovery() + client.post(f"/api/v1/companies/{company['id']}/run", headers=headers) + + refreshed = client.get(f"/api/v1/companies/{company['id']}", headers=headers).json() + assert refreshed["monitor_configuration"]["next_run"] == original_next_run + assert refreshed["monitor_configuration"]["last_run"] is not None diff --git a/apps/api/tests/unit/test_noise_filters.py b/apps/api/tests/unit/test_noise_filters.py new file mode 100644 index 0000000..c7f6036 --- /dev/null +++ b/apps/api/tests/unit/test_noise_filters.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from app.change_detection.noise_filters import strip_noise + + +def test_strips_dynamic_timestamps(): + text = "Page content.\nLast modified: 2026-03-14 09:00:00\nMore content." + cleaned = strip_noise(text) + assert "2026-03-14" not in cleaned + assert "Page content." in cleaned + assert "More content." in cleaned + + +def test_strips_cookie_banner_boilerplate(): + text = "We use cookies to improve your experience. Real content here." + cleaned = strip_noise(text) + assert "cookies" not in cleaned.lower() + assert "Real content here." in cleaned + + +def test_strips_copyright_year(): + text = "Footer text. Copyright (c) 2026 Acme Corp. All rights reserved." + cleaned = strip_noise(text) + assert "2026" not in cleaned + + +def test_leaves_ordinary_prose_untouched(): + text = "Acme Corp announced a new electric vehicle platform this week." + cleaned = strip_noise(text) + assert "Acme Corp announced a new electric vehicle platform this week." in cleaned diff --git a/apps/api/tests/unit/test_notification_destinations.py b/apps/api/tests/unit/test_notification_destinations.py new file mode 100644 index 0000000..e9e798e --- /dev/null +++ b/apps/api/tests/unit/test_notification_destinations.py @@ -0,0 +1,331 @@ +"""Notification destination CRUD, ownership isolation, and the +company-linking behavior this feature is built around: reusing an existing +destination by (type, value) instead of duplicating it, dedupe display, +and garbage-collecting a destination once no company references it.""" + +from __future__ import annotations + +import uuid + + +def _register_and_login(client) -> dict[str, str]: + email = f"user-{uuid.uuid4().hex[:12]}@example.com" + client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "Test User"}, + ) + tokens = client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ).json() + return {"Authorization": f"Bearer {tokens['access_token']}"} + + +def _create_company(client, headers, name: str | None = None) -> dict: + return client.post( + "/api/v1/companies", + json={"name": name or f"Co {uuid.uuid4().hex[:6]}", "frequency_type": "weekly"}, + headers=headers, + ).json() + + +def test_create_email_destination(client): + headers = _register_and_login(client) + company = _create_company(client, headers) + resp = client.post( + "/api/v1/notification-destinations", + json={ + "type": "email", + "destination_value": "alerts@example.com", + "company_ids": [company["id"]], + }, + headers=headers, + ) + assert resp.status_code == 201 + body = resp.json() + assert body["type"] == "email" + assert body["verified"] is False + assert body["minimum_severity"] == "medium" + assert [c["id"] for c in body["companies"]] == [company["id"]] + + +def test_create_destination_requires_at_least_one_company(client): + headers = _register_and_login(client) + resp = client.post( + "/api/v1/notification-destinations", + json={"type": "email", "destination_value": "alerts@example.com", "company_ids": []}, + headers=headers, + ) + assert resp.status_code == 422 + + +def test_create_destination_rejects_a_company_id_the_user_doesnt_own(client): + owner_headers = _register_and_login(client) + other_headers = _register_and_login(client) + other_company = _create_company(client, other_headers) + + resp = client.post( + "/api/v1/notification-destinations", + json={ + "type": "email", + "destination_value": "alerts@example.com", + "company_ids": [other_company["id"]], + }, + headers=owner_headers, + ) + assert resp.status_code == 400 + + +def test_create_email_destination_rejects_invalid_email(client): + headers = _register_and_login(client) + company = _create_company(client, headers) + resp = client.post( + "/api/v1/notification-destinations", + json={ + "type": "email", + "destination_value": "not-an-email", + "company_ids": [company["id"]], + }, + headers=headers, + ) + assert resp.status_code == 422 + + +def test_create_sms_destination_rejects_invalid_phone(client): + headers = _register_and_login(client) + company = _create_company(client, headers) + resp = client.post( + "/api/v1/notification-destinations", + json={"type": "sms", "destination_value": "not-a-phone", "company_ids": [company["id"]]}, + headers=headers, + ) + assert resp.status_code == 422 + + +def test_create_sms_destination_accepts_e164(client): + headers = _register_and_login(client) + company = _create_company(client, headers) + resp = client.post( + "/api/v1/notification-destinations", + json={ + "type": "sms", + "destination_value": "+15551234567", + "company_ids": [company["id"]], + }, + headers=headers, + ) + assert resp.status_code == 201 + + +def test_reusing_the_same_email_for_a_second_company_links_instead_of_duplicating(client): + """The bug this feature exists to fix: the wizard used to create a brand + new NotificationDestination row per company even when the email was + already registered. Now it must reuse the same row and just add a link.""" + headers = _register_and_login(client) + company_a = _create_company(client, headers, name="Company A") + company_b = _create_company(client, headers, name="Company B") + + first = client.post( + "/api/v1/notification-destinations", + json={ + "type": "email", + "destination_value": "shared@example.com", + "company_ids": [company_a["id"]], + }, + headers=headers, + ).json() + second = client.post( + "/api/v1/notification-destinations", + json={ + "type": "email", + "destination_value": "SHARED@example.com", # different casing on purpose + "company_ids": [company_b["id"]], + }, + headers=headers, + ).json() + + assert first["id"] == second["id"] + listing = client.get("/api/v1/notification-destinations", headers=headers).json() + assert len(listing) == 1 + linked_ids = {c["id"] for c in listing[0]["companies"]} + assert linked_ids == {company_a["id"], company_b["id"]} + + +def test_update_destination_value_resets_verification(client): + headers = _register_and_login(client) + company = _create_company(client, headers) + created = client.post( + "/api/v1/notification-destinations", + json={ + "type": "email", + "destination_value": "alerts@example.com", + "company_ids": [company["id"]], + }, + headers=headers, + ).json() + + resp = client.patch( + f"/api/v1/notification-destinations/{created['id']}", + json={"destination_value": "new-alerts@example.com"}, + headers=headers, + ) + assert resp.status_code == 200 + assert resp.json()["verified"] is False + + +def test_destinations_scoped_to_owner(client): + owner_headers = _register_and_login(client) + other_headers = _register_and_login(client) + company = _create_company(client, owner_headers) + + created = client.post( + "/api/v1/notification-destinations", + json={ + "type": "email", + "destination_value": "alerts@example.com", + "company_ids": [company["id"]], + }, + headers=owner_headers, + ).json() + + resp = client.patch( + f"/api/v1/notification-destinations/{created['id']}", + json={"enabled": False}, + headers=other_headers, + ) + assert resp.status_code == 404 + + +def test_delete_destination(client): + headers = _register_and_login(client) + company = _create_company(client, headers) + created = client.post( + "/api/v1/notification-destinations", + json={ + "type": "email", + "destination_value": "alerts@example.com", + "company_ids": [company["id"]], + }, + headers=headers, + ).json() + + resp = client.delete(f"/api/v1/notification-destinations/{created['id']}", headers=headers) + assert resp.status_code == 204 + + listing = client.get("/api/v1/notification-destinations", headers=headers).json() + assert all(d["id"] != created["id"] for d in listing) + + +def test_deleting_a_companys_only_destination_link_garbage_collects_it(client): + headers = _register_and_login(client) + company = _create_company(client, headers) + client.post( + "/api/v1/notification-destinations", + json={ + "type": "email", + "destination_value": "alerts@example.com", + "company_ids": [company["id"]], + }, + headers=headers, + ) + + resp = client.delete(f"/api/v1/companies/{company['id']}", headers=headers) + assert resp.status_code == 204 + + listing = client.get("/api/v1/notification-destinations", headers=headers).json() + assert listing == [] + + +def test_deleting_one_of_two_linked_companies_keeps_the_destination(client): + headers = _register_and_login(client) + company_a = _create_company(client, headers, name="Keep") + company_b = _create_company(client, headers, name="Delete me") + client.post( + "/api/v1/notification-destinations", + json={ + "type": "email", + "destination_value": "alerts@example.com", + "company_ids": [company_a["id"], company_b["id"]], + }, + headers=headers, + ) + + resp = client.delete(f"/api/v1/companies/{company_b['id']}", headers=headers) + assert resp.status_code == 204 + + listing = client.get("/api/v1/notification-destinations", headers=headers).json() + assert len(listing) == 1 + assert [c["id"] for c in listing[0]["companies"]] == [company_a["id"]] + + +def test_unlink_company_keeps_the_destination_when_other_links_remain(client): + headers = _register_and_login(client) + company_a = _create_company(client, headers, name="Keep") + company_b = _create_company(client, headers, name="Unlink me") + created = client.post( + "/api/v1/notification-destinations", + json={ + "type": "email", + "destination_value": "alerts@example.com", + "company_ids": [company_a["id"], company_b["id"]], + }, + headers=headers, + ).json() + + resp = client.delete( + f"/api/v1/notification-destinations/{created['id']}/companies/{company_b['id']}", + headers=headers, + ) + assert resp.status_code == 204 + + listing = client.get("/api/v1/notification-destinations", headers=headers).json() + assert len(listing) == 1 + assert [c["id"] for c in listing[0]["companies"]] == [company_a["id"]] + + +def test_unlinking_the_last_company_garbage_collects_the_destination(client): + headers = _register_and_login(client) + company = _create_company(client, headers) + created = client.post( + "/api/v1/notification-destinations", + json={ + "type": "email", + "destination_value": "alerts@example.com", + "company_ids": [company["id"]], + }, + headers=headers, + ).json() + + resp = client.delete( + f"/api/v1/notification-destinations/{created['id']}/companies/{company['id']}", + headers=headers, + ) + assert resp.status_code == 204 + + listing = client.get("/api/v1/notification-destinations", headers=headers).json() + assert listing == [] + + +def test_unlink_company_is_scoped_to_the_destinations_owner(client): + owner_headers = _register_and_login(client) + other_headers = _register_and_login(client) + company = _create_company(client, owner_headers) + created = client.post( + "/api/v1/notification-destinations", + json={ + "type": "email", + "destination_value": "alerts@example.com", + "company_ids": [company["id"]], + }, + headers=owner_headers, + ).json() + + resp = client.delete( + f"/api/v1/notification-destinations/{created['id']}/companies/{company['id']}", + headers=other_headers, + ) + assert resp.status_code == 404 + + # The destination and its link both survive the rejected attempt. + listing = client.get("/api/v1/notification-destinations", headers=owner_headers).json() + assert len(listing) == 1 + assert [c["id"] for c in listing[0]["companies"]] == [company["id"]] diff --git a/apps/api/tests/unit/test_notification_destinations_test_endpoint.py b/apps/api/tests/unit/test_notification_destinations_test_endpoint.py new file mode 100644 index 0000000..cbec3f8 --- /dev/null +++ b/apps/api/tests/unit/test_notification_destinations_test_endpoint.py @@ -0,0 +1,121 @@ +"""POST /notification-destinations/{id}/test - dedicated from +test_notification_destinations.py since it exercises actual send-path +dispatch (console/SMTP) rather than just CRUD.""" + +from __future__ import annotations + +import uuid + + +def _register_and_login(client) -> dict[str, str]: + email = f"user-{uuid.uuid4().hex[:12]}@example.com" + client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "Test User"}, + ) + tokens = client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ).json() + return {"Authorization": f"Bearer {tokens['access_token']}"} + + +def _create_company(client, headers) -> dict: + return client.post( + "/api/v1/companies", + json={"name": f"Co {uuid.uuid4().hex[:6]}", "frequency_type": "weekly"}, + headers=headers, + ).json() + + +def test_test_endpoint_sends_to_email_destination(client, monkeypatch): + sent = {} + + class FakeSmtp: + def __init__(self, host, port, timeout=10): + pass + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def starttls(self): + pass + + def login(self, username, password): + pass + + def sendmail(self, from_addr, to_addrs, message): + sent["to_addrs"] = to_addrs + + monkeypatch.setattr("app.notifications.smtp_email.smtplib.SMTP", FakeSmtp) + + headers = _register_and_login(client) + company = _create_company(client, headers) + destination = client.post( + "/api/v1/notification-destinations", + json={ + "type": "email", + "destination_value": "alerts@example.com", + "company_ids": [company["id"]], + }, + headers=headers, + ).json() + + resp = client.post( + f"/api/v1/notification-destinations/{destination['id']}/test", headers=headers + ) + assert resp.status_code == 200 + body = resp.json() + assert body["success"] is True + assert body["error"] is None + assert sent["to_addrs"] == ["alerts@example.com"] + + +def test_test_endpoint_reports_failure(client, monkeypatch): + class FailingSmtp: + def __init__(self, host, port, timeout=10): + raise ConnectionRefusedError("no mailpit running") + + monkeypatch.setattr("app.notifications.smtp_email.smtplib.SMTP", FailingSmtp) + + headers = _register_and_login(client) + company = _create_company(client, headers) + destination = client.post( + "/api/v1/notification-destinations", + json={ + "type": "email", + "destination_value": "alerts@example.com", + "company_ids": [company["id"]], + }, + headers=headers, + ).json() + + resp = client.post( + f"/api/v1/notification-destinations/{destination['id']}/test", headers=headers + ) + assert resp.status_code == 200 + body = resp.json() + assert body["success"] is False + assert "no mailpit running" in body["error"] + + +def test_test_endpoint_not_owned_returns_404(client): + owner_headers = _register_and_login(client) + other_headers = _register_and_login(client) + company = _create_company(client, owner_headers) + destination = client.post( + "/api/v1/notification-destinations", + json={ + "type": "email", + "destination_value": "alerts@example.com", + "company_ids": [company["id"]], + }, + headers=owner_headers, + ).json() + + resp = client.post( + f"/api/v1/notification-destinations/{destination['id']}/test", headers=other_headers + ) + assert resp.status_code == 404 diff --git a/apps/api/tests/unit/test_notification_providers.py b/apps/api/tests/unit/test_notification_providers.py new file mode 100644 index 0000000..0020d48 --- /dev/null +++ b/apps/api/tests/unit/test_notification_providers.py @@ -0,0 +1,256 @@ +"""Notification provider unit tests: console (always succeeds), SMTP (stdlib +smtplib mocked, never touches a real socket), Twilio (respx-mocked REST +calls), the factory's type routing, and the message builder's per-channel +formatting.""" + +from __future__ import annotations + +import uuid + +import httpx +import pytest +import respx + +from app.core.config import Settings +from app.models.alert import Alert +from app.models.company import Company +from app.models.enums import NotificationType, SeverityLevel +from app.notifications.base import NotificationMessage +from app.notifications.console import ConsoleProvider +from app.notifications.factory import get_notification_provider +from app.notifications.message_builder import build_alert_message +from app.notifications.smtp_email import SmtpEmailProvider +from app.notifications.telnyx_sms import TelnyxSmsProvider +from app.notifications.twilio_sms import TwilioSmsProvider + + +@pytest.mark.asyncio +async def test_console_provider_always_succeeds(): + result = await ConsoleProvider().send( + NotificationMessage(destination_value="dev@local", subject="Test", body_text="hi") + ) + assert result.success is True + + +@pytest.mark.asyncio +async def test_smtp_provider_sends_via_smtplib(monkeypatch): + sent = {} + + class FakeSmtp: + def __init__(self, host, port, timeout=10): + sent["host"] = host + sent["port"] = port + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def starttls(self): + sent["starttls"] = True + + def login(self, username, password): + sent["login"] = (username, password) + + def sendmail(self, from_addr, to_addrs, message): + sent["from_addr"] = from_addr + sent["to_addrs"] = to_addrs + sent["message"] = message + + monkeypatch.setattr("app.notifications.smtp_email.smtplib.SMTP", FakeSmtp) + + settings = Settings( + smtp_host="mailpit", smtp_port=1025, smtp_from_email="alerts@ci-agent.local" + ) + provider = SmtpEmailProvider(settings) + result = await provider.send( + NotificationMessage( + destination_value="user@example.com", + subject="Alert: Pricing change", + body_text="Plain text body", + body_html="

HTML body

", + ) + ) + + assert result.success is True + assert sent["host"] == "mailpit" + assert sent["to_addrs"] == ["user@example.com"] + assert "starttls" not in sent # smtp_use_tls defaults False + + +@pytest.mark.asyncio +async def test_smtp_provider_reports_failure_without_raising(monkeypatch): + class FailingSmtp: + def __init__(self, host, port, timeout=10): + raise ConnectionRefusedError("no mailpit running") + + monkeypatch.setattr("app.notifications.smtp_email.smtplib.SMTP", FailingSmtp) + + provider = SmtpEmailProvider(Settings()) + result = await provider.send( + NotificationMessage(destination_value="user@example.com", subject="s", body_text="b") + ) + + assert result.success is False + assert "no mailpit running" in result.error + + +@pytest.mark.asyncio +async def test_twilio_provider_not_configured_fails_cleanly(): + settings = Settings(twilio_account_sid="", twilio_auth_token="", twilio_from_number="") + result = await TwilioSmsProvider(settings).send( + NotificationMessage(destination_value="+15551234567", subject="s", body_text="b") + ) + assert result.success is False + assert "not configured" in result.error + + +@pytest.mark.asyncio +async def test_twilio_provider_sends_via_rest_api(): + settings = Settings( + twilio_account_sid="ACxxxx", twilio_auth_token="secret", twilio_from_number="+15559990000" + ) + with respx.mock: + respx.post("https://api.twilio.com/2010-04-01/Accounts/ACxxxx/Messages.json").mock( + return_value=httpx.Response(201, json={"sid": "SMxxxxx"}) + ) + result = await TwilioSmsProvider(settings).send( + NotificationMessage( + destination_value="+15551234567", subject="s", body_text="Alert text" + ) + ) + + assert result.success is True + assert result.external_message_id == "SMxxxxx" + + +@pytest.mark.asyncio +async def test_twilio_provider_surfaces_api_error(): + settings = Settings( + twilio_account_sid="ACxxxx", twilio_auth_token="secret", twilio_from_number="+15559990000" + ) + with respx.mock: + respx.post("https://api.twilio.com/2010-04-01/Accounts/ACxxxx/Messages.json").mock( + return_value=httpx.Response(400, text="Invalid 'To' number") + ) + result = await TwilioSmsProvider(settings).send( + NotificationMessage(destination_value="not-a-number", subject="s", body_text="b") + ) + + assert result.success is False + assert "400" in result.error + + +@pytest.mark.asyncio +async def test_telnyx_provider_not_configured_fails_cleanly(): + settings = Settings(telnyx_api_key="", telnyx_from_number="") + result = await TelnyxSmsProvider(settings).send( + NotificationMessage(destination_value="+15551234567", subject="s", body_text="b") + ) + assert result.success is False + assert "not configured" in result.error + + +@pytest.mark.asyncio +async def test_telnyx_provider_sends_via_rest_api(): + settings = Settings(telnyx_api_key="KEYxxxx", telnyx_from_number="+15559990000") + with respx.mock: + respx.post("https://api.telnyx.com/v2/messages").mock( + return_value=httpx.Response(200, json={"data": {"id": "msg-abc123"}}) + ) + result = await TelnyxSmsProvider(settings).send( + NotificationMessage( + destination_value="+15551234567", subject="s", body_text="Alert text" + ) + ) + + assert result.success is True + assert result.external_message_id == "msg-abc123" + + +@pytest.mark.asyncio +async def test_telnyx_provider_surfaces_api_error(): + settings = Settings(telnyx_api_key="KEYxxxx", telnyx_from_number="+15559990000") + with respx.mock: + respx.post("https://api.telnyx.com/v2/messages").mock( + return_value=httpx.Response( + 403, + json={ + "errors": [ + { + "code": "40300", + "title": "Forbidden", + "detail": "The from number is not assigned to a messaging profile.", + } + ] + }, + ) + ) + result = await TelnyxSmsProvider(settings).send( + NotificationMessage(destination_value="+15551234567", subject="s", body_text="b") + ) + + assert result.success is False + assert "403" in result.error + assert "messaging profile" in result.error + + +def test_factory_routes_by_notification_type(): + settings = Settings() + assert get_notification_provider(NotificationType.EMAIL, settings).provider_name == "smtp" + assert get_notification_provider(NotificationType.SMS, settings).provider_name == "twilio_sms" + assert get_notification_provider(NotificationType.CONSOLE, settings).provider_name == "console" + + +def test_factory_routes_sms_by_sms_provider_setting(): + twilio_settings = Settings(sms_provider="twilio") + assert ( + get_notification_provider(NotificationType.SMS, twilio_settings).provider_name + == "twilio_sms" + ) + + telnyx_settings = Settings(sms_provider="telnyx") + assert ( + get_notification_provider(NotificationType.SMS, telnyx_settings).provider_name + == "telnyx_sms" + ) + + +def _sample_alert() -> Alert: + return Alert( + id=uuid.uuid4(), + company_id=uuid.uuid4(), + detected_change_id=uuid.uuid4(), + user_id=uuid.uuid4(), + title="New leadership hire announced", + summary="A new VP of Engineering was announced on the careers page.", + why_it_matters="Signals a scaling push in engineering.", + severity=SeverityLevel.HIGH, + confidence=0.82, + ) + + +def test_message_builder_sms_is_short_and_truncated(): + company = Company(id=uuid.uuid4(), user_id=uuid.uuid4(), name="Acme Mobility", slug="acme") + message = build_alert_message( + NotificationType.SMS, "+15551234567", company, _sample_alert(), Settings() + ) + assert len(message.body_text) <= 480 + assert "HIGH" in message.body_text + assert message.body_html is None + + +def test_message_builder_email_includes_context_and_links(): + company = Company(id=uuid.uuid4(), user_id=uuid.uuid4(), name="Acme Mobility", slug="acme") + alert = _sample_alert() + message = build_alert_message( + NotificationType.EMAIL, "user@example.com", company, alert, Settings() + ) + assert "Acme Mobility" in message.subject + assert alert.summary in message.body_text + assert alert.why_it_matters in message.body_text + assert "/alerts" in message.body_text + assert "/settings" in message.body_text + assert message.body_html is not None + assert alert.summary in message.body_html diff --git a/apps/api/tests/unit/test_rate_limit.py b/apps/api/tests/unit/test_rate_limit.py new file mode 100644 index 0000000..9399d8c --- /dev/null +++ b/apps/api/tests/unit/test_rate_limit.py @@ -0,0 +1,89 @@ +"""The suite disables the rate limiter globally (see conftest.py) so the many +auth calls other tests make don't trip real limits. This test re-enables it +temporarily to verify the limiter itself actually works.""" + +from __future__ import annotations + +import uuid + +from fastapi import Request + +from app.core.config import get_settings +from app.core.rate_limit import _client_ip_key, limiter + + +def _build_request(headers: dict[str, str], client_ip: str) -> Request: + scope = { + "type": "http", + "headers": [(k.lower().encode(), v.encode()) for k, v in headers.items()], + "client": (client_ip, 12345), + } + return Request(scope) + + +def test_client_ip_key_uses_direct_peer_when_no_proxy_header_configured(): + # Default test settings have trusted_proxy_ip_header="" - the header + # must be ignored even if a client sends it, or a spoofed header could + # forge any rate-limit identity with no proxy actually in front. + request = _build_request({"CF-Connecting-IP": "203.0.113.5"}, client_ip="10.0.0.9") + assert _client_ip_key(request) == "10.0.0.9" + + +def test_client_ip_key_honors_configured_proxy_header(monkeypatch): + # Once deployed behind Cloudflare, trusted_proxy_ip_header="CF-Connecting-IP" + # must make the limiter key off the real visitor, not Nginx's own address - + # this is the exact bug slowapi's default get_remote_address had. + settings = get_settings().model_copy(update={"trusted_proxy_ip_header": "CF-Connecting-IP"}) + monkeypatch.setattr("app.core.rate_limit.get_settings", lambda: settings) + + request = _build_request({"CF-Connecting-IP": "203.0.113.5"}, client_ip="10.0.0.9") + assert _client_ip_key(request) == "203.0.113.5" + + +def test_register_endpoint_enforces_rate_limit(client): + limiter.enabled = True + try: + responses = [ + client.post( + "/api/v1/auth/register", + json={ + "email": f"rl-{uuid.uuid4().hex[:10]}@example.com", + "password": "correct-horse-1", + "display_name": "Rate Limit Test", + }, + ) + for _ in range(6) + ] + finally: + limiter.enabled = False + + statuses = [r.status_code for r in responses] + assert 429 in statuses, f"Expected a 429 among {statuses} after 6 rapid registrations" + + +def test_create_company_enforces_rate_limit(client): + email = f"rl-company-{uuid.uuid4().hex[:10]}@example.com" + client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "RL Test"}, + ) + tokens = client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ).json() + headers = {"Authorization": f"Bearer {tokens['access_token']}"} + + limiter.enabled = True + try: + responses = [ + client.post( + "/api/v1/companies", + json={"name": f"RL Co {i}", "frequency_type": "weekly"}, + headers=headers, + ) + for i in range(22) + ] + finally: + limiter.enabled = False + + statuses = [r.status_code for r in responses] + assert 429 in statuses, f"Expected a 429 among {statuses} after 22 rapid company creations" diff --git a/apps/api/tests/unit/test_report_generation_prompt.py b/apps/api/tests/unit/test_report_generation_prompt.py new file mode 100644 index 0000000..8b7daee --- /dev/null +++ b/apps/api/tests/unit/test_report_generation_prompt.py @@ -0,0 +1,69 @@ +"""generate_report(): the company_enrichment evidence block reaches the +prompt exactly like company_profile does, and is an honest empty dict +when no enrichment data is available - never fabricated.""" + +from __future__ import annotations + +import pytest + +from app.prompts.base import extract_evidence_block +from app.prompts.report_generation import ReportContent, SwotAnalysis, generate_report + + +class _CapturingLLMProvider: + provider_name = "capturing" + + def __init__(self) -> None: + self.last_user_prompt: str | None = None + + async def generate_structured(self, system_prompt, user_prompt, response_model): + self.last_user_prompt = user_prompt + return ReportContent( + executive_summary="", + company_overview="", + market_positioning="", + customer_sentiment="", + competitor_comparison="", + swot=SwotAnalysis(), + methodology="", + limitations="", + ) + + async def generate_text(self, system_prompt, user_prompt) -> str: + return "" + + +async def _generate(llm: _CapturingLLMProvider, **kwargs) -> None: + await generate_report( + llm, + company_name="Acme Corp", + company_aliases=[], + competitors=[], + monitoring_focus=None, + industry=None, + documents=[], + detected_changes=[], + sources_failed=[], + **kwargs, + ) + + +@pytest.mark.asyncio +async def test_enrichment_data_reaches_the_prompt_as_a_named_evidence_block(): + llm = _CapturingLLMProvider() + await _generate( + llm, enrichment={"employee_count": "1001-5000", "funding": {"total_raised": "$1M"}} + ) + + evidence = extract_evidence_block(llm.last_user_prompt) + assert evidence["company_enrichment"]["employee_count"] == "1001-5000" + assert evidence["company_enrichment"]["funding"]["total_raised"] == "$1M" + + +@pytest.mark.asyncio +async def test_no_enrichment_data_is_an_honest_empty_block_not_fabricated(): + llm = _CapturingLLMProvider() + await _generate(llm) # enrichment defaults to None + + evidence = extract_evidence_block(llm.last_user_prompt) + assert evidence["company_enrichment"] == {} diff --git a/apps/api/tests/unit/test_reports_api.py b/apps/api/tests/unit/test_reports_api.py new file mode 100644 index 0000000..d3a9c42 --- /dev/null +++ b/apps/api/tests/unit/test_reports_api.py @@ -0,0 +1,91 @@ +"""Reports API: ownership isolation, manual generation, and the raw +markdown/json export endpoints.""" + +from __future__ import annotations + +import uuid + + +def _register_and_login(client) -> dict[str, str]: + email = f"user-{uuid.uuid4().hex[:12]}@example.com" + client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "Test User"}, + ) + tokens = client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ).json() + return {"Authorization": f"Bearer {tokens['access_token']}"} + + +def _create_company(client, headers): + return client.post( + "/api/v1/companies", + json={"name": f"Report Co {uuid.uuid4().hex[:6]}", "frequency_type": "weekly"}, + headers=headers, + ).json() + + +def test_generate_report_creates_and_returns_report(client): + headers = _register_and_login(client) + company = _create_company(client, headers) + + resp = client.post(f"/api/v1/companies/{company['id']}/reports/generate", headers=headers) + assert resp.status_code == 201 + body = resp.json() + assert body["report_type"] == "manual" + assert body["model_provider"] == "mock" + assert company["name"] in body["executive_summary"] + + +def test_list_reports_and_get_detail(client): + headers = _register_and_login(client) + company = _create_company(client, headers) + created = client.post( + f"/api/v1/companies/{company['id']}/reports/generate", headers=headers + ).json() + + listing = client.get(f"/api/v1/companies/{company['id']}/reports", headers=headers).json() + assert any(r["id"] == created["id"] for r in listing) + + detail = client.get(f"/api/v1/reports/{created['id']}", headers=headers) + assert detail.status_code == 200 + assert detail.json()["structured_report"]["executive_summary"] + + +def test_report_markdown_and_json_export(client): + headers = _register_and_login(client) + company = _create_company(client, headers) + created = client.post( + f"/api/v1/companies/{company['id']}/reports/generate", headers=headers + ).json() + + md_resp = client.get(f"/api/v1/reports/{created['id']}/markdown", headers=headers) + assert md_resp.status_code == 200 + assert md_resp.headers["content-type"].startswith("text/markdown") + assert "# Competitive Intelligence Report" in md_resp.text + + json_resp = client.get(f"/api/v1/reports/{created['id']}/json", headers=headers) + assert json_resp.status_code == 200 + assert "executive_summary" in json_resp.json() + + +def test_reports_scoped_to_owner(client): + owner_headers = _register_and_login(client) + other_headers = _register_and_login(client) + company = _create_company(client, owner_headers) + created = client.post( + f"/api/v1/companies/{company['id']}/reports/generate", headers=owner_headers + ).json() + + assert ( + client.get(f"/api/v1/companies/{company['id']}/reports", headers=other_headers).status_code + == 404 + ) + assert client.get(f"/api/v1/reports/{created['id']}", headers=other_headers).status_code == 404 + assert ( + client.post( + f"/api/v1/companies/{company['id']}/reports/generate", headers=other_headers + ).status_code + == 404 + ) diff --git a/apps/api/tests/unit/test_scoring.py b/apps/api/tests/unit/test_scoring.py new file mode 100644 index 0000000..4659934 --- /dev/null +++ b/apps/api/tests/unit/test_scoring.py @@ -0,0 +1,130 @@ +"""Tests for the documented significance/confidence/severity formula in +app/change_detection/scoring.py (see ARCHITECTURE.md).""" + +from __future__ import annotations + +from app.change_detection.scoring import classify_severity, compute_confidence, compute_significance +from app.models.enums import ChangeType, SeverityLevel + + +def test_significance_scales_with_source_trust(): + high_trust = compute_significance( + change_type=ChangeType.NEW_DOCUMENT, source_trust_score=1.0, independent_source_count=2 + ) + low_trust = compute_significance( + change_type=ChangeType.NEW_DOCUMENT, source_trust_score=0.3, independent_source_count=2 + ) + assert high_trust > low_trust + + +def test_significance_uncorroborated_signal_is_halved(): + corroborated = compute_significance( + change_type=ChangeType.NEW_DOCUMENT, source_trust_score=1.0, independent_source_count=2 + ) + single_source = compute_significance( + change_type=ChangeType.NEW_DOCUMENT, source_trust_score=1.0, independent_source_count=1 + ) + assert single_source == round(corroborated / 2, 4) + + +def test_significance_focus_match_boosts_score(): + matched = compute_significance( + change_type=ChangeType.NEW_DOCUMENT, + source_trust_score=1.0, + independent_source_count=2, + focus_match=True, + ) + unmatched = compute_significance( + change_type=ChangeType.NEW_DOCUMENT, + source_trust_score=1.0, + independent_source_count=2, + focus_match=False, + ) + assert matched > unmatched + + +def test_significance_repeat_change_is_dampened(): + fresh = compute_significance( + change_type=ChangeType.NEW_DOCUMENT, source_trust_score=1.0, is_repeat=False + ) + repeat = compute_significance( + change_type=ChangeType.NEW_DOCUMENT, source_trust_score=1.0, is_repeat=True + ) + assert repeat == round(fresh / 2, 4) + + +def test_significance_content_modified_scales_with_diff_ratio(): + small_edit = compute_significance( + change_type=ChangeType.CONTENT_MODIFIED, + source_trust_score=1.0, + independent_source_count=2, + diff_ratio=0.05, + ) + big_rewrite = compute_significance( + change_type=ChangeType.CONTENT_MODIFIED, + source_trust_score=1.0, + independent_source_count=2, + diff_ratio=0.9, + ) + assert big_rewrite > small_edit + + +def test_significance_never_exceeds_one(): + value = compute_significance( + change_type=ChangeType.LEADERSHIP_CHANGE, + source_trust_score=1.0, + independent_source_count=10, + focus_match=True, + ) + assert value <= 1.0 + + +def test_confidence_increases_with_corroboration(): + single = compute_confidence( + extraction_confidence=0.8, source_trust_score=0.8, independent_source_count=1 + ) + corroborated = compute_confidence( + extraction_confidence=0.8, source_trust_score=0.8, independent_source_count=2 + ) + assert corroborated > single + + +def test_confidence_bounded_between_zero_and_one(): + assert ( + 0.0 + <= compute_confidence( + extraction_confidence=0.0, source_trust_score=0.0, independent_source_count=1 + ) + <= 1.0 + ) + assert ( + 0.0 + <= compute_confidence( + extraction_confidence=1.0, source_trust_score=1.0, independent_source_count=5 + ) + <= 1.0 + ) + + +def test_classify_severity_buckets_by_score(): + assert classify_severity(significance=0.9, confidence=0.9) == SeverityLevel.CRITICAL + assert classify_severity(significance=0.6, confidence=0.9) == SeverityLevel.HIGH + assert classify_severity(significance=0.3, confidence=0.9) == SeverityLevel.MEDIUM + assert classify_severity(significance=0.1, confidence=0.9) == SeverityLevel.LOW + + +def test_classify_severity_critical_requires_high_confidence(): + """A score that would otherwise land in the Critical bucket (>= 0.6) + must be downgraded to High when confidence is below the floor - an + uncorroborated single-source signal can't carry the Critical label.""" + score = 1.0 * 0.65 + assert score >= 0.6 # would be CRITICAL by score alone + severity = classify_severity(significance=1.0, confidence=0.65) + assert severity == SeverityLevel.HIGH + + +def test_classify_severity_high_confidence_allows_critical(): + score = 1.0 * 0.8 + assert score >= 0.6 + severity = classify_severity(significance=1.0, confidence=0.8) + assert severity == SeverityLevel.CRITICAL diff --git a/apps/api/tests/unit/test_search_providers.py b/apps/api/tests/unit/test_search_providers.py new file mode 100644 index 0000000..6ba08a2 --- /dev/null +++ b/apps/api/tests/unit/test_search_providers.py @@ -0,0 +1,78 @@ +"""SearchProvider: Mock (deterministic, honest about having no real +evidence) and Brave (respx-mocked REST call), plus the factory's routing.""" + +from __future__ import annotations + +import httpx +import pytest +import respx + +from app.core.config import Settings +from app.search.factory import get_search_provider +from app.search.mock import MockSearchProvider + + +@pytest.mark.asyncio +async def test_mock_provider_guesses_a_domain_for_official_website_queries(): + provider = MockSearchProvider() + results = await provider.search("Acme Mobility official website") + + assert len(results) == 1 + assert results[0].url == "https://acmemobility.com" + assert "Acme Mobility" in results[0].title + + +@pytest.mark.asyncio +async def test_mock_provider_is_honest_about_no_evidence_for_other_queries(): + provider = MockSearchProvider() + results = await provider.search("Acme Mobility competitors") + + assert len(results) == 1 + assert "no information available" in results[0].snippet.lower() + + +@pytest.mark.asyncio +async def test_mock_provider_respects_count(): + provider = MockSearchProvider() + results = await provider.search("Acme official website", count=0) + assert results == [] + + +@pytest.mark.asyncio +async def test_brave_provider_maps_api_response_to_search_results(): + from app.search.brave import BraveSearchProvider + + settings = Settings(search_provider="brave", brave_search_api_key="test-key") + provider = BraveSearchProvider(settings) + + with respx.mock: + route = respx.get("https://api.search.brave.com/res/v1/web/search").mock( + return_value=httpx.Response( + 200, + json={ + "web": { + "results": [ + { + "title": "Acme Mobility - Official Site", + "url": "https://acmemobility.com", + "description": "Acme Mobility builds electric scooters.", + } + ] + } + }, + ) + ) + results = await provider.search("Acme Mobility official website", count=3) + + assert route.calls.last.request.headers["X-Subscription-Token"] == "test-key" + assert len(results) == 1 + assert results[0].url == "https://acmemobility.com" + assert results[0].snippet == "Acme Mobility builds electric scooters." + + +def test_factory_routes_by_search_provider_setting(): + mock_provider = get_search_provider(Settings(search_provider="mock")) + assert mock_provider.provider_name == "mock" + + brave_provider = get_search_provider(Settings(search_provider="brave")) + assert brave_provider.provider_name == "brave" diff --git a/apps/api/tests/unit/test_security_flows.py b/apps/api/tests/unit/test_security_flows.py new file mode 100644 index 0000000..5b3b364 --- /dev/null +++ b/apps/api/tests/unit/test_security_flows.py @@ -0,0 +1,654 @@ +"""End-to-end HTTP + service-level tests for Phase 19: email verification, +password reset, account lockout, and Turnstile enforcement. + +Each test that exercises the IP-throttle-gated endpoints uses its own +synthetic client IP (`_unique_ip` + a fresh `TestClient(app, client=(ip, +...))`), rather than the shared `client` fixture's fake "testclient" peer - +sharing that IP across tests previously caused a real bug (register() +exhausting the resend-verification ladder and permanently banning +"testclient", see auth_service.register's docstring) and every test here +would risk reintroducing the same class of collision if it shared IPs. + +The login-lockout ladder is walked at the service level (like +test_ip_throttle_service.py) with `ip_throttle_service._now` monkeypatched +forward, so 12 escalating attempts take milliseconds of real test time +instead of ~30 real minutes. +""" + +from __future__ import annotations + +import re +import uuid +from datetime import UTC, datetime, timedelta + +import httpx +import pytest +import respx +from fastapi.testclient import TestClient + +from app.core.config import get_settings +from app.core.errors import AuthenticationError +from app.db.base import ensure_aware_utc +from app.main import app +from app.models.enums import ThrottleAction +from app.notifications.base import DeliveryResult +from app.repositories.ip_throttle_repository import IpThrottleRepository +from app.repositories.user_repository import UserRepository +from app.schemas.auth import LoginRequest, RegisterRequest +from app.services import auth_service, ip_throttle_service +from app.services.turnstile_service import verify_turnstile + + +def _unique_email() -> str: + return f"user-{uuid.uuid4().hex[:12]}@example.com" + + +def _unique_ip() -> str: + # Randomize all three trailing octets (same convention as + # test_ip_throttle_service.py) - a single-octet range only has ~250 + # values, which collides often enough across a full suite run (birthday + # paradox) to cause real, intermittent failures between unrelated tests + # that happen to share ip_throttle_state rows. + return f"10.{uuid.uuid4().int % 250}.{uuid.uuid4().int % 250}.{uuid.uuid4().int % 250}" + + +def _install_fake_smtp(monkeypatch) -> list[dict]: + sent: list[dict] = [] + + class FakeSmtp: + def __init__(self, host, port, timeout=10): + pass + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def starttls(self): + pass + + def login(self, username, password): + pass + + def sendmail(self, from_addr, to_addrs, message): + sent.append({"to": to_addrs, "message": message}) + + monkeypatch.setattr("app.notifications.smtp_email.smtplib.SMTP", FakeSmtp) + return sent + + +def _extract_code(sent_emails: list[dict]) -> str: + message = sent_emails[-1]["message"] + match = re.search(r"code is[^\d]*(\d{6})", message) + assert match, f"no 6-digit code found in most recent sent email: {message!r}" + return match.group(1) + + +# --- Email verification -------------------------------------------------- + + +def test_verify_email_wrong_code_is_generic_failure(client: TestClient): + resp = client.post( + "/api/v1/auth/verify-email", json={"email": _unique_email(), "code": "000000"} + ) + assert resp.status_code == 401 + + +def test_verify_email_code_guessing_throttled_after_five_attempts(monkeypatch): + """A 6-digit code has only 1M possible values - without this, an + attacker could brute-force it well within its 36h validity window.""" + _install_fake_smtp(monkeypatch) + ip = _unique_ip() + with TestClient(app, client=(ip, 51234)) as c: + email = _unique_email() + c.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "T"}, + ) + for _ in range(5): + resp = c.post("/api/v1/auth/verify-email", json={"email": email, "code": "000000"}) + assert resp.status_code == 401 + + throttled = c.post("/api/v1/auth/verify-email", json={"email": email, "code": "000000"}) + assert throttled.status_code == 429 + # See the resend-verification test above for why this tolerates a + # 1-second real-clock rounding jitter. + assert throttled.json()["retry_after_seconds"] in (4, 5) + + +def test_confirm_password_reset_code_guessing_throttled_after_five_attempts(monkeypatch): + _install_fake_smtp(monkeypatch) + ip = _unique_ip() + with TestClient(app, client=(ip, 51234)) as c: + email = _unique_email() + c.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "T"}, + ) + c.post("/api/v1/auth/request-password-reset", json={"email": email}) + + for _ in range(5): + resp = c.post( + "/api/v1/auth/confirm-password-reset", + json={"email": email, "code": "000000", "new_password": "new-horse-2"}, + ) + assert resp.status_code == 401 + + throttled = c.post( + "/api/v1/auth/confirm-password-reset", + json={"email": email, "code": "000000", "new_password": "new-horse-2"}, + ) + assert throttled.status_code == 429 + assert throttled.json()["retry_after_seconds"] in (4, 5) + + +async def test_login_blocked_until_verified_then_succeeds_after_verify_email(monkeypatch): + sent = _install_fake_smtp(monkeypatch) + ip = _unique_ip() + with TestClient(app, client=(ip, 51234)) as c: + email = _unique_email() + c.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "T"}, + ) + # Test env auto-verifies on register (no real inbox to read from) - + # flip it back off directly to exercise the actual gate. + from app.db.session import get_sessionmaker + + session_factory = get_sessionmaker() + async with session_factory() as db: + user = await UserRepository(db).get_by_email(email) + user.email_verified = False + await db.commit() + + blocked = c.post("/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"}) + assert blocked.status_code == 401 + assert "verify" in blocked.json()["detail"].lower() + + code = _extract_code(sent) + verify_resp = c.post("/api/v1/auth/verify-email", json={"email": email, "code": code}) + assert verify_resp.status_code == 204 + + allowed = c.post("/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"}) + assert allowed.status_code == 200 + + +async def test_verify_email_old_code_invalidated_by_resend(monkeypatch): + """A resend must fully supersede the prior code, not just make it + harder to guess - the old one must stop working entirely.""" + sent = _install_fake_smtp(monkeypatch) + ip = _unique_ip() + with TestClient(app, client=(ip, 51234)) as c: + email = _unique_email() + c.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "T"}, + ) + old_code = _extract_code(sent) + + # Test env auto-verifies on register, which would make resend() a + # no-op (it only resends for a not-yet-verified account) - flip it + # back off directly so a genuine second code actually gets issued. + from app.db.session import get_sessionmaker + + session_factory = get_sessionmaker() + async with session_factory() as db: + user = await UserRepository(db).get_by_email(email) + user.email_verified = False + await db.commit() + + c.post("/api/v1/auth/resend-verification", json={"email": email}) + new_code = _extract_code(sent) + + stale = c.post("/api/v1/auth/verify-email", json={"email": email, "code": old_code}) + assert stale.status_code == 401 + + fresh = c.post("/api/v1/auth/verify-email", json={"email": email, "code": new_code}) + assert fresh.status_code == 204 + + +def test_resend_verification_throttled_immediately_after_first_click(monkeypatch): + _install_fake_smtp(monkeypatch) + ip = _unique_ip() + with TestClient(app, client=(ip, 51234)) as c: + email = _unique_email() + c.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "T"}, + ) + first = c.post("/api/v1/auth/resend-verification", json={"email": email}) + assert first.status_code == 204 + + second = c.post("/api/v1/auth/resend-verification", json={"email": email}) + assert second.status_code == 429 + # retry_after_seconds is int((next_allowed_at - now).total_seconds()), + # computed against the real clock here - a few ms of real elapsed + # time between the two HTTP calls can round it down from 30 to 29. + retry_after = second.json()["retry_after_seconds"] + assert retry_after in (29, 30) + assert second.headers["retry-after"] == str(retry_after) + + +def test_resend_verification_unknown_email_is_generic_success(monkeypatch): + sent = _install_fake_smtp(monkeypatch) + ip = _unique_ip() + with TestClient(app, client=(ip, 51234)) as c: + resp = c.post("/api/v1/auth/resend-verification", json={"email": _unique_email()}) + assert resp.status_code == 204 + assert sent == [] # no account -> nothing actually sent, but no enumeration signal either + + +# --- Password reset -------------------------------------------------------- + + +def test_password_reset_old_code_invalidated_by_new_request(monkeypatch): + sent = _install_fake_smtp(monkeypatch) + email = _unique_email() + ip1, ip2 = _unique_ip(), _unique_ip() + + with TestClient(app, client=(ip1, 51234)) as c1: + c1.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "T"}, + ) + c1.post("/api/v1/auth/request-password-reset", json={"email": email}) + old_code = _extract_code(sent) + + with TestClient(app, client=(ip2, 51235)) as c2: + c2.post("/api/v1/auth/request-password-reset", json={"email": email}) + new_code = _extract_code(sent) + + stale = c2.post( + "/api/v1/auth/confirm-password-reset", + json={"email": email, "code": old_code, "new_password": "new-horse-2"}, + ) + assert stale.status_code == 401 + + fresh = c2.post( + "/api/v1/auth/confirm-password-reset", + json={"email": email, "code": new_code, "new_password": "new-horse-2"}, + ) + assert fresh.status_code == 204 + + +def test_password_reset_rejects_reusing_current_password(monkeypatch): + sent = _install_fake_smtp(monkeypatch) + ip = _unique_ip() + with TestClient(app, client=(ip, 51234)) as c: + email = _unique_email() + c.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "T"}, + ) + c.post("/api/v1/auth/request-password-reset", json={"email": email}) + code = _extract_code(sent) + + rejected = c.post( + "/api/v1/auth/confirm-password-reset", + json={"email": email, "code": code, "new_password": "correct-horse-1"}, + ) + assert rejected.status_code == 400 + assert "used this password before" in rejected.json()["detail"].lower() + + # Rejection must not consume the code - it still works with a + # genuinely different password right after. + retry = c.post( + "/api/v1/auth/confirm-password-reset", + json={"email": email, "code": code, "new_password": "different-horse-9"}, + ) + assert retry.status_code == 204 + + +def test_password_reset_rejects_reusing_a_previous_not_just_current_password(monkeypatch): + sent = _install_fake_smtp(monkeypatch) + email = _unique_email() + ip1, ip2 = _unique_ip(), _unique_ip() + + with TestClient(app, client=(ip1, 51234)) as c1: + c1.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "T"}, + ) + c1.post("/api/v1/auth/request-password-reset", json={"email": email}) + code1 = _extract_code(sent) + first = c1.post( + "/api/v1/auth/confirm-password-reset", + json={"email": email, "code": code1, "new_password": "new-horse-2"}, + ) + assert first.status_code == 204 + + with TestClient(app, client=(ip2, 51235)) as c2: + c2.post("/api/v1/auth/request-password-reset", json={"email": email}) + code2 = _extract_code(sent) + + # correct-horse-1 is no longer the current password, but it's still + # in this account's history - must still be rejected. + rejected = c2.post( + "/api/v1/auth/confirm-password-reset", + json={"email": email, "code": code2, "new_password": "correct-horse-1"}, + ) + assert rejected.status_code == 400 + + accepted = c2.post( + "/api/v1/auth/confirm-password-reset", + json={"email": email, "code": code2, "new_password": "third-horse-3"}, + ) + assert accepted.status_code == 204 + + +def test_password_reset_full_round_trip_then_old_sessions_revoked(monkeypatch): + sent = _install_fake_smtp(monkeypatch) + ip = _unique_ip() + with TestClient(app, client=(ip, 51234)) as c: + email = _unique_email() + c.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "T"}, + ) + tokens = c.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ).json() + + reset_req = c.post("/api/v1/auth/request-password-reset", json={"email": email}) + assert reset_req.status_code == 204 + + code = _extract_code(sent) + confirm = c.post( + "/api/v1/auth/confirm-password-reset", + json={"email": email, "code": code, "new_password": "new-horse-2"}, + ) + assert confirm.status_code == 204 + + old_password_login = c.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ) + assert old_password_login.status_code == 401 + + new_password_login = c.post( + "/api/v1/auth/login", json={"email": email, "password": "new-horse-2"} + ) + assert new_password_login.status_code == 200 + + # A reset invalidates sessions that existed before it. + stale_refresh = c.post( + "/api/v1/auth/refresh", json={"refresh_token": tokens["refresh_token"]} + ) + assert stale_refresh.status_code == 401 + + +def test_request_password_reset_unknown_email_is_generic_success(monkeypatch): + sent = _install_fake_smtp(monkeypatch) + ip = _unique_ip() + with TestClient(app, client=(ip, 51234)) as c: + resp = c.post("/api/v1/auth/request-password-reset", json={"email": _unique_email()}) + assert resp.status_code == 204 + assert sent == [] + + +def test_confirm_password_reset_wrong_code_is_generic_failure(client: TestClient): + resp = client.post( + "/api/v1/auth/confirm-password-reset", + json={"email": _unique_email(), "code": "000000", "new_password": "new-horse-2"}, + ) + assert resp.status_code == 401 + + +# --- Account lockout (service-level, clock-walked) ------------------------- + + +class _Clock: + def __init__(self) -> None: + self.now = datetime(2026, 1, 1, tzinfo=UTC) + + def __call__(self) -> datetime: + return self.now + + def advance(self, seconds: float) -> None: + self.now += timedelta(seconds=seconds) + + +@pytest.fixture() +def clock(monkeypatch): + fake_clock = _Clock() + monkeypatch.setattr(ip_throttle_service, "_now", fake_clock) + return fake_clock + + +async def _advance_past_login_throttle(db_session, clock: _Clock, ip: str) -> None: + peeked = await ip_throttle_service.peek_throttle(db_session, ip, ThrottleAction.FAILED_LOGIN) + if not peeked.allowed and peeked.retry_after_seconds: + clock.advance(peeked.retry_after_seconds) + elif not peeked.allowed: + # Banned outright (no retry_after) - shouldn't happen within this + # test's 12-attempt ladder, but fail loudly if it ever does. + state = await IpThrottleRepository(db_session).get_state(ip, ThrottleAction.FAILED_LOGIN) + if state and state.timeout_until is not None: + clock.advance((ensure_aware_utc(state.timeout_until) - clock.now).total_seconds() + 1) + + +async def test_login_lockout_after_twelve_failed_attempts_locks_account_and_notifies( + db_session, clock, monkeypatch +): + settings = get_settings() + ip = _unique_ip() + email = _unique_email() + + user = await auth_service.register( + db_session, + settings, + ip, + RegisterRequest(email=email, password="correct-horse-1", display_name="T"), + ) + assert user.email_verified is True # app_env == "test" auto-verify precedent + + locked_emails: list[str] = [] + + async def fake_send_locked(_settings, to): + locked_emails.append(to) + return DeliveryResult(success=True) + + monkeypatch.setattr( + auth_service.security_email_service, "send_account_locked_email", fake_send_locked + ) + + wrong_login = LoginRequest(email=email, password="wrong-password-1") + for _ in range(12): + await _advance_past_login_throttle(db_session, clock, ip) + with pytest.raises(AuthenticationError): + await auth_service.login(db_session, settings, ip, wrong_login) + + refreshed = await UserRepository(db_session).get_by_id(user.id) + assert refreshed.failed_login_count == 12 + assert refreshed.locked_at is not None + assert locked_emails == [email] + + # The account stays locked even with the *correct* password, and even + # once the IP itself is no longer throttled. + await _advance_past_login_throttle(db_session, clock, ip) + correct_login = LoginRequest(email=email, password="correct-horse-1") + with pytest.raises(AuthenticationError, match="locked"): + await auth_service.login(db_session, settings, ip, correct_login) + + +async def test_login_correct_password_resets_failed_count_before_lockout(db_session, clock): + settings = get_settings() + ip = _unique_ip() + email = _unique_email() + + user = await auth_service.register( + db_session, + settings, + ip, + RegisterRequest(email=email, password="correct-horse-1", display_name="T"), + ) + + wrong_login = LoginRequest(email=email, password="wrong-password-1") + for _ in range(3): + await _advance_past_login_throttle(db_session, clock, ip) + with pytest.raises(AuthenticationError): + await auth_service.login(db_session, settings, ip, wrong_login) + + await _advance_past_login_throttle(db_session, clock, ip) + correct_login = LoginRequest(email=email, password="correct-horse-1") + await auth_service.login(db_session, settings, ip, correct_login) + + refreshed = await UserRepository(db_session).get_by_id(user.id) + assert refreshed.failed_login_count == 0 + assert refreshed.locked_at is None + + +# --- Turnstile --------------------------------------------------------- + + +def test_register_without_turnstile_token_rejected_when_configured(client: TestClient): + settings = get_settings().model_copy(update={"turnstile_secret": "fake-secret"}) + app.dependency_overrides[get_settings] = lambda: settings + try: + resp = client.post( + "/api/v1/auth/register", + json={"email": _unique_email(), "password": "correct-horse-1", "display_name": "T"}, + ) + assert resp.status_code == 400 + finally: + app.dependency_overrides.pop(get_settings, None) + + +def test_register_with_verified_turnstile_token_succeeds(client: TestClient, monkeypatch): + settings = get_settings().model_copy(update={"turnstile_secret": "fake-secret"}) + app.dependency_overrides[get_settings] = lambda: settings + + async def fake_verify(token, remote_ip, _settings): + assert token == "good-token" + return True + + monkeypatch.setattr("app.api.v1.auth.verify_turnstile", fake_verify) + try: + resp = client.post( + "/api/v1/auth/register", + json={ + "email": _unique_email(), + "password": "correct-horse-1", + "display_name": "T", + "turnstile_token": "good-token", + }, + ) + assert resp.status_code == 201 + finally: + app.dependency_overrides.pop(get_settings, None) + + +def test_register_skips_turnstile_entirely_on_localhost_even_when_configured(): + settings = get_settings().model_copy(update={"turnstile_secret": "fake-secret"}) + app.dependency_overrides[get_settings] = lambda: settings + try: + with TestClient(app, client=("127.0.0.1", 54321)) as loopback_client: + resp = loopback_client.post( + "/api/v1/auth/register", + json={ + "email": _unique_email(), + "password": "correct-horse-1", + "display_name": "T", + }, + ) + assert resp.status_code == 201 + finally: + app.dependency_overrides.pop(get_settings, None) + + +async def test_register_succeeds_for_non_localhost_caller_when_configured_secret_is_invalid( + client: TestClient, +): + """End-to-end proof (not just verify_turnstile in isolation): a + non-loopback caller can still register when the admin's configured + secret is itself broken, regardless of what token they submitted.""" + settings = get_settings().model_copy(update={"turnstile_secret": "a-typo-d-secret"}) + app.dependency_overrides[get_settings] = lambda: settings + try: + with respx.mock: + respx.post("https://challenges.cloudflare.com/turnstile/v0/siteverify").mock( + return_value=httpx.Response( + 200, json={"success": False, "error-codes": ["invalid-input-secret"]} + ) + ) + resp = client.post( + "/api/v1/auth/register", + json={ + "email": _unique_email(), + "password": "correct-horse-1", + "display_name": "T", + "turnstile_token": "whatever-token", + }, + ) + assert resp.status_code == 201 + finally: + app.dependency_overrides.pop(get_settings, None) + + +async def test_verify_turnstile_returns_true_on_cloudflare_success(): + settings = get_settings().model_copy(update={"turnstile_secret": "fake-secret"}) + with respx.mock: + respx.post("https://challenges.cloudflare.com/turnstile/v0/siteverify").mock( + return_value=httpx.Response(200, json={"success": True}) + ) + result = await verify_turnstile("some-token", "1.2.3.4", settings) + assert result is True + + +async def test_verify_turnstile_fails_closed_on_network_error(): + settings = get_settings().model_copy(update={"turnstile_secret": "fake-secret"}) + with respx.mock: + respx.post("https://challenges.cloudflare.com/turnstile/v0/siteverify").mock( + side_effect=httpx.ConnectError("boom") + ) + result = await verify_turnstile("some-token", "1.2.3.4", settings) + assert result is False + + +async def test_verify_turnstile_fails_open_when_the_configured_secret_itself_is_invalid(): + """A typo'd/invalid secret is a detectable config problem (Cloudflare + reports it via error-codes), not an ambiguous failure - locking out + every real visitor over an admin's own mistake is worse than briefly + running with reduced bot protection.""" + settings = get_settings().model_copy(update={"turnstile_secret": "a-typo-d-secret"}) + with respx.mock: + respx.post("https://challenges.cloudflare.com/turnstile/v0/siteverify").mock( + return_value=httpx.Response( + 200, json={"success": False, "error-codes": ["invalid-input-secret"]} + ) + ) + result = await verify_turnstile("some-token", "1.2.3.4", settings) + assert result is True + + +async def test_verify_turnstile_still_fails_closed_for_a_genuinely_bad_user_token(): + """The fail-open carve-out is scoped to secret-level error codes only - + a real rejection of the user's own token must still fail closed.""" + settings = get_settings().model_copy(update={"turnstile_secret": "fake-secret"}) + with respx.mock: + respx.post("https://challenges.cloudflare.com/turnstile/v0/siteverify").mock( + return_value=httpx.Response( + 200, json={"success": False, "error-codes": ["invalid-input-response"]} + ) + ) + result = await verify_turnstile("bad-token", "1.2.3.4", settings) + assert result is False + + +# --- IP ban ----------------------------------------------------------------- + + +async def test_register_rejected_when_ip_is_banned(db_session): + """A ban is IP-global - it must block brand-new account creation too, + not just actions against existing accounts (login/resend/reset).""" + ip = _unique_ip() + await IpThrottleRepository(db_session).create_ban(ip, "failed_login", datetime.now(UTC)) + await db_session.commit() + + with TestClient(app, client=(ip, 51234)) as c: + resp = c.post( + "/api/v1/auth/register", + json={"email": _unique_email(), "password": "correct-horse-1", "display_name": "T"}, + ) + assert resp.status_code == 429 + assert resp.json()["detail"] == "This IP address has been temporarily blocked." diff --git a/apps/api/tests/unit/test_snapshots_api.py b/apps/api/tests/unit/test_snapshots_api.py new file mode 100644 index 0000000..e093c38 --- /dev/null +++ b/apps/api/tests/unit/test_snapshots_api.py @@ -0,0 +1,98 @@ +"""Snapshots API: read-only history listing, newest-first, ownership-scoped. +Snapshots have no creation endpoint (they're written internally by +collection_service.py during a monitoring run), so tests insert one directly +via db_session against the same DB the `client` fixture's TestClient uses.""" + +from __future__ import annotations + +import uuid + +import pytest + +from app.models.snapshot import Snapshot + + +def _register_and_login(client) -> dict[str, str]: + email = f"user-{uuid.uuid4().hex[:12]}@example.com" + client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "Test User"}, + ) + tokens = client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ).json() + return {"Authorization": f"Bearer {tokens['access_token']}"} + + +def _create_company(client, headers): + return client.post( + "/api/v1/companies", + json={"name": f"Co {uuid.uuid4().hex[:6]}", "frequency_type": "weekly"}, + headers=headers, + ).json() + + +def _create_source(client, headers, company_id): + return client.post( + f"/api/v1/companies/{company_id}/sources", + json={"source_type": "custom_url", "name": "Pricing", "base_url": "https://example.com"}, + headers=headers, + ).json() + + +def test_snapshots_empty_before_any_collection(client): + headers = _register_and_login(client) + company = _create_company(client, headers) + + resp = client.get(f"/api/v1/companies/{company['id']}/snapshots", headers=headers) + + assert resp.status_code == 200 + assert resp.json() == [] + + +@pytest.mark.asyncio +async def test_snapshots_list_newest_first(client, db_session): + headers = _register_and_login(client) + company = _create_company(client, headers) + source = _create_source(client, headers, company["id"]) + + older = Snapshot( + company_id=uuid.UUID(company["id"]), + source_id=uuid.UUID(source["id"]), + snapshot_type="website", + hash="hash-older", + structured_summary={"title": "Old"}, + text_summary="Old page text", + ) + db_session.add(older) + await db_session.commit() + + newer = Snapshot( + company_id=uuid.UUID(company["id"]), + source_id=uuid.UUID(source["id"]), + snapshot_type="website", + hash="hash-newer", + structured_summary={"title": "New"}, + text_summary="New page text", + ) + db_session.add(newer) + await db_session.commit() + + resp = client.get(f"/api/v1/companies/{company['id']}/snapshots", headers=headers) + + assert resp.status_code == 200 + body = resp.json() + assert len(body) == 2 + assert body[0]["hash"] == "hash-newer" + assert body[1]["hash"] == "hash-older" + assert body[0]["text_summary"] == "New page text" + + +def test_snapshots_scoped_to_owner(client): + owner_headers = _register_and_login(client) + other_headers = _register_and_login(client) + company = _create_company(client, owner_headers) + + resp = client.get(f"/api/v1/companies/{company['id']}/snapshots", headers=other_headers) + + assert resp.status_code == 404 diff --git a/apps/api/tests/unit/test_sources_api.py b/apps/api/tests/unit/test_sources_api.py new file mode 100644 index 0000000..98c1d63 --- /dev/null +++ b/apps/api/tests/unit/test_sources_api.py @@ -0,0 +1,190 @@ +"""Sources API: ownership isolation, user-creatable type restriction, and +the ad-hoc test action - via HTTP, with respx mocking the network call the +test action makes.""" + +from __future__ import annotations + +import uuid +from unittest.mock import patch + +import httpx +import respx + + +def _register_and_login(client) -> dict[str, str]: + email = f"user-{uuid.uuid4().hex[:12]}@example.com" + client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "Test User"}, + ) + tokens = client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ).json() + return {"Authorization": f"Bearer {tokens['access_token']}"} + + +def _create_company(client, headers): + return client.post( + "/api/v1/companies", + json={"name": f"Co {uuid.uuid4().hex[:6]}", "frequency_type": "weekly"}, + headers=headers, + ).json() + + +def test_create_custom_url_source(client): + headers = _register_and_login(client) + company = _create_company(client, headers) + + resp = client.post( + f"/api/v1/companies/{company['id']}/sources", + json={"source_type": "custom_url", "name": "Pricing", "base_url": "example.com/pricing"}, + headers=headers, + ) + assert resp.status_code == 201 + body = resp.json() + assert body["base_url"] == "https://example.com/pricing" + assert body["status"] == "active" + + +def test_create_source_rejects_non_user_creatable_type(client): + headers = _register_and_login(client) + company = _create_company(client, headers) + + resp = client.post( + f"/api/v1/companies/{company['id']}/sources", + json={"source_type": "github", "name": "GitHub", "base_url": "https://github.com/acme"}, + headers=headers, + ) + assert resp.status_code == 422 + + +def test_sources_scoped_to_owner(client): + owner_headers = _register_and_login(client) + other_headers = _register_and_login(client) + company = _create_company(client, owner_headers) + + created = client.post( + f"/api/v1/companies/{company['id']}/sources", + json={ + "source_type": "custom_url", + "name": "Pricing", + "base_url": "https://example.com/pricing", + }, + headers=owner_headers, + ).json() + + # Another user can't list this company's sources... + resp = client.get(f"/api/v1/companies/{company['id']}/sources", headers=other_headers) + assert resp.status_code == 404 + + # ...or update/delete the source directly. + resp = client.patch( + f"/api/v1/sources/{created['id']}", json={"active": False}, headers=other_headers + ) + assert resp.status_code == 404 + + +def test_update_and_delete_source(client): + headers = _register_and_login(client) + company = _create_company(client, headers) + created = client.post( + f"/api/v1/companies/{company['id']}/sources", + json={ + "source_type": "custom_url", + "name": "Pricing", + "base_url": "https://example.com/pricing", + }, + headers=headers, + ).json() + + resp = client.patch(f"/api/v1/sources/{created['id']}", json={"active": False}, headers=headers) + assert resp.status_code == 200 + assert resp.json()["active"] is False + + resp = client.delete(f"/api/v1/sources/{created['id']}", headers=headers) + assert resp.status_code == 204 + + resp = client.get(f"/api/v1/companies/{company['id']}/sources", headers=headers) + assert resp.json() == [] + + +def test_update_source_sets_a_frequency_override(client): + headers = _register_and_login(client) + company = _create_company(client, headers) + created = client.post( + f"/api/v1/companies/{company['id']}/sources", + json={ + "source_type": "custom_url", + "name": "Pricing", + "base_url": "https://example.com/pricing", + }, + headers=headers, + ).json() + + resp = client.patch( + f"/api/v1/sources/{created['id']}", json={"frequency_type": "daily"}, headers=headers + ) + assert resp.status_code == 200 + body = resp.json() + assert body["frequency_type"] == "daily" + assert body["next_check"] is None # takes effect on the next scheduler tick + + # Clearing the override back to "same as company" is an explicit null. + resp = client.patch( + f"/api/v1/sources/{created['id']}", json={"frequency_type": None}, headers=headers + ) + assert resp.status_code == 200 + assert resp.json()["frequency_type"] is None + + +def test_update_source_rejects_a_custom_frequency_below_the_minimum_interval(client): + headers = _register_and_login(client) + company = _create_company(client, headers) + created = client.post( + f"/api/v1/companies/{company['id']}/sources", + json={ + "source_type": "custom_url", + "name": "Pricing", + "base_url": "https://example.com/pricing", + }, + headers=headers, + ).json() + + resp = client.patch( + f"/api/v1/sources/{created['id']}", + json={"frequency_type": "custom", "interval_minutes": 1}, + headers=headers, + ) + assert resp.status_code == 400 + + +def test_source_test_action_runs_a_real_collection(client): + headers = _register_and_login(client) + company = _create_company(client, headers) + created = client.post( + f"/api/v1/companies/{company['id']}/sources", + json={ + "source_type": "custom_url", + "name": "Pricing", + "base_url": "https://example.com/pricing", + }, + headers=headers, + ).json() + + with patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]): + with respx.mock: + respx.get("https://example.com/robots.txt").mock(return_value=httpx.Response(404)) + respx.get("https://example.com/pricing").mock( + return_value=httpx.Response( + 200, + html="Pricing" + "

Pricing

Plans start at $10/month.

" + "", + ) + ) + resp = client.post(f"/api/v1/sources/{created['id']}/test", headers=headers) + + assert resp.status_code == 200 + body = resp.json() + assert body["status"] == "active" + assert body["documents_found"] == 1 diff --git a/apps/api/tests/unit/test_ssrf.py b/apps/api/tests/unit/test_ssrf.py new file mode 100644 index 0000000..eef5ba2 --- /dev/null +++ b/apps/api/tests/unit/test_ssrf.py @@ -0,0 +1,77 @@ +"""SSRF guard tests - see SECURITY.md.""" + +from __future__ import annotations + +from unittest.mock import patch + +import httpx +import pytest +import respx + +from app.core.http import SsrfBlockedError, safe_fetch, validate_url + + +def test_validate_url_rejects_disallowed_scheme(): + with pytest.raises(SsrfBlockedError): + validate_url("file:///etc/passwd") + + +def test_validate_url_rejects_url_with_no_hostname(): + with pytest.raises(SsrfBlockedError): + validate_url("http://") + + +@pytest.mark.parametrize( + "hostname,ip", + [ + ("localhost-test", "127.0.0.1"), + ("private-test", "10.0.0.5"), + ("private-test-2", "192.168.1.1"), + ("link-local-test", "169.254.1.1"), + ("metadata-test", "169.254.169.254"), + ], +) +def test_validate_url_blocks_private_and_metadata_addresses(hostname, ip): + with patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", (ip, 0))]): + with pytest.raises(SsrfBlockedError): + validate_url(f"http://{hostname}/") + + +def test_validate_url_allows_public_address(): + with patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]): + validate_url("http://example.com/") # should not raise + + +@pytest.mark.asyncio +async def test_safe_fetch_revalidates_each_redirect_hop(): + """A redirect to a private address must be blocked even if the initial + URL resolves to a public one.""" + with patch("socket.getaddrinfo") as mock_resolve: + + def resolve(hostname, *_args, **_kwargs): + if hostname == "public.example": + return [(2, 1, 6, "", ("93.184.216.34", 0))] + if hostname == "internal.example": + return [(2, 1, 6, "", ("10.0.0.5", 0))] + raise AssertionError(f"unexpected hostname {hostname}") + + mock_resolve.side_effect = resolve + + with respx.mock: + respx.get("http://public.example/").mock( + return_value=httpx.Response(302, headers={"Location": "http://internal.example/"}) + ) + with pytest.raises(SsrfBlockedError): + await safe_fetch("http://public.example/") + + +@pytest.mark.asyncio +async def test_safe_fetch_returns_final_response_body(): + with patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]): + with respx.mock: + respx.get("http://public.example/").mock( + return_value=httpx.Response(200, text="hello world") + ) + result = await safe_fetch("http://public.example/") + assert result.status_code == 200 + assert result.text == "hello world" diff --git a/apps/api/tests/unit/test_structured_diff.py b/apps/api/tests/unit/test_structured_diff.py new file mode 100644 index 0000000..9aba562 --- /dev/null +++ b/apps/api/tests/unit/test_structured_diff.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from app.change_detection.structured_diff import diff_item_sets + + +def test_diff_item_sets_detects_additions_and_removals(): + previous = ["https://x.com/a", "https://x.com/b"] + current = ["https://x.com/b", "https://x.com/c"] + diff = diff_item_sets(previous, current) + assert diff.added == ["https://x.com/c"] + assert diff.removed == ["https://x.com/a"] + assert diff.has_changes is True + + +def test_diff_item_sets_no_change_when_identical(): + items = ["https://x.com/a", "https://x.com/b"] + diff = diff_item_sets(items, list(items)) + assert diff.added == [] + assert diff.removed == [] + assert diff.has_changes is False + + +def test_diff_item_sets_handles_empty_previous(): + diff = diff_item_sets([], ["https://x.com/a"]) + assert diff.added == ["https://x.com/a"] + assert diff.removed == [] diff --git a/apps/api/tests/unit/test_system_endpoints.py b/apps/api/tests/unit/test_system_endpoints.py new file mode 100644 index 0000000..508ab74 --- /dev/null +++ b/apps/api/tests/unit/test_system_endpoints.py @@ -0,0 +1,121 @@ +"""/system/status and /system/logs - especially that /system/logs is +admin-only (Phase 19 - it exposes operational internals, not something any +registered user should read) and that the live log feed actually captures +what the app logs. Server-wide secret management (Turnstile site +key/secret) moved to /system/secrets - see test_system_secrets.py.""" + +from __future__ import annotations + +import uuid + +from fastapi.testclient import TestClient +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.config import get_settings +from app.core.logging import get_logger +from app.main import app +from app.repositories.user_repository import UserRepository + + +def _register_and_login(client: TestClient) -> dict[str, str]: + email = f"user-{uuid.uuid4().hex[:12]}@example.com" + client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "Test User"}, + ) + tokens = client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ).json() + return {"Authorization": f"Bearer {tokens['access_token']}"} + + +async def _register_admin_and_login(client: TestClient, db_session: AsyncSession) -> dict[str, str]: + """Registration never accepts is_admin from the client - promote + directly in the DB, the same way a real operator would via a one-off + script/console, not through any HTTP-exposed path.""" + email = f"admin-{uuid.uuid4().hex[:12]}@example.com" + client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "Admin User"}, + ) + user = await UserRepository(db_session).get_by_email(email) + user.is_admin = True + await db_session.commit() + tokens = client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ).json() + return {"Authorization": f"Bearer {tokens['access_token']}"} + + +def test_system_status_reports_not_localhost_for_default_test_client(client: TestClient): + # Starlette's TestClient defaults its ASGI scope client to + # ("testclient", 50000), not a loopback address - this is the "someone + # not on this machine" case. + resp = client.get("/api/v1/system/status") + assert resp.status_code == 200 + assert resp.json()["is_localhost"] is False + + +def test_system_status_reports_localhost_for_loopback_client(): + with TestClient(app, client=("127.0.0.1", 54321)) as loopback_client: + resp = loopback_client.get("/api/v1/system/status") + assert resp.status_code == 200 + assert resp.json()["is_localhost"] is True + + +def test_system_status_treats_configured_extra_ip_as_localhost(): + # Docker Desktop's bridge networking means host-originated traffic + # never arrives as literal loopback - additional_trusted_local_ips is + # the opt-in escape hatch for that, see app.core.security.is_localhost. + test_settings = get_settings().model_copy( + update={"additional_trusted_local_ips": "172.18.0.1, 10.0.0.5"} + ) + app.dependency_overrides[get_settings] = lambda: test_settings + try: + with TestClient(app, client=("172.18.0.1", 54321)) as bridge_client: + resp = bridge_client.get("/api/v1/system/status") + assert resp.status_code == 200 + assert resp.json()["is_localhost"] is True + finally: + app.dependency_overrides.pop(get_settings, None) + + +def test_system_status_does_not_trust_an_unlisted_ip(): + test_settings = get_settings().model_copy(update={"additional_trusted_local_ips": "172.18.0.1"}) + app.dependency_overrides[get_settings] = lambda: test_settings + try: + with TestClient(app, client=("203.0.113.9", 54321)) as stranger_client: + resp = stranger_client.get("/api/v1/system/status") + assert resp.status_code == 200 + assert resp.json()["is_localhost"] is False + finally: + app.dependency_overrides.pop(get_settings, None) + + +async def test_system_logs_captures_and_categorizes_real_log_calls( + client: TestClient, db_session: AsyncSession +): + headers = await _register_admin_and_login(client, db_session) + marker = f"phase17-test-marker-{uuid.uuid4().hex[:8]}" + + logger = get_logger("tests.system_logs") + logger.warning("test_api_style_failure", event_marker=marker) + + resp = client.get("/api/v1/system/logs", headers=headers) + assert resp.status_code == 200 + entries = resp.json() + match = next(e for e in entries if e["context"].get("event_marker") == marker) + assert match["category"] == "api_error" + assert match["level"] == "warning" + assert match["event"] == "test_api_style_failure" + + +def test_system_logs_requires_auth(client: TestClient): + resp = client.get("/api/v1/system/logs") + assert resp.status_code == 401 + + +def test_system_logs_non_admin_forbidden(client: TestClient): + headers = _register_and_login(client) + resp = client.get("/api/v1/system/logs", headers=headers) + assert resp.status_code == 403 diff --git a/apps/api/tests/unit/test_system_secrets.py b/apps/api/tests/unit/test_system_secrets.py new file mode 100644 index 0000000..626ad87 --- /dev/null +++ b/apps/api/tests/unit/test_system_secrets.py @@ -0,0 +1,294 @@ +"""Server-wide secrets (Turnstile site key/secret) admin-managed from the +Settings page instead of only .env: repository/service behavior, endpoint +auth/admin-gating (deliberately NOT localhost-gated, unlike the old +/system/api-keys this replaces), /system/status exposing the site key live, +and an end-to-end proof that a DB-only (no .env) secret actually drives +Turnstile enforcement on register.""" + +from __future__ import annotations + +import uuid + +import httpx +import pytest +import respx +from fastapi.testclient import TestClient +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.config import get_settings +from app.models.enums import SystemSecretKey +from app.repositories.system_secret_repository import SystemSecretRepository +from app.repositories.user_repository import UserRepository +from app.services import system_secret_service + + +@pytest.fixture(autouse=True) +async def _clean_system_secrets(db_session): + """SystemSecret rows are true global singletons (one per key, not + per-user like UserApiKey) - unlike other tests in this suite that + dodge cross-test pollution by randomizing an id, there's no such trick + here. Every test starts from a clean slate, and leaves one behind for + whatever test file runs next in a full-suite run.""" + + async def _clear() -> None: + repo = SystemSecretRepository(db_session) + for row in await repo.list_all(): + await db_session.delete(row) + await db_session.commit() + + await _clear() + yield + await _clear() + + +def _unique_email() -> str: + return f"user-{uuid.uuid4().hex[:12]}@example.com" + + +def _register_and_login(client: TestClient) -> dict[str, str]: + email = _unique_email() + client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "T"}, + ) + tokens = client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ).json() + return {"Authorization": f"Bearer {tokens['access_token']}"} + + +async def _register_admin_and_login(client: TestClient, db_session: AsyncSession) -> dict[str, str]: + email = f"admin-{uuid.uuid4().hex[:12]}@example.com" + client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "Admin"}, + ) + user = await UserRepository(db_session).get_by_email(email) + user.is_admin = True + await db_session.commit() + tokens = client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ).json() + return {"Authorization": f"Bearer {tokens['access_token']}"} + + +# --- Repository -------------------------------------------------------- + + +async def test_repository_upsert_then_get_then_delete(db_session): + repo = SystemSecretRepository(db_session) + + await repo.upsert(SystemSecretKey.TURNSTILE_SECRET, "encrypted-1") + row = await repo.get(SystemSecretKey.TURNSTILE_SECRET) + assert row is not None + assert row.encrypted_value == "encrypted-1" + + await repo.upsert(SystemSecretKey.TURNSTILE_SECRET, "encrypted-2") + row = await repo.get(SystemSecretKey.TURNSTILE_SECRET) + assert row.encrypted_value == "encrypted-2" # updated in place, not duplicated + + await repo.delete(SystemSecretKey.TURNSTILE_SECRET) + assert await repo.get(SystemSecretKey.TURNSTILE_SECRET) is None + + +# --- Service ------------------------------------------------------------- + + +async def test_list_status_shows_both_keys_unconfigured_by_default(db_session): + settings = get_settings() + statuses = await system_secret_service.list_status(db_session, settings) + assert {s["key"] for s in statuses} == {"turnstile_site_key", "turnstile_secret"} + assert all(s["configured"] is False for s in statuses) + assert all(s["value"] is None for s in statuses) + + +async def test_set_secret_then_list_status_shows_it_configured(db_session): + settings = get_settings() + await system_secret_service.set_secret( + db_session, + SystemSecretKey.TURNSTILE_SITE_KEY, + "0x-my-site-key", + settings, + admin_user_id=uuid.uuid4(), + client_ip="10.0.0.1", + ) + statuses = await system_secret_service.list_status(db_session, settings) + site_key = next(s for s in statuses if s["key"] == "turnstile_site_key") + assert site_key["configured"] is True + assert site_key["value"] == "0x-my-site-key" + + +async def test_set_blank_secret_clears_a_previously_set_one(db_session): + settings = get_settings() + await system_secret_service.set_secret( + db_session, + SystemSecretKey.TURNSTILE_SECRET, + "0x-my-secret", + settings, + admin_user_id=uuid.uuid4(), + client_ip="10.0.0.1", + ) + await system_secret_service.set_secret( + db_session, + SystemSecretKey.TURNSTILE_SECRET, + " ", + settings, + admin_user_id=uuid.uuid4(), + client_ip="10.0.0.1", + ) + statuses = await system_secret_service.list_status(db_session, settings) + secret = next(s for s in statuses if s["key"] == "turnstile_secret") + assert secret["configured"] is False + assert secret["value"] is None + + +async def test_get_effective_settings_falls_back_to_global_when_unset(db_session): + settings = get_settings().model_copy(update={"turnstile_site_key": "global-site-key"}) + effective = await system_secret_service.get_effective_settings(db_session, settings) + assert effective.turnstile_site_key == "global-site-key" + + +async def test_get_effective_settings_overrides_only_the_keys_that_were_set(db_session): + settings = get_settings().model_copy( + update={"turnstile_site_key": "global-site-key", "turnstile_secret": "global-secret"} + ) + await system_secret_service.set_secret( + db_session, + SystemSecretKey.TURNSTILE_SITE_KEY, + "admin-site-key", + settings, + admin_user_id=uuid.uuid4(), + client_ip="10.0.0.1", + ) + effective = await system_secret_service.get_effective_settings(db_session, settings) + assert effective.turnstile_site_key == "admin-site-key" + assert effective.turnstile_secret == "global-secret" # untouched, no override set + + +# --- Endpoints ----------------------------------------------------------- + + +def test_list_system_secrets_requires_auth(client: TestClient): + resp = client.get("/api/v1/system/secrets") + assert resp.status_code == 401 + + +def test_list_system_secrets_non_admin_forbidden(client: TestClient): + headers = _register_and_login(client) + resp = client.get("/api/v1/system/secrets", headers=headers) + assert resp.status_code == 403 + + +async def test_admin_can_list_and_set_secrets_even_from_a_non_loopback_client( + client: TestClient, db_session: AsyncSession +): + """Deliberately different from the old /system/api-keys this replaces: + admin-gated only, no additional is_localhost requirement - the default + TestClient here has a non-loopback fake peer.""" + headers = await _register_admin_and_login(client, db_session) + + initial = client.get("/api/v1/system/secrets", headers=headers) + assert initial.status_code == 200 + assert all(not s["configured"] for s in initial.json()) + + set_resp = client.put( + "/api/v1/system/secrets/turnstile_secret", + json={"value": "sk-set-via-api"}, + headers=headers, + ) + assert set_resp.status_code == 200 + assert set_resp.json()["configured"] is True + assert set_resp.json()["value"] == "sk-set-via-api" + + after = client.get("/api/v1/system/secrets", headers=headers) + secret = next(s for s in after.json() if s["key"] == "turnstile_secret") + assert secret["configured"] is True + assert secret["value"] == "sk-set-via-api" + + +async def test_updating_a_server_secret_is_logged_to_the_acting_admins_account_activity( + client: TestClient, db_session: AsyncSession +): + headers = await _register_admin_and_login(client, db_session) + + client.put( + "/api/v1/system/secrets/turnstile_secret", + json={"value": "sk-set-via-api"}, + headers=headers, + ) + + events = client.get("/api/v1/auth/security-events", headers=headers).json() + assert any(e["event_type"] == "server_secret_updated" for e in events) + + +async def test_set_system_secret_rejects_unknown_key(client: TestClient, db_session: AsyncSession): + headers = await _register_admin_and_login(client, db_session) + resp = client.put("/api/v1/system/secrets/not-a-real-key", json={"value": "x"}, headers=headers) + assert resp.status_code == 422 + + +# --- /system/status exposes the site key live ------------------------- + + +async def test_system_status_exposes_admin_configured_turnstile_site_key( + client: TestClient, db_session: AsyncSession +): + settings = get_settings() + await system_secret_service.set_secret( + db_session, + SystemSecretKey.TURNSTILE_SITE_KEY, + "admin-set-site-key", + settings, + admin_user_id=uuid.uuid4(), + client_ip="10.0.0.1", + ) + resp = client.get("/api/v1/system/status") + assert resp.status_code == 200 + assert resp.json()["turnstile_site_key"] == "admin-set-site-key" + + +def test_system_status_turnstile_site_key_is_null_when_unconfigured(client: TestClient): + resp = client.get("/api/v1/system/status") + assert resp.status_code == 200 + assert resp.json()["turnstile_site_key"] is None + + +# --- End-to-end: a DB-only secret (no .env value) drives enforcement ---- + + +async def test_register_requires_turnstile_when_only_db_configured_secret_exists( + client: TestClient, db_session: AsyncSession +): + """Proves _enforce_turnstile actually resolves effective (DB-aware) + settings, not just the raw .env-backed global Settings object.""" + settings = get_settings() + assert not settings.turnstile_secret # sanity: nothing set in .env for this test run + await system_secret_service.set_secret( + db_session, + SystemSecretKey.TURNSTILE_SECRET, + "admin-set-secret", + settings, + admin_user_id=uuid.uuid4(), + client_ip="10.0.0.1", + ) + + no_token_resp = client.post( + "/api/v1/auth/register", + json={"email": _unique_email(), "password": "correct-horse-1", "display_name": "T"}, + ) + assert no_token_resp.status_code == 400 + + with respx.mock: + respx.post("https://challenges.cloudflare.com/turnstile/v0/siteverify").mock( + return_value=httpx.Response(200, json={"success": True}) + ) + with_token_resp = client.post( + "/api/v1/auth/register", + json={ + "email": _unique_email(), + "password": "correct-horse-1", + "display_name": "T", + "turnstile_token": "good-token", + }, + ) + assert with_token_resp.status_code == 201 diff --git a/apps/api/tests/unit/test_text_diff.py b/apps/api/tests/unit/test_text_diff.py new file mode 100644 index 0000000..44803a4 --- /dev/null +++ b/apps/api/tests/unit/test_text_diff.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from app.change_detection.text_diff import bounded_text_diff + + +def test_identical_text_has_zero_diff_ratio(): + text = "Acme Corp builds electric trucks.\nWe are hiring engineers." + result = bounded_text_diff(text, text) + assert result.diff_ratio == 0.0 + assert result.is_identical is True + + +def test_changed_text_has_nonzero_diff_ratio_and_captures_lines(): + previous = "Acme Corp builds gasoline trucks.\nContact us for a quote." + current = "Acme Corp builds electric trucks.\nContact us for a quote." + result = bounded_text_diff(previous, current) + assert result.diff_ratio > 0.0 + assert any("electric" in line for line in result.added_lines) + assert any("gasoline" in line for line in result.removed_lines) + + +def test_noise_only_changes_do_not_register_as_a_diff(): + previous = "About us.\nUpdated: 2026-01-01 10:00\nWe build trucks." + current = "About us.\nUpdated: 2026-06-15 14:30\nWe build trucks." + result = bounded_text_diff(previous, current) + assert result.diff_ratio == 0.0 + + +def test_diff_is_bounded_in_size(): + previous = "\n".join(f"line {i} original" for i in range(200)) + current = "\n".join(f"line {i} changed" for i in range(200)) + result = bounded_text_diff(previous, current) + assert len(result.added_lines) <= 40 + assert len(result.removed_lines) <= 40 diff --git a/apps/api/tests/unit/test_unban_admin.py b/apps/api/tests/unit/test_unban_admin.py new file mode 100644 index 0000000..561195a --- /dev/null +++ b/apps/api/tests/unit/test_unban_admin.py @@ -0,0 +1,263 @@ +"""Unban-request intake + admin IP-ban management (Phase 19). No dedicated +coverage existed for this endpoint group before - added alongside the +Mailpit removal, which changed submit_unban_request to notify every +is_admin=True account instead of a single fixed admin_notification_email.""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime + +from fastapi.testclient import TestClient +from sqlalchemy.ext.asyncio import AsyncSession + +from app.main import app +from app.repositories.ip_throttle_repository import IpThrottleRepository +from app.repositories.unban_request_repository import UnbanRequestRepository +from app.repositories.user_repository import UserRepository + + +def _unique_email() -> str: + return f"user-{uuid.uuid4().hex[:12]}@example.com" + + +def _unique_ip() -> str: + # Randomize all three trailing octets (same convention as + # test_ip_throttle_service.py) - a single-octet range only has ~250 + # values, which collides often enough across a full suite run (birthday + # paradox) to cause real, intermittent failures between unrelated tests + # that happen to share ip_throttle_state rows. + return f"10.{uuid.uuid4().int % 250}.{uuid.uuid4().int % 250}.{uuid.uuid4().int % 250}" + + +def _install_fake_smtp(monkeypatch) -> list[dict]: + sent: list[dict] = [] + + class FakeSmtp: + def __init__(self, host, port, timeout=10): + pass + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def starttls(self): + pass + + def login(self, username, password): + pass + + def sendmail(self, from_addr, to_addrs, message): + sent.append({"to": to_addrs, "message": message}) + + monkeypatch.setattr("app.notifications.smtp_email.smtplib.SMTP", FakeSmtp) + return sent + + +async def _register_admin_and_login(client: TestClient, db_session: AsyncSession) -> dict[str, str]: + email = _unique_email() + client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "Admin"}, + ) + user = await UserRepository(db_session).get_by_email(email) + user.is_admin = True + await db_session.commit() + tokens = client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ).json() + return {"Authorization": f"Bearer {tokens['access_token']}"} + + +def test_unban_request_requires_no_auth_and_returns_204(monkeypatch): + _install_fake_smtp(monkeypatch) + ip = _unique_ip() + with TestClient(app, client=(ip, 51234)) as c: + resp = c.post("/api/v1/unban-requests", json={"message": "please unban me"}) + assert resp.status_code == 204 + + +def test_unban_request_cooldown_rejects_second_request_within_24h(monkeypatch): + _install_fake_smtp(monkeypatch) + ip = _unique_ip() + with TestClient(app, client=(ip, 51234)) as c: + first = c.post("/api/v1/unban-requests", json={"message": "one"}) + assert first.status_code == 204 + + second = c.post("/api/v1/unban-requests", json={"message": "two"}) + assert second.status_code == 429 + + +async def test_unban_request_notifies_every_admin_account_only(monkeypatch, db_session): + sent = _install_fake_smtp(monkeypatch) + + admin_repo = UserRepository(db_session) + admin_one = await admin_repo.create( + email=_unique_email(), + password_hash="x", + display_name="Admin One", + timezone="UTC", + is_admin=True, + email_verified=True, + ) + admin_two = await admin_repo.create( + email=_unique_email(), + password_hash="x", + display_name="Admin Two", + timezone="UTC", + is_admin=True, + email_verified=True, + ) + not_admin = await admin_repo.create( + email=_unique_email(), + password_hash="x", + display_name="Not Admin", + timezone="UTC", + is_admin=False, + email_verified=True, + ) + await db_session.commit() + + ip = _unique_ip() + with TestClient(app, client=(ip, 51234)) as c: + resp = c.post("/api/v1/unban-requests", json={"message": "please unban me"}) + assert resp.status_code == 204 + + # Other tests in the same run may have their own admin accounts (plus + # the fixed local-dev user, always admin) - assert membership, not an + # exact total count. + recipients = {msg["to"][0] for msg in sent} + assert admin_one.email in recipients + assert admin_two.email in recipients + assert not_admin.email not in recipients + + +async def test_admin_ip_ban_endpoints_work_for_an_admin(db_session: AsyncSession): + ip = _unique_ip() + with TestClient(app, client=(ip, 51234)) as c: + headers = await _register_admin_and_login(c, db_session) + + bans_resp = c.get("/api/v1/admin/ip-bans", headers=headers) + assert bans_resp.status_code == 200 + + requests_resp = c.get("/api/v1/admin/unban-requests", headers=headers) + assert requests_resp.status_code == 200 + + +def test_admin_ip_ban_endpoints_reject_non_admin(client: TestClient): + email = _unique_email() + client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "T"}, + ) + tokens = client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ).json() + headers = {"Authorization": f"Bearer {tokens['access_token']}"} + + resp = client.get("/api/v1/admin/ip-bans", headers=headers) + assert resp.status_code == 403 + + +async def test_admin_can_delete_ip_ban_and_it_clears_throttle_state(db_session: AsyncSession): + banned_ip = _unique_ip() + await IpThrottleRepository(db_session).create_ban(banned_ip, "failed_login", datetime.now(UTC)) + await db_session.commit() + + admin_ip = _unique_ip() + with TestClient(app, client=(admin_ip, 51235)) as c: + headers = await _register_admin_and_login(c, db_session) + resp = c.delete(f"/api/v1/admin/ip-bans/{banned_ip}", headers=headers) + assert resp.status_code == 204 + + ban = await IpThrottleRepository(db_session).get_ban(banned_ip) + assert ban is None + + +async def test_admin_can_manually_ban_an_ip(db_session: AsyncSession): + target_ip = _unique_ip() + admin_ip = _unique_ip() + with TestClient(app, client=(admin_ip, 51236)) as c: + headers = await _register_admin_and_login(c, db_session) + resp = c.post("/api/v1/admin/ip-bans", json={"ip_address": target_ip}, headers=headers) + assert resp.status_code == 201 + assert resp.json()["ip_address"] == target_ip + + ban = await IpThrottleRepository(db_session).get_ban(target_ip) + assert ban is not None + + +async def test_admin_ban_ip_rejects_already_banned_ip(db_session: AsyncSession): + target_ip = _unique_ip() + await IpThrottleRepository(db_session).create_ban(target_ip, "failed_login", datetime.now(UTC)) + await db_session.commit() + + admin_ip = _unique_ip() + with TestClient(app, client=(admin_ip, 51237)) as c: + headers = await _register_admin_and_login(c, db_session) + resp = c.post("/api/v1/admin/ip-bans", json={"ip_address": target_ip}, headers=headers) + assert resp.status_code == 409 + + +async def test_admin_ban_ip_rejects_malformed_address(db_session: AsyncSession): + admin_ip = _unique_ip() + with TestClient(app, client=(admin_ip, 51238)) as c: + headers = await _register_admin_and_login(c, db_session) + resp = c.post("/api/v1/admin/ip-bans", json={"ip_address": "not-an-ip"}, headers=headers) + assert resp.status_code == 422 + + +async def test_admin_can_accept_an_unban_request_and_it_unbans_the_ip(db_session: AsyncSession): + requester_ip = _unique_ip() + await IpThrottleRepository(db_session).create_ban( + requester_ip, "failed_login", datetime.now(UTC) + ) + await db_session.commit() + + with TestClient(app, client=(requester_ip, 51240)) as c: + c.post("/api/v1/unban-requests", json={"message": "please unban me"}) + + request = await UnbanRequestRepository(db_session).most_recent_for_ip(requester_ip) + assert request is not None + + admin_ip = _unique_ip() + with TestClient(app, client=(admin_ip, 51241)) as c: + headers = await _register_admin_and_login(c, db_session) + resp = c.post(f"/api/v1/admin/unban-requests/{request.id}/accept", headers=headers) + assert resp.status_code == 204 + + assert await IpThrottleRepository(db_session).get_ban(requester_ip) is None + assert await UnbanRequestRepository(db_session).get(request.id) is None + + +async def test_admin_can_reject_an_unban_request_and_the_ip_stays_banned(db_session: AsyncSession): + requester_ip = _unique_ip() + await IpThrottleRepository(db_session).create_ban( + requester_ip, "failed_login", datetime.now(UTC) + ) + await db_session.commit() + + with TestClient(app, client=(requester_ip, 51242)) as c: + c.post("/api/v1/unban-requests", json={"message": "please unban me"}) + + request = await UnbanRequestRepository(db_session).most_recent_for_ip(requester_ip) + assert request is not None + + admin_ip = _unique_ip() + with TestClient(app, client=(admin_ip, 51243)) as c: + headers = await _register_admin_and_login(c, db_session) + resp = c.delete(f"/api/v1/admin/unban-requests/{request.id}", headers=headers) + assert resp.status_code == 204 + + assert await IpThrottleRepository(db_session).get_ban(requester_ip) is not None + assert await UnbanRequestRepository(db_session).get(request.id) is None + + +async def test_admin_accept_unban_request_404s_for_unknown_id(db_session: AsyncSession): + admin_ip = _unique_ip() + with TestClient(app, client=(admin_ip, 51244)) as c: + headers = await _register_admin_and_login(c, db_session) + resp = c.post(f"/api/v1/admin/unban-requests/{uuid.uuid4()}/accept", headers=headers) + assert resp.status_code == 404 diff --git a/apps/api/tests/unit/test_user_api_keys.py b/apps/api/tests/unit/test_user_api_keys.py new file mode 100644 index 0000000..98d11b0 --- /dev/null +++ b/apps/api/tests/unit/test_user_api_keys.py @@ -0,0 +1,289 @@ +"""Per-user API keys: encryption roundtrip, repository/service behavior, +endpoint auth/ownership, and one end-to-end check that a user's own key is +actually used (not just stored) for a real provider call.""" + +from __future__ import annotations + +import uuid + +import httpx +import pytest +import respx +from fastapi.testclient import TestClient + +from app.core.config import get_settings +from app.core.crypto import decrypt_secret, encrypt_secret +from app.main import app +from app.models.enums import ApiKeyProvider +from app.repositories.user_api_key_repository import UserApiKeyRepository +from app.repositories.user_repository import UserRepository +from app.services import user_api_key_service + + +def _unique_email() -> str: + return f"user-{uuid.uuid4().hex[:12]}@example.com" + + +def _register_and_login(client: TestClient) -> dict[str, str]: + email = _unique_email() + client.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "T"}, + ) + tokens = client.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ).json() + return {"Authorization": f"Bearer {tokens['access_token']}"} + + +# --- Encryption -------------------------------------------------------- + + +def test_encrypt_decrypt_roundtrip(): + settings = get_settings() + ciphertext = encrypt_secret("sk-real-secret-value", settings) + assert ciphertext != "sk-real-secret-value" + assert decrypt_secret(ciphertext, settings) == "sk-real-secret-value" + + +def test_decrypt_with_wrong_key_raises(): + settings = get_settings() + other_key_settings = settings.model_copy( + update={"api_key_encryption_secret": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="} + ) + ciphertext = encrypt_secret("sk-real-secret-value", settings) + with pytest.raises(ValueError): + decrypt_secret(ciphertext, other_key_settings) + + +# --- Repository ---------------------------------------------------------- + + +async def test_repository_upsert_then_get_then_delete(db_session): + repo = UserApiKeyRepository(db_session) + user_id = uuid.uuid4() + + await repo.upsert(user_id, ApiKeyProvider.ANTHROPIC, "encrypted-1") + row = await repo.get(user_id, ApiKeyProvider.ANTHROPIC) + assert row is not None + assert row.encrypted_key == "encrypted-1" + + await repo.upsert(user_id, ApiKeyProvider.ANTHROPIC, "encrypted-2") + row = await repo.get(user_id, ApiKeyProvider.ANTHROPIC) + assert row.encrypted_key == "encrypted-2" # updated in place, not duplicated + + await repo.delete(user_id, ApiKeyProvider.ANTHROPIC) + assert await repo.get(user_id, ApiKeyProvider.ANTHROPIC) is None + + +# --- Service --------------------------------------------------------------- + + +async def test_list_status_shows_all_four_providers_unconfigured_by_default(db_session): + settings = get_settings() + statuses = await user_api_key_service.list_status(db_session, uuid.uuid4(), settings) + assert {s["provider"] for s in statuses} == { + "anthropic", + "brave_search", + "ninjapear", + "uspto", + } + assert all(s["configured"] is False for s in statuses) + assert all(s["value"] is None for s in statuses) + + uspto = next(s for s in statuses if s["provider"] == "uspto") + assert uspto["free"] is True + assert uspto["requires_government_id"] is True + + +async def test_set_key_then_list_status_shows_it_configured(db_session): + settings = get_settings() + user_id = uuid.uuid4() + + await user_api_key_service.set_key( + db_session, + user_id, + ApiKeyProvider.ANTHROPIC, + "sk-my-real-key", + settings, + client_ip="10.0.0.1", + ) + statuses = await user_api_key_service.list_status(db_session, user_id, settings) + anthropic = next(s for s in statuses if s["provider"] == "anthropic") + assert anthropic["configured"] is True + assert anthropic["value"] == "sk-my-real-key" + + +async def test_set_blank_key_clears_a_previously_set_one(db_session): + settings = get_settings() + user_id = uuid.uuid4() + + await user_api_key_service.set_key( + db_session, + user_id, + ApiKeyProvider.ANTHROPIC, + "sk-my-real-key", + settings, + client_ip="10.0.0.1", + ) + await user_api_key_service.set_key( + db_session, user_id, ApiKeyProvider.ANTHROPIC, " ", settings, client_ip="10.0.0.1" + ) + + statuses = await user_api_key_service.list_status(db_session, user_id, settings) + anthropic = next(s for s in statuses if s["provider"] == "anthropic") + assert anthropic["configured"] is False + assert anthropic["value"] is None + + +async def test_get_effective_settings_falls_back_to_global_when_unset(db_session): + settings = get_settings().model_copy(update={"anthropic_api_key": "global-key"}) + effective = await user_api_key_service.get_effective_settings( + db_session, uuid.uuid4(), settings + ) + assert effective.anthropic_api_key == "global-key" + + +async def test_get_effective_settings_overrides_only_the_providers_the_user_set(db_session): + settings = get_settings().model_copy( + update={"anthropic_api_key": "global-anthropic", "brave_search_api_key": "global-brave"} + ) + user_id = uuid.uuid4() + await user_api_key_service.set_key( + db_session, + user_id, + ApiKeyProvider.ANTHROPIC, + "my-own-anthropic-key", + settings, + client_ip="10.0.0.1", + ) + + effective = await user_api_key_service.get_effective_settings(db_session, user_id, settings) + assert effective.anthropic_api_key == "my-own-anthropic-key" + assert effective.brave_search_api_key == "global-brave" # untouched, no override set + + +async def test_list_status_never_fetches_ninjapear_credits_itself(db_session): + """list_status must never make its own live NinjaPear call - the + frontend sources that number from /system/status's already-fetched + ninjapear_credit_balance instead (see the Settings page's System + configuration box), so credits is always None from this endpoint + regardless of whether a key is configured.""" + settings = get_settings() + user_id = uuid.uuid4() + await user_api_key_service.set_key( + db_session, + user_id, + ApiKeyProvider.NINJAPEAR, + "my-ninjapear-key", + settings, + client_ip="10.0.0.1", + ) + + with respx.mock: + # No mock registered for nubela.co - respx raises if anything tries + # to call it, proving list_status makes no such request. + statuses = await user_api_key_service.list_status(db_session, user_id, settings) + + ninjapear = next(s for s in statuses if s["provider"] == "ninjapear") + assert ninjapear["configured"] is True + assert ninjapear["credits"] is None + + +# --- Endpoints --------------------------------------------------------- + + +def test_list_user_api_keys_requires_auth(client: TestClient): + resp = client.get("/api/v1/user-api-keys") + assert resp.status_code == 401 + + +def test_list_and_set_user_api_key_round_trip(client: TestClient): + headers = _register_and_login(client) + + initial = client.get("/api/v1/user-api-keys", headers=headers) + assert initial.status_code == 200 + assert all(not s["configured"] for s in initial.json()) + + set_resp = client.put( + "/api/v1/user-api-keys/anthropic", json={"key": "sk-set-via-api"}, headers=headers + ) + assert set_resp.status_code == 200 + assert set_resp.json()["configured"] is True + assert set_resp.json()["value"] == "sk-set-via-api" + + after = client.get("/api/v1/user-api-keys", headers=headers) + anthropic = next(s for s in after.json() if s["provider"] == "anthropic") + assert anthropic["configured"] is True + assert anthropic["value"] == "sk-set-via-api" + + +def test_updating_your_own_api_key_is_logged_to_account_activity(client: TestClient): + headers = _register_and_login(client) + + client.put("/api/v1/user-api-keys/anthropic", json={"key": "sk-set-via-api"}, headers=headers) + + events = client.get("/api/v1/auth/security-events", headers=headers).json() + assert any(e["event_type"] == "api_key_updated" for e in events) + + +def test_set_user_api_key_rejects_unknown_provider(client: TestClient): + headers = _register_and_login(client) + resp = client.put( + "/api/v1/user-api-keys/not-a-real-provider", json={"key": "x"}, headers=headers + ) + assert resp.status_code == 422 + + +async def test_two_users_keys_are_fully_isolated(client: TestClient, db_session): + headers_a = _register_and_login(client) + headers_b = _register_and_login(client) + + client.put("/api/v1/user-api-keys/anthropic", json={"key": "a-key"}, headers=headers_a) + + b_keys = client.get("/api/v1/user-api-keys", headers=headers_b).json() + anthropic_b = next(s for s in b_keys if s["provider"] == "anthropic") + assert anthropic_b["configured"] is False + assert anthropic_b["value"] is None + + +# --- Provider wiring: the user's own key is actually used ------------------ + + +async def test_discover_endpoint_uses_the_callers_own_anthropic_and_brave_keys( + client: TestClient, db_session, monkeypatch +): + """End-to-end proof this isn't just stored and ignored - the actual + outbound Brave Search call for this request carries the user's own + key, not the server's global one.""" + monkeypatch.setattr("app.core.config.Settings.search_provider", "brave", raising=False) + headers = _register_and_login(client) + user = await UserRepository(db_session).get_by_email( + client.get("/api/v1/auth/me", headers=headers).json()["email"] + ) + settings = get_settings() + await user_api_key_service.set_key( + db_session, + user.id, + ApiKeyProvider.BRAVE_SEARCH, + "my-own-brave-key", + settings, + client_ip="10.0.0.1", + ) + + seen_auth_tokens: list[str] = [] + + def _capture(request: httpx.Request) -> httpx.Response: + seen_auth_tokens.append(request.headers.get("X-Subscription-Token", "")) + return httpx.Response(200, json={"web": {"results": []}}) + + test_settings = get_settings().model_copy(update={"search_provider": "brave"}) + app.dependency_overrides[get_settings] = lambda: test_settings + try: + with respx.mock: + respx.get(url__regex=r"https://api\.search\.brave\.com/.*").mock(side_effect=_capture) + client.post("/api/v1/companies/discover", json={"name": "Acme Corp"}, headers=headers) + finally: + app.dependency_overrides.pop(get_settings, None) + + assert "my-own-brave-key" in seen_auth_tokens diff --git a/apps/api/tests/unit/test_user_known_ips.py b/apps/api/tests/unit/test_user_known_ips.py new file mode 100644 index 0000000..0854046 --- /dev/null +++ b/apps/api/tests/unit/test_user_known_ips.py @@ -0,0 +1,118 @@ +"""Per-account known-IP ledger: pure data capture on every recorded sign-in +(real login and the local-dev bypass), one row per distinct (user, ip) pair, +touched rather than duplicated on repeat visits from the same IP. Nothing +currently reads this data - it's the foundation a later "new IP" security +feature would query against.""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime, timedelta + +from fastapi.testclient import TestClient + +from app.db.base import ensure_aware_utc +from app.main import app +from app.models.user import LOCAL_DEV_USER_ID +from app.repositories.user_known_ip_repository import UserKnownIpRepository +from app.repositories.user_repository import UserRepository + + +def _unique_email() -> str: + return f"user-{uuid.uuid4().hex[:12]}@example.com" + + +def _unique_ip() -> str: + return f"10.{uuid.uuid4().int % 250}.{uuid.uuid4().int % 250}.{uuid.uuid4().int % 250}" + + +# --- Repository -------------------------------------------------------- + + +async def test_record_login_creates_a_row_for_a_new_ip_and_reports_it_as_new(db_session): + repo = UserKnownIpRepository(db_session) + user_id = uuid.uuid4() + now = datetime.now(UTC) + + is_new = await repo.record_login(user_id, "203.0.113.5", now) + assert is_new is True + + row = await repo.get(user_id, "203.0.113.5") + assert row is not None + assert ensure_aware_utc(row.first_seen_at) == now + assert ensure_aware_utc(row.last_seen_at) == now + + +async def test_record_login_touches_last_seen_instead_of_duplicating(db_session): + repo = UserKnownIpRepository(db_session) + user_id = uuid.uuid4() + first_seen = datetime.now(UTC) - timedelta(days=1) + second_visit = datetime.now(UTC) + + await repo.record_login(user_id, "203.0.113.6", first_seen) + is_new = await repo.record_login(user_id, "203.0.113.6", second_visit) + + assert is_new is False + rows = await repo.list_for_user(user_id) + assert len(rows) == 1 + assert ensure_aware_utc(rows[0].first_seen_at) == first_seen + assert ensure_aware_utc(rows[0].last_seen_at) == second_visit + + +async def test_a_second_distinct_ip_creates_a_second_row(db_session): + repo = UserKnownIpRepository(db_session) + user_id = uuid.uuid4() + now = datetime.now(UTC) + + await repo.record_login(user_id, "203.0.113.7", now) + await repo.record_login(user_id, "203.0.113.8", now) + + rows = await repo.list_for_user(user_id) + assert {r.ip_address for r in rows} == {"203.0.113.7", "203.0.113.8"} + + +# --- Wired into real sign-in flows -------------------------------------- + + +async def test_real_login_records_the_client_ip(db_session): + ip = _unique_ip() + email = _unique_email() + with TestClient(app, client=(ip, 51234)) as c: + c.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "T"}, + ) + login_resp = c.post( + "/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"} + ) + assert login_resp.status_code == 200 + + user = await UserRepository(db_session).get_by_email(email) + rows = await UserKnownIpRepository(db_session).list_for_user(user.id) + assert [r.ip_address for r in rows] == [ip] + + +async def test_repeat_login_from_the_same_ip_does_not_duplicate_the_row(db_session): + ip = _unique_ip() + email = _unique_email() + with TestClient(app, client=(ip, 51235)) as c: + c.post( + "/api/v1/auth/register", + json={"email": email, "password": "correct-horse-1", "display_name": "T"}, + ) + c.post("/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"}) + c.post("/api/v1/auth/login", json={"email": email, "password": "correct-horse-1"}) + + user = await UserRepository(db_session).get_by_email(email) + rows = await UserKnownIpRepository(db_session).list_for_user(user.id) + assert len(rows) == 1 + + +async def test_local_dev_sign_in_records_the_known_ip(local_mode_client, db_session): + # local_mode_client's fixed loopback peer (see conftest.py) should show + # up as a known IP for the local-dev account after this request. + resp = local_mode_client.get("/api/v1/auth/me") + assert resp.status_code == 200 + + rows = await UserKnownIpRepository(db_session).list_for_user(LOCAL_DEV_USER_ID) + assert "127.0.0.1" in {r.ip_address for r in rows} diff --git a/apps/web/.eslintrc.json b/apps/web/.eslintrc.json new file mode 100644 index 0000000..12568f6 --- /dev/null +++ b/apps/web/.eslintrc.json @@ -0,0 +1,6 @@ +{ + "extends": ["next/core-web-vitals"], + "rules": { + "@next/next/no-html-link-for-pages": "off" + } +} diff --git a/apps/web/.prettierignore b/apps/web/.prettierignore new file mode 100644 index 0000000..19b5886 --- /dev/null +++ b/apps/web/.prettierignore @@ -0,0 +1,7 @@ +.next/ +node_modules/ +next-env.d.ts +tsconfig.json +coverage/ +playwright-report/ +test-results/ diff --git a/apps/web/.prettierrc.json b/apps/web/.prettierrc.json new file mode 100644 index 0000000..18b9dc5 --- /dev/null +++ b/apps/web/.prettierrc.json @@ -0,0 +1,7 @@ +{ + "semi": true, + "singleQuote": false, + "trailingComma": "all", + "printWidth": 100, + "plugins": ["prettier-plugin-tailwindcss"] +} diff --git a/apps/web/app/(app)/alerts/[id]/page.tsx b/apps/web/app/(app)/alerts/[id]/page.tsx new file mode 100644 index 0000000..e460fd3 --- /dev/null +++ b/apps/web/app/(app)/alerts/[id]/page.tsx @@ -0,0 +1,138 @@ +"use client"; + +import { useParams } from "next/navigation"; +import Link from "next/link"; +import { ArrowLeft, Mail, MessageSquare, Terminal } from "lucide-react"; +import { SeverityBadge } from "@/components/ui/badge"; +import { useAlert, useMarkAlertRead, useResolveAlert } from "@/hooks/use-alerts"; +import { useCompany } from "@/hooks/use-companies"; +import { formatDateTime } from "@/lib/format"; +import type { NotificationDeliveryStatus } from "@/lib/types"; + +const DELIVERY_STATUS_CLASSES: Record = { + sent: "bg-green-100 text-green-800", + failed: "bg-red-100 text-red-800", + pending: "bg-slate-200 text-slate-700", +}; + +const PROVIDER_ICON = { + smtp: Mail, + twilio_sms: MessageSquare, + console: Terminal, +} as const; + +export default function AlertDetailPage() { + const params = useParams<{ id: string }>(); + const { data: alert, isLoading } = useAlert(params.id); + const { data: company } = useCompany(alert?.company_id); + const markRead = useMarkAlertRead(); + const resolve = useResolveAlert(); + + if (isLoading || !alert) { + return

Loading…

; + } + + return ( +
+ + Back to alerts + + +
+
+ + {!alert.read && ( + + Unread + + )} + {alert.resolved && ( + + Resolved + + )} + + {Math.round(alert.confidence * 100)}% confidence + +
+ +

{alert.title}

+

+ {company ? ( + + {company.name} + + ) : ( + "—" + )}{" "} + · {formatDateTime(alert.created_at)} +

+ +
+
+

What changed

+

{alert.summary}

+
+
+

Why it matters

+

{alert.why_it_matters}

+
+
+ +
+ {!alert.read && ( + + )} + {!alert.resolved && ( + + )} +
+
+ +
+

Notification deliveries

+ {alert.deliveries.length === 0 ? ( +

+ No destinations were notified for this alert. +

+ ) : ( +
    + {alert.deliveries.map((delivery) => { + const Icon = + PROVIDER_ICON[delivery.provider as keyof typeof PROVIDER_ICON] ?? Terminal; + return ( +
  • +
    + + {delivery.provider.replace("_", " ")} + {delivery.error_message && ( + {delivery.error_message} + )} +
    + + {delivery.status} + +
    • + ); + })} +
+ )} +
+
+ ); +} diff --git a/apps/web/app/(app)/alerts/page.tsx b/apps/web/app/(app)/alerts/page.tsx new file mode 100644 index 0000000..39347ac --- /dev/null +++ b/apps/web/app/(app)/alerts/page.tsx @@ -0,0 +1,152 @@ +"use client"; + +import Link from "next/link"; +import { useMemo, useState } from "react"; +import { AlertTriangle, Check, CheckCheck } from "lucide-react"; +import { SeverityBadge } from "@/components/ui/badge"; +import { Select } from "@/components/ui/select"; +import { useAlerts, useMarkAlertRead, useResolveAlert } from "@/hooks/use-alerts"; +import { useCompanies } from "@/hooks/use-companies"; +import { formatRelative } from "@/lib/format"; +import { SEVERITY_LABELS, SEVERITY_LEVELS, type SeverityLevel } from "@/lib/types"; + +export default function AlertsPage() { + const [companyId, setCompanyId] = useState(""); + const [severity, setSeverity] = useState(""); + const [unreadOnly, setUnreadOnly] = useState(false); + const [showResolved, setShowResolved] = useState(false); + + const { data: companies } = useCompanies(); + const { data: alerts, isLoading } = useAlerts({ + company_id: companyId || undefined, + severity: severity || undefined, + read: unreadOnly ? false : undefined, + resolved: showResolved ? undefined : false, + }); + const markRead = useMarkAlertRead(); + const resolve = useResolveAlert(); + + const companyNameById = useMemo( + () => new Map((companies ?? []).map((c) => [c.id, c.name])), + [companies], + ); + + const alertList = alerts ?? []; + + return ( +
+
+

Alerts

+

+ Meaningful changes detected across your monitored companies. +

+
+ +
+ setSeverity(v as SeverityLevel | "")} + placeholder="All severities" + ariaLabel="Filter by severity" + options={[ + { value: "", label: "All severities" }, + ...SEVERITY_LEVELS.map((s) => ({ value: s, label: SEVERITY_LABELS[s] })), + ]} + triggerClassName="px-3 py-1.5 text-sm" + /> + + + + +
+ + {isLoading ? ( +

Loading…

+ ) : alertList.length === 0 ? ( +
+ +

No alerts match these filters.

+
+ ) : ( +
    + {alertList.map((alert) => ( +
  • +
    +
    + + {!alert.read && ( + + )} + {alert.resolved && ( + Resolved + )} +
    + + {alert.title} + +

    + {companyNameById.get(alert.company_id) ?? "Unknown company"} ·{" "} + {formatRelative(alert.created_at)} · {Math.round(alert.confidence * 100)}% + confidence +

    +
    +
    + {!alert.read && ( + + )} + {!alert.resolved && ( + + )} +
    +
    • + ))} +
+ )} +
+ ); +} diff --git a/apps/web/app/(app)/companies/[id]/page.tsx b/apps/web/app/(app)/companies/[id]/page.tsx new file mode 100644 index 0000000..11faa08 --- /dev/null +++ b/apps/web/app/(app)/companies/[id]/page.tsx @@ -0,0 +1,1269 @@ +"use client"; + +import Link from "next/link"; +import { useRouter, useSearchParams } from "next/navigation"; +import { Suspense, use, useState } from "react"; +import { + AlertTriangle, + Check, + CheckCheck, + ChevronDown, + ChevronRight, + Loader2, + Plus, +} from "lucide-react"; +import { CompanyPillLink } from "@/components/ui/company-pill"; +import { CopyableEmail } from "@/components/ui/copyable-email"; +import { CompanyStatusBadge, SeverityBadge } from "@/components/ui/badge"; +import { NotificationChannelBox } from "@/components/ui/notification-channel-box"; +import { Select } from "@/components/ui/select"; +import { useAlerts, useMarkAlertRead, useResolveAlert } from "@/hooks/use-alerts"; +import { useSystemStatus } from "@/hooks/use-auth"; +import { + useCompanies, + useCompany, + useDeleteCompany, + useSetCompanyPaused, + useUpdateMonitorConfiguration, +} from "@/hooks/use-companies"; +import { useNotificationDestinations } from "@/hooks/use-notification-destinations"; +import { + useCreateSource, + useDeleteSource, + useSources, + useTestSource, + useUpdateSource, +} from "@/hooks/use-sources"; +import { useCompanyRuns, useRunCompanyNow } from "@/hooks/use-monitoring-runs"; +import { useCompanyReports, useGenerateReport } from "@/hooks/use-reports"; +import { useSnapshots } from "@/hooks/use-snapshots"; +import type { MonitoringFrequency } from "@/lib/types"; +import { + FREQUENCY_LABELS, + MONITORING_FREQUENCIES, + SEVERITY_LABELS, + SEVERITY_LEVELS, +} from "@/lib/types"; +import { + companyNameFromUrl, + formatDateTime, + formatEnrichmentReason, + formatMoney, + formatRelative, + summarizeEnrichmentError, +} from "@/lib/format"; + +const TABS = [ + "Overview", + "Enrichment", + "Latest report", + "Alerts", + "Sources", + "Monitoring history", + "Snapshots", + "Configuration", +] as const; +type Tab = (typeof TABS)[number]; + +const TAB_DESCRIPTIONS: Record = { + Overview: "A snapshot of this company — description, competitors, key details, and its monitoring schedule.", + Enrichment: "Rich company data — funding, leadership, products, and customers.", + "Latest report": + "The most recent AI-generated competitive intelligence report, grounded in everything collected so far.", + Alerts: + "Notable changes detected for this company, ranked by severity, with actions to mark them read or resolved.", + Sources: "Every data source being monitored for this company, and how often each one is checked.", + "Monitoring history": + "A log of every monitoring run for this company — what it collected and whether it succeeded.", + Snapshots: + "The raw content collected from each source on each run, so you can see exactly what was captured.", + Configuration: "Scheduling, severity threshold, and notification settings for this company.", +}; + +function charCount(text: string | null): string { + return `${(text ?? "").length.toLocaleString()} chars`; +} + +const RUN_STATUS_CLASSES: Record = { + queued: "bg-slate-100 text-slate-700", + running: "bg-blue-100 text-blue-700", + successful: "bg-green-100 text-green-700", + partial: "bg-amber-100 text-amber-700", + failed: "bg-red-100 text-red-700", +}; + +const ERROR_TONE_CLASSES = { + amber: { + box: "border-amber-200 bg-amber-50 text-amber-800", + button: "border-amber-300 text-amber-800 hover:bg-amber-100", + }, + red: { + box: "border-red-200 bg-red-50 text-red-800", + button: "border-red-300 text-red-800 hover:bg-red-100", + }, +} as const; + +function EnrichmentErrorSummary({ + tone, + summary, + errors, +}: { + tone: keyof typeof ERROR_TONE_CLASSES; + summary: string; + errors: Record; +}) { + const [expanded, setExpanded] = useState(false); + const sections = Object.entries(errors); + const classes = ERROR_TONE_CLASSES[tone]; + + return ( +
+
+

{summary}

+ {sections.length > 0 && ( + + )} +
+
+
+
    + {sections.map(([section, raw]) => ( +
  • +

    {section}

    +

    {summarizeEnrichmentError(raw)}

    + 
    {raw}
    +
    • + ))} +
+
+
+
+ ); +} + +export default function CompanyDetailPage({ params }: { params: Promise<{ id: string }> }) { + const { id } = use(params); + return ( + Loading…

}> + + + ); +} + +function CompanyDetailPageInner({ id }: { id: string }) { + const router = useRouter(); + const searchParams = useSearchParams(); + const requestedTab = searchParams.get("tab"); + const initialTab = (TABS as readonly string[]).includes(requestedTab ?? "") + ? (requestedTab as Tab) + : "Overview"; + + const { data: company, isLoading } = useCompany(id); + const { data: allCompanies } = useCompanies(); + const setPaused = useSetCompanyPaused(); + const deleteCompany = useDeleteCompany(); + const updateMonitor = useUpdateMonitorConfiguration(id); + const { data: sources, isLoading: sourcesLoading } = useSources(id); + const createSource = useCreateSource(id); + const deleteSource = useDeleteSource(id); + const testSource = useTestSource(id); + const updateSource = useUpdateSource(id); + const { data: runs, isLoading: runsLoading } = useCompanyRuns(id); + const runNow = useRunCompanyNow(id); + const { data: reports, isLoading: reportsLoading } = useCompanyReports(id); + const generateReport = useGenerateReport(id); + const { data: snapshots, isLoading: snapshotsLoading } = useSnapshots(id); + const { data: alerts, isLoading: alertsLoading } = useAlerts({ company_id: id }); + const markAlertRead = useMarkAlertRead(); + const resolveAlert = useResolveAlert(); + const { data: notificationDestinations } = useNotificationDestinations(); + const { data: systemStatus } = useSystemStatus(); + const [tab, setTab] = useState(initialTab); + const [confirmingDelete, setConfirmingDelete] = useState(false); + const [newSourceName, setNewSourceName] = useState(""); + const [newSourceUrl, setNewSourceUrl] = useState(""); + const [testResults, setTestResults] = useState>({}); + const [expandedSnapshots, setExpandedSnapshots] = useState>(new Set()); + const [frequencyOverride, setFrequencyOverride] = useState(null); + const [severityOverride, setSeverityOverride] = useState(null); + + if (isLoading) { + return

Loading…

; + } + + if (!company) { + return

Company not found.

; + } + + const config = company.monitor_configuration; + // "Running" covers the whole onboarding/run chain, not just the + // monitoring-run row itself: enrichment is a separate fire-and-forget + // task fired at company creation, and its own tab shouldn't look done + // (nor should Pause/Delete be usable) while it's still in flight either. + const isRunning = + (runs?.some((r) => r.status === "queued" || r.status === "running") ?? false) || + company.enrichment?.status === "pending"; + const returnTo = `/companies/${id}?tab=${encodeURIComponent(tab)}`; + + const monitoredByName = new Map( + (allCompanies ?? []) + .filter((c) => c.id !== company.id) + .map((c) => [c.name.trim().toLowerCase(), c.id] as const), + ); + + const existingCompetitorKeys = new Set(company.competitors.map((c) => c.trim().toLowerCase())); + const enrichmentCompetitors = + (company.enrichment?.data.competitors as { name?: string }[] | undefined) ?? []; + const suggestedCompetitors = [ + ...new Set( + enrichmentCompetitors + .map((c) => companyNameFromUrl((c.name ?? "").trim())) + .filter((name) => name && !existingCompetitorKeys.has(name.toLowerCase())), + ), + ]; + + const companyDestinations = (notificationDestinations ?? []).filter((d) => + d.companies.some((c) => c.id === company.id), + ); + const emailDestination = companyDestinations.find((d) => d.type === "email"); + const smsDestination = companyDestinations.find((d) => d.type === "sms"); + + return ( +
+ + ← Back to companies + + +
+
+
+

{company.name}

+ +
+ {company.official_website && ( + + {company.official_website} + + )} +
+
+ + +
+
+
+ Delete this company? + + +
+
+
+
+
+ +
+
+
+
+ +
+ +
+ +
+

+ {TAB_DESCRIPTIONS[tab]} +

+ + {tab === "Overview" && ( +
+
+
+

Monitoring focus

+

+ {company.monitoring_focus || "No specific focus provided — general monitoring."} +

+
+ {company.description && ( +
+

Description

+

{company.description}

+
+ )} +
+

Competitors

+ {company.competitors.length === 0 && suggestedCompetitors.length === 0 ? ( +

None listed.

+ ) : ( +
    + {company.competitors.map((c) => ( +
  • + +
    • + ))} + {suggestedCompetitors.map((c) => ( +
  • + +
    • + ))} +
+ )} +
+
+ +
+ )} + + {tab === "Enrichment" && ( +
+ {!company.enrichment ? ( +
+ Company enrichment (NinjaPear) isn't configured for this app — no data + provider key is set. +
+ ) : company.enrichment.status === "pending" ? ( +
+ Enriching this company from external data sources… this can take a few minutes. +
+ ) : ( + <> + {company.enrichment.status === "partial" && ( + + )} + {company.enrichment.status === "failed" && ( + + )} + +
+
+

Company snapshot

+
+
+
Employees
+
{String(company.enrichment.data.employee_count ?? "—")}
+
+
+
Founded
+
{String(company.enrichment.data.founded_year ?? "—")}
+
+
+ {Array.isArray(company.enrichment.data.specialties) && + (company.enrichment.data.specialties as string[]).length > 0 && ( + <> +

+ Specialties +

+
    + {(company.enrichment.data.specialties as string[]).map((s) => ( +
  • + {s} +
    • + ))} +
+ + )} +
+ +
+

Funding

+ {(() => { + const funding = company.enrichment.data.funding as + | { total_raised?: string; rounds?: Record[] } + | undefined; + if (!funding || (!funding.total_raised && !funding.rounds?.length)) { + return

No funding data found.

; + } + return ( + <> + {funding.total_raised && ( +

+ Total raised:{" "} + + {formatMoney(funding.total_raised)} + +

+ )} +
    + {(funding.rounds ?? []).map((r, i) => ( +
  • +

    + {String(r.round_name ?? "Round")} —{" "} + {r.amount ? formatMoney(String(r.amount)) : "—"} + {r.date ? ` (${String(r.date)})` : ""} +

    + {Array.isArray(r.investors) && + (r.investors as string[]).length > 0 && ( +

    + Investors: {(r.investors as string[]).join(", ")} +

    + )} +
    • + ))} +
+ + ); + })()} +
+
+ +
+

Leadership team

+ {!Array.isArray(company.enrichment.data.leadership_team) || + (company.enrichment.data.leadership_team as unknown[]).length === 0 ? ( +

No leadership team data found.

+ ) : ( +
    + {( + company.enrichment.data.leadership_team as Record[] + ).map((m, i) => ( +
  • + {i > 0 &&
    } +
    +
    +

    {String(m.name ?? "")}

    + {m.title ? ( +

    {String(m.title)}

    + ) : null} +
    +
    + {m.work_email ? : null} + {m.work_email && m.profile_url ? ( + - + ) : null} + {m.profile_url ? ( + + Profile + + ) : null} +
    +
    +
    • + ))} +
+ )} +
+ +
+
+

Customers

+ {!Array.isArray(company.enrichment.data.customers) || + (company.enrichment.data.customers as unknown[]).length === 0 ? ( +

None found.

+ ) : ( +
    + {(company.enrichment.data.customers as Record[]).map( + (c, i) => { + const name = String(c.name ?? ""); + if (!name) return null; + return ( +
  • + +
    • + ); + }, + )} +
+ )} +
+ +
+

Products

+ {!Array.isArray(company.enrichment.data.products) || + (company.enrichment.data.products as unknown[]).length === 0 ? ( +

None found.

+ ) : ( +
    + {(company.enrichment.data.products as Record[]).map( + (p, i) => ( +
  • + {String(p.name ?? "")} + {p.description ? ( + + {" "} + — {String(p.description)} + + ) : null} +
    • + ), + )} +
+ )} +
+
+ +
+
+

Recent updates

+ {!Array.isArray(company.enrichment.data.recent_updates) || + (company.enrichment.data.recent_updates as unknown[]).length === 0 ? ( +

None found.

+ ) : ( +
    + {( + company.enrichment.data.recent_updates as Record[] + ).map((u, i) => ( +
  • + {i > 0 &&
    } +

    {String(u.text ?? "")}

    + {u.url ? ( + + {String(u.url)} + + ) : null} +
    • + ))} +
+ )} +
+ +
+

+ Competitors{" "} + (API-suggested) +

+ {!Array.isArray(company.enrichment.data.competitors) || + (company.enrichment.data.competitors as unknown[]).length === 0 ? ( +

None found.

+ ) : ( +
    + {( + company.enrichment.data.competitors as Record[] + ).map((c, i) => { + const name = companyNameFromUrl(String(c.name ?? "")); + if (!name) return null; + return ( +
  • + +
    • + ); + })} +
+ )} +
+
+ + {company.enrichment.credits_spent !== null && ( +

+ {company.enrichment.credits_spent} NinjaPear Credits spent fetching this data + {company.enrichment.fetched_at + ? ` on ${formatDateTime(company.enrichment.fetched_at)}` + : ""} + . +

+ )} + + )} +
+ )} + + {tab === "Alerts" && ( +
+ {alertsLoading ? ( +

Loading…

+ ) : !alerts || alerts.length === 0 ? ( +
+ +

No alerts for this company yet.

+
+ ) : ( +
    + {alerts.map((alert) => ( +
  • +
    +
    + + {!alert.read && ( + + )} + {alert.resolved && ( + Resolved + )} +
    + + {alert.title} + +

    + {formatRelative(alert.created_at)} · {Math.round(alert.confidence * 100)}% + confidence +

    +
    +
    + {!alert.read && ( + + )} + {!alert.resolved && ( + + )} +
    +
    • + ))} +
+ )} +
+ )} + + {tab === "Configuration" && config && ( +
+

Monitoring configuration

+
{ + e.preventDefault(); + const form = new FormData(e.currentTarget); + updateMonitor.mutate({ + frequency_type: form.get("frequency_type") as never, + severity_threshold: form.get("severity_threshold") as never, + timezone: String(form.get("timezone")), + }); + }} + > +
+ +
+ ({ value: s, label: SEVERITY_LABELS[s] }))} + triggerClassName="w-full px-3 py-2 text-sm" + /> +
+
+
+ + +
+ + {updateMonitor.isSuccess && ( +

Configuration updated.

+ )} +
+
+ )} + + {tab === "Configuration" && config && ( +
+

Notifications

+

+ Where alerts for this company get sent. Managing them here also updates the + Settings page, and vice versa. +

+
+ + +
+
+ )} + + {tab === "Latest report" && ( +
+
+

+ {reports && reports.length > 0 + ? `${reports.length} report(s) generated for this company.` + : "No reports yet."} +

+ +
+ + {!runsLoading && (!runs || runs.length === 0) && ( +

+ No monitoring run has collected any evidence yet, so a report generated now will + come back mostly empty by design — this app never fabricates findings without real + evidence. Click "Run now" above first, then generate the report. +

+ )} + + {reportsLoading ? ( +

Loading…

+ ) : !reports || reports.length === 0 || !reports[0] ? ( +
+ No reports yet. Run a collection or click "Generate report now". +
+ ) : ( +
+
+ + View latest report ({reports[0].report_type},{" "} + {formatDateTime(reports[0].created_at)}) → + +

{reports[0].executive_summary}

+
+ {reports.length > 1 && ( +
    + {reports.slice(1).map((r) => ( +
  • + + {r.title} — {r.report_type} + + + {formatDateTime(r.created_at)} + +
    • + ))} +
+ )} +
+ )} +
+ )} + + {tab === "Sources" && ( +
+
+

Add a custom URL source

+

+ Website, GitHub, SEC EDGAR, and careers-page sources are found automatically the + first time this company runs. Add any other public URL or RSS feed here to + monitor it too. +

+
{ + e.preventDefault(); + if (!newSourceName || !newSourceUrl) return; + createSource.mutate( + { source_type: "custom_url", name: newSourceName, base_url: newSourceUrl }, + { + onSuccess: () => { + setNewSourceName(""); + setNewSourceUrl(""); + }, + }, + ); + }} + > +
+ + setNewSourceName(e.target.value)} + className="focus-ring mt-1 rounded-md border border-slate-300 px-3 py-2 text-sm" + placeholder="Pricing page" + /> +
+
+ + setNewSourceUrl(e.target.value)} + className="focus-ring mt-1 block w-full rounded-md border border-slate-300 px-3 py-2 text-sm" + placeholder="example.com/pricing" + /> +
+ +
+
+ +
+ {sourcesLoading ? ( +

Loading…

+ ) : !sources || sources.length === 0 ? ( +

+ {runs && runs.length > 0 + ? "No sources configured yet." + : "Website, GitHub, and other sources are discovered automatically on the " + + 'first monitoring run — click "Run now" above to get started, or add a ' + + "custom URL yourself above."} +

+ ) : ( + + + + + + + + + + + + + {sources.map((source) => { + const isTesting = + testSource.isPending && testSource.variables === source.id; + const isDeleting = + deleteSource.isPending && deleteSource.variables === source.id; + return ( + + + + + + + + + ); + })} + +
NameTypeStatusLast checkedCheck frequencyActions
+ {source.name} + {source.base_url && ( +
{source.base_url}
+ )} + 		{source.source_type}{source.status} + {formatDateTime(source.last_checked)} + + +
+ + +
+ {testResults[source.id] && ( +

+ {testResults[source.id]} +

+ )} +
+ )} +
+
+ )} + + {tab === "Monitoring history" && ( +
+ {runsLoading ? ( +

Loading…

+ ) : !runs || runs.length === 0 ? ( +

+ No monitoring runs yet. Click "Run now" above to start one. +

+ ) : ( + + + + + + + + + + + + + {runs.map((run) => ( + + + + + + + + + ))} + +
StartedTriggerStatusSourcesItems collectedNotes
+ {formatDateTime(run.started_at ?? run.created_at)} + {run.trigger_type} + + {run.status} + + + {run.sources_successful}/{run.sources_attempted} succeeded + {run.items_collected} + {run.error_summary ?? "—"} +
+ )} +
+ )} + + {tab === "Snapshots" && ( +
+ {snapshotsLoading ? ( +

Loading…

+ ) : !snapshots || snapshots.length === 0 ? ( +

+ {runs && runs.length > 0 + ? "No snapshots recorded yet." + : 'No snapshots yet — snapshots are captured during monitoring runs. Click "Run now" above to get started.'} +

+ ) : ( +
    + {snapshots.map((snapshot) => { + const isExpanded = expandedSnapshots.has(snapshot.id); + const sourceName = + sources?.find((s) => s.id === snapshot.source_id)?.name ?? "Unknown source"; + return ( +
  • + + {isExpanded && ( +
    +
    +

    + Hash +

    +

    + {snapshot.hash} +

    +
    + {snapshot.text_summary && ( +
    +

    + Text summary +

    +

    + {snapshot.text_summary} +

    +
    + )} + {Object.keys(snapshot.structured_summary).length > 0 && ( +
    +

    + Structured summary +

    + 
    +                                {JSON.stringify(snapshot.structured_summary, null, 2)}
+
    +
    + )} +
    + )} +
    • + ); + })} +
+ )} +
+ )} +
+
+ ); +} diff --git a/apps/web/app/(app)/companies/new/page.tsx b/apps/web/app/(app)/companies/new/page.tsx new file mode 100644 index 0000000..7c5c0c3 --- /dev/null +++ b/apps/web/app/(app)/companies/new/page.tsx @@ -0,0 +1,680 @@ +"use client"; + +import { zodResolver } from "@hookform/resolvers/zod"; +import Link from "next/link"; +import { useRouter, useSearchParams } from "next/navigation"; +import { Suspense, useState } from "react"; +import { Controller, useForm } from "react-hook-form"; +import { Loader2 } from "lucide-react"; +import { z } from "zod"; +import { FormField } from "@/components/ui/form-field"; +import { Select } from "@/components/ui/select"; +import { api } from "@/lib/api-client"; +import { useCompanies, useCreateCompany } from "@/hooks/use-companies"; +import { useCreateNotificationDestination } from "@/hooks/use-notification-destinations"; +import { useDiscoverCompany } from "@/hooks/use-discovery"; +import { authErrorMessage, useSystemStatus } from "@/hooks/use-auth"; +import { + FREQUENCY_LABELS, + MONITORING_FREQUENCIES, + SEVERITY_LABELS, + SEVERITY_LEVELS, + type CompanyResponse, + type DiscoveredCompanyProfile, + type MonitoringFrequency, + type SeverityLevel, +} from "@/lib/types"; + +const _LEGAL_SUFFIX_RE = /\b(inc|incorporated|llc|ltd|limited|corp|corporation|co|company|plc)\b/g; + +function normalizeCompanyName(name: string): string { + return name + .toLowerCase() + .replace(/[.,]/g, "") + .replace(_LEGAL_SUFFIX_RE, "") + .replace(/\s+/g, " ") + .trim(); +} + +function findPossibleDuplicate( + name: string, + companies: CompanyResponse[], +): CompanyResponse | undefined { + const normalized = normalizeCompanyName(name); + if (!normalized) return undefined; + return companies.find((c) => { + const existing = normalizeCompanyName(c.name); + return ( + existing.length > 0 && + (existing === normalized || existing.includes(normalized) || normalized.includes(existing)) + ); + }); +} + +const companySchema = z + .object({ + name: z.string().min(1, "Company name is required").max(200), + officialWebsite: z.string().max(500).optional().or(z.literal("")), + industry: z.string().max(120).optional().or(z.literal("")), + country: z.string().max(120).optional().or(z.literal("")), + region: z.string().max(120).optional().or(z.literal("")), + headquarters: z.string().max(200).optional().or(z.literal("")), + description: z.string().max(4000).optional().or(z.literal("")), + monitoringFocus: z.string().max(2000).optional().or(z.literal("")), + competitorNames: z.string().optional().or(z.literal("")), + aliasNames: z.string().optional().or(z.literal("")), + frequencyType: z.enum(MONITORING_FREQUENCIES), + intervalMinutes: z.coerce.number().int().positive().optional(), + cronExpression: z.string().max(120).optional().or(z.literal("")), + timezone: z.string().min(1).max(64), + severityThreshold: z.enum(SEVERITY_LEVELS), + notificationEmail: z.string().email("Enter a valid email address"), + consent: z.literal(true, { + errorMap: () => ({ message: "You must acknowledge the collection policy to continue" }), + }), + }) + .refine( + (data) => + data.frequencyType !== "custom" || + Boolean(data.intervalMinutes) || + Boolean(data.cronExpression), + { + message: "Provide either an interval in minutes or a cron expression for a custom schedule", + path: ["intervalMinutes"], + }, + ); + +type CompanyForm = z.infer; + +const STEPS = ["Discover", "Review", "Schedule", "Notifications", "Confirm"] as const; + +const STEP_FIELDS: Record = { + 0: ["name"], + 1: [], + 2: ["frequencyType", "intervalMinutes", "cronExpression", "timezone"], + 3: ["severityThreshold", "notificationEmail", "consent"], + 4: [], +}; + +function splitNames(value: string | undefined): string[] { + return (value ?? "") + .split(",") + .map((v) => v.trim()) + .filter(Boolean); +} + +const MONITORING_FOCUS_PLACEHOLDER = + "E.g. new product or pricing changes, leadership hires/departures, hiring trends, " + + "funding/financial signals, partnerships and acquisitions, patents, manufacturing or " + + "expansion moves, regulatory or legal news, and how they compare to competitors."; + +function AddCompanyWizard() { + const router = useRouter(); + const searchParams = useSearchParams(); + const prefillName = searchParams.get("name") ?? ""; + const returnTo = searchParams.get("returnTo"); + const [step, setStep] = useState(0); + const [discovery, setDiscovery] = useState(null); + const [isFinalizing, setIsFinalizing] = useState(false); + const [duplicateWarning, setDuplicateWarning] = useState<{ + company: CompanyResponse; + forName: string; + } | null>(null); + const [acknowledgedDuplicateFor, setAcknowledgedDuplicateFor] = useState(null); + const { data: allCompanies } = useCompanies(); + const createCompany = useCreateCompany(); + const createDestination = useCreateNotificationDestination(); + const discoverCompany = useDiscoverCompany(); + const { data: systemStatus } = useSystemStatus(); + + const { + register, + handleSubmit, + control, + watch, + setValue, + getValues, + trigger, + formState: { errors }, + } = useForm({ + resolver: zodResolver(companySchema), + defaultValues: { + name: prefillName, + frequencyType: "weekly", + timezone: "America/New_York", + severityThreshold: "medium", + }, + }); + + const frequencyType = watch("frequencyType"); + + const goNext = async () => { + const valid = await trigger(STEP_FIELDS[step]); + if (valid) setStep((s) => Math.min(s + 1, STEPS.length - 1)); + }; + const goBack = () => { + if (step === 0) { + // Nothing earlier in the wizard to step back to - leave to wherever + // the user actually came from (Dashboard, Companies, or a rival + // company's page via the "add this competitor" link). A `returnTo` + // param (set by the company page's competitor/customer pills) is + // preferred over plain browser-history back, since a client + // component's local state (e.g. which tab was open) doesn't reliably + // survive a back-navigation in the App Router. + if (returnTo) { + router.push(returnTo); + return; + } + router.back(); + return; + } + setStep((s) => Math.max(s - 1, 0)); + }; + + const handleDiscover = async () => { + const valid = await trigger(["name"]); + if (!valid) return; + + const values = getValues(); + + if (acknowledgedDuplicateFor !== values.name) { + const possibleDuplicate = findPossibleDuplicate(values.name, allCompanies ?? []); + if (possibleDuplicate) { + setDuplicateWarning({ company: possibleDuplicate, forName: values.name }); + return; + } + } + setDuplicateWarning(null); + + const profile = await discoverCompany.mutateAsync({ + name: values.name, + official_website: values.officialWebsite || undefined, + monitoring_focus: values.monitoringFocus || undefined, + competitor_names: splitNames(values.competitorNames), + alias_names: splitNames(values.aliasNames), + }); + + setDiscovery(profile); + setValue("officialWebsite", profile.official_website ?? ""); + setValue("description", profile.description ?? ""); + setValue("industry", profile.industry ?? ""); + setValue("country", profile.country ?? ""); + setValue("region", profile.region ?? ""); + setValue("headquarters", profile.headquarters ?? ""); + setValue("monitoringFocus", profile.monitoring_focus ?? values.monitoringFocus ?? ""); + setValue("competitorNames", profile.competitors.join(", ")); + setValue("aliasNames", profile.aliases.join(", ")); + setStep(1); + }; + + const onSubmit = handleSubmit(async (values) => { + setIsFinalizing(true); + try { + const company = await createCompany.mutateAsync({ + name: values.name, + official_website: values.officialWebsite || null, + industry: values.industry || null, + country: values.country || null, + region: values.region || null, + headquarters: values.headquarters || null, + public_identifiers: discovery?.public_identifiers ?? {}, + description: values.description || null, + monitoring_focus: values.monitoringFocus || null, + competitor_names: splitNames(values.competitorNames), + alias_names: splitNames(values.aliasNames), + frequency_type: values.frequencyType, + interval_minutes: + values.frequencyType === "custom" ? (values.intervalMinutes ?? null) : null, + cron_expression: values.frequencyType === "custom" ? values.cronExpression || null : null, + timezone: values.timezone, + severity_threshold: values.severityThreshold, + }); + + await createDestination + .mutateAsync({ + type: "email", + destination_value: values.notificationEmail, + company_ids: [company.id], + }) + .catch(() => undefined); // Best-effort; company creation still succeeded either way. + + // Best-effort, same as the destination above - a failed enqueue here + // shouldn't block landing on the new company's page, it just means + // the user sees an idle Run Now button instead of the running state. + await api.runCompanyNow(company.id).catch(() => undefined); + + router.push(`/companies/${company.id}`); + } catch { + setIsFinalizing(false); + } + }); + + return ( +
+

Add a company

+

+ Give us a name — we'll find the rest and let you correct anything before we start + monitoring. +

+ +
    + {STEPS.map((label, i) => ( +
  1. + + {i + 1} + + {label} + {i < STEPS.length - 1 && /} +
    2. + ))} +
+ +
+
+ {step === 0 && ( +
+ + + {duplicateWarning && duplicateWarning.forName === watch("name") && ( +
+

+ You might already be monitoring{" "} + "{duplicateWarning.company.name}". Adding another + with a similar name is fine, but double-check it's not the same company. +

+
+ + View existing company + + +
+
+ )} + +
+

+ Optional — helps discovery be more accurate +

+
+ +
+ +