Initial commit: CI Agent competitive-intelligence monitoring app
FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification).
This commit is contained in:
@@ -0,0 +1,178 @@
|
||||
"""Centralized application configuration.
|
||||
|
||||
Every environment-dependent value is read here, once, via pydantic-settings.
|
||||
Application code should depend on `get_settings()`, never on `os.environ`
|
||||
directly - that's what keeps provider selection (LLM/search/notifications/auth)
|
||||
swappable from a single place.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
|
||||
|
||||
# --- App ---
|
||||
app_env: Literal["development", "test", "production"] = "development"
|
||||
app_name: str = "CI Agent"
|
||||
frontend_url: str = "http://localhost:3000"
|
||||
backend_url: str = "http://localhost:8000"
|
||||
|
||||
# --- Reverse proxy ---
|
||||
# Empty (default) = trust only the direct TCP connection for client-IP
|
||||
# resolution (app.core.security.get_client_ip) - correct today, no proxy
|
||||
# exists. Set to "CF-Connecting-IP" once deployed behind Cloudflare's
|
||||
# proxy so IP-based localhost detection and the ban/throttle system read
|
||||
# the real visitor IP instead of the proxy's own address. Only ever set
|
||||
# this when it's actually known a trusted proxy sits in front and strips
|
||||
# this header from untrusted clients - see KNOWN_LIMITATIONS.md.
|
||||
trusted_proxy_ip_header: str = ""
|
||||
|
||||
# --- Local-dev convenience ---
|
||||
# Comma-separated extra IPs that `is_localhost` treats as equivalent to
|
||||
# real loopback, on top of 127.0.0.1/::1. Needed because Docker
|
||||
# Desktop's bridge networking means even traffic that originates on the
|
||||
# host machine itself (dev tooling driving a browser against
|
||||
# localhost:3000/8000) arrives at the container from the bridge
|
||||
# gateway address, not literal loopback - so without this,
|
||||
# is_localhost is always false for that traffic, which both re-exposes
|
||||
# the admin-only API-keys/logs boxes' loopback gate and forces the
|
||||
# Turnstile widget to render (a real crash risk for automated browser
|
||||
# tooling - see KNOWN_LIMITATIONS.md). Empty by default (no bypass);
|
||||
# only ever populate this with IPs you know are your own dev host,
|
||||
# never in a real deployment.
|
||||
additional_trusted_local_ips: str = ""
|
||||
|
||||
# --- Auth ---
|
||||
auth_mode: Literal["local", "jwt"] = "local"
|
||||
jwt_secret: str = "dev-only-change-me-32-characters-minimum"
|
||||
jwt_access_token_minutes: int = 15
|
||||
jwt_refresh_token_days: int = 7
|
||||
# Encrypts each user's own stored API keys at rest (app/core/crypto.py)
|
||||
# - a Fernet key (44-char urlsafe-base64). This dev-only default is
|
||||
# fixed/insecure by design (same precedent as jwt_secret above); a real
|
||||
# deployment must set its own via `Fernet.generate_key()`. Rotating
|
||||
# this value makes every already-stored user key undecryptable, so
|
||||
# treat it like any other production secret - never regenerate it
|
||||
# casually once real keys exist.
|
||||
api_key_encryption_secret: str = "_wYtsm3nJ070987snBFp2eWVI5pyC0H9gGFUb6Cy4cQ="
|
||||
|
||||
# --- Database ---
|
||||
database_url: str = "sqlite+aiosqlite:///./ciagent_dev.db"
|
||||
|
||||
# --- Redis / Celery ---
|
||||
redis_url: str = "redis://localhost:6379/0"
|
||||
celery_task_always_eager: bool = False
|
||||
|
||||
# --- LLM ---
|
||||
llm_provider: Literal["mock", "anthropic", "ollama", "gemini"] = "mock"
|
||||
anthropic_api_key: str = ""
|
||||
anthropic_model: str = "claude-sonnet-5"
|
||||
ollama_base_url: str = "http://localhost:11434"
|
||||
ollama_model: str = "llama3.1"
|
||||
# Gemini has an actual free rate-limited tier (unlike OpenAI's expiring
|
||||
# trial credits), so it's the production option this app ships wired up.
|
||||
gemini_api_key: str = ""
|
||||
gemini_model: str = "gemini-2.0-flash"
|
||||
llm_max_tokens_per_request: int = 4000
|
||||
llm_max_retries: int = 2
|
||||
|
||||
# --- Search ---
|
||||
search_provider: Literal["mock", "brave"] = "mock"
|
||||
brave_search_api_key: str = ""
|
||||
serpapi_api_key: str = ""
|
||||
bing_search_api_key: str = ""
|
||||
|
||||
# --- Patents ---
|
||||
# Free key via account registration at data.uspto.gov/apis/getting-started.
|
||||
# Unset by default - PatentSourceCollector falls back to its existing
|
||||
# honest disabled/fixture behavior when this is empty.
|
||||
uspto_api_key: str = ""
|
||||
|
||||
# --- Company enrichment (NinjaPear / nubela.co) ---
|
||||
# Paid, per-credit API - unset by default. Enrichment only ever fires
|
||||
# once, at company-creation time (never on a recurring schedule), and
|
||||
# the enqueue itself is skipped entirely when this is empty - see
|
||||
# company_service.create_company.
|
||||
ninjapear_api_key: str = ""
|
||||
ninjapear_max_leadership_lookups: int = 5
|
||||
|
||||
# --- Email ---
|
||||
smtp_host: str = "localhost"
|
||||
smtp_port: int = 1025
|
||||
smtp_username: str = ""
|
||||
smtp_password: str = ""
|
||||
smtp_from_email: str = "[email protected]"
|
||||
smtp_use_tls: bool = False
|
||||
|
||||
# --- Resend (transactional security email: verification/reset/lockout) ---
|
||||
# Unset by default - security_email_service falls back to the SMTP
|
||||
# provider above (Mailpit locally) when this is empty, so the whole
|
||||
# verification/reset flow is testable with zero Resend account needed.
|
||||
# Deliberately separate from the alert-notification path (smtp_from_email
|
||||
# above) - a different sender identity for account-security mail.
|
||||
resend_api_key: str = ""
|
||||
resend_security_from_email: str = "[email protected]"
|
||||
|
||||
# --- Cloudflare Turnstile (CAPTCHA on register/login/password-reset) ---
|
||||
# Unset by default - skipped entirely for register/login/password-reset
|
||||
# when either the caller is on loopback (see is_localhost) or no secret
|
||||
# is configured, matching this app's usual optional-provider convention.
|
||||
turnstile_site_key: str = ""
|
||||
turnstile_secret: str = ""
|
||||
|
||||
# --- SMS ---
|
||||
notification_sms_enabled: bool = False
|
||||
sms_provider: Literal["twilio", "telnyx"] = "twilio"
|
||||
twilio_account_sid: str = ""
|
||||
twilio_auth_token: str = ""
|
||||
twilio_from_number: str = ""
|
||||
telnyx_api_key: str = ""
|
||||
telnyx_from_number: str = ""
|
||||
sms_monthly_cap: int = 50
|
||||
|
||||
# --- GitHub ---
|
||||
github_token: str = ""
|
||||
|
||||
# --- Scheduling ---
|
||||
default_timezone: str = "America/New_York"
|
||||
default_monitoring_frequency: str = "weekly"
|
||||
minimum_monitoring_interval_minutes: int = 60
|
||||
|
||||
# --- Scraper ---
|
||||
scraper_user_agent: str = "CIAgentBot/1.0 (+https://ci-agent.local/bot)"
|
||||
max_pages_per_domain: int = 25
|
||||
scraper_request_timeout_seconds: int = 30
|
||||
scraper_domain_delay_seconds: float = 2.0
|
||||
|
||||
# --- Cost controls ---
|
||||
max_companies_per_user: int = 25
|
||||
max_manual_runs_per_day: int = 10
|
||||
|
||||
# --- Retention / logging ---
|
||||
data_retention_days: int = 365
|
||||
log_level: str = "INFO"
|
||||
|
||||
@property
|
||||
def is_production(self) -> bool:
|
||||
return self.app_env == "production"
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _forbid_local_auth_in_production(self) -> Settings:
|
||||
if self.app_env == "production" and self.auth_mode == "local":
|
||||
raise ValueError(
|
||||
"AUTH_MODE=local is a development convenience and must not be used "
|
||||
"when APP_ENV=production. Set AUTH_MODE=jwt."
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_settings() -> Settings:
|
||||
return Settings()
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Symmetric encryption for secrets stored at rest - currently just
|
||||
per-user API keys (app/models/user_api_key.py). Not used for passwords
|
||||
(those are one-way hashed via app.core.security, never decrypted) - this
|
||||
is specifically for secrets the app must later read back out in plaintext
|
||||
to actually call a third-party API on the user's behalf.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from cryptography.fernet import Fernet, InvalidToken
|
||||
|
||||
from app.core.config import Settings
|
||||
|
||||
|
||||
def encrypt_secret(plaintext: str, settings: Settings) -> str:
|
||||
return Fernet(settings.api_key_encryption_secret).encrypt(plaintext.encode()).decode()
|
||||
|
||||
|
||||
def decrypt_secret(ciphertext: str, settings: Settings) -> str:
|
||||
try:
|
||||
return Fernet(settings.api_key_encryption_secret).decrypt(ciphertext.encode()).decode()
|
||||
except InvalidToken as exc:
|
||||
# Only real cause in practice: api_key_encryption_secret was
|
||||
# rotated after this value was encrypted under the old one.
|
||||
raise ValueError("Stored value cannot be decrypted with the current key") from exc
|
||||
@@ -0,0 +1,46 @@
|
||||
"""App-level exceptions, mapped to HTTP responses in one place (main.py).
|
||||
|
||||
Services raise these instead of `fastapi.HTTPException` so business logic
|
||||
stays importable/testable from Celery tasks, which don't have an HTTP
|
||||
response to raise into.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
class AppError(Exception):
|
||||
"""Base class for all app-level errors."""
|
||||
|
||||
|
||||
class NotFoundError(AppError):
|
||||
pass
|
||||
|
||||
|
||||
class ConflictError(AppError):
|
||||
pass
|
||||
|
||||
|
||||
class AuthenticationError(AppError):
|
||||
pass
|
||||
|
||||
|
||||
class ForbiddenError(AppError):
|
||||
pass
|
||||
|
||||
|
||||
class ValidationAppError(AppError):
|
||||
pass
|
||||
|
||||
|
||||
class RateLimitedError(AppError):
|
||||
pass
|
||||
|
||||
|
||||
class ThrottledError(RateLimitedError):
|
||||
"""Raised by the IP throttle/ban engine (app/services/ip_throttle_service.py)
|
||||
- carries a machine-readable retry_after_seconds so the frontend can
|
||||
drive a live countdown instead of just showing a generic message."""
|
||||
|
||||
def __init__(self, message: str, retry_after_seconds: int | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.retry_after_seconds = retry_after_seconds
|
||||
@@ -0,0 +1,176 @@
|
||||
"""SSRF-safe HTTP fetching. Every collector and the custom-URL feature must
|
||||
route network requests through `safe_fetch` / `fetch_with_retries` - see
|
||||
SECURITY.md for the full threat model this defends against.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import ipaddress
|
||||
import socket
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import httpx
|
||||
import tenacity
|
||||
|
||||
from app.core.config import Settings, get_settings
|
||||
from app.core.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
_ALLOWED_SCHEMES = {"http", "https"}
|
||||
_MAX_REDIRECTS = 5
|
||||
_METADATA_IPS = {"169.254.169.254", "fd00:ec2::254"}
|
||||
|
||||
# Per-hostname request spacing. In-process only - a multi-worker Celery
|
||||
# deployment would need a shared store (e.g. Redis) for this to be a true
|
||||
# global rate limit across workers; see KNOWN_LIMITATIONS.md.
|
||||
_last_request_at: dict[str, float] = {}
|
||||
_domain_locks: dict[str, asyncio.Lock] = {}
|
||||
|
||||
|
||||
class SsrfBlockedError(Exception):
|
||||
"""Raised when a URL resolves to, or points at, a disallowed network target."""
|
||||
|
||||
|
||||
class FetchError(Exception):
|
||||
"""Raised for network-level failures after retries are exhausted."""
|
||||
|
||||
|
||||
def _is_blocked_ip(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
|
||||
return (
|
||||
ip.is_private
|
||||
or ip.is_loopback
|
||||
or ip.is_link_local
|
||||
or ip.is_multicast
|
||||
or ip.is_reserved
|
||||
or ip.is_unspecified
|
||||
or str(ip) in _METADATA_IPS
|
||||
)
|
||||
|
||||
|
||||
def _resolve_and_validate(hostname: str) -> None:
|
||||
try:
|
||||
infos = socket.getaddrinfo(hostname, None)
|
||||
except socket.gaierror as exc:
|
||||
raise SsrfBlockedError(f"Could not resolve host: {hostname}") from exc
|
||||
|
||||
if not infos:
|
||||
raise SsrfBlockedError(f"Could not resolve host: {hostname}")
|
||||
|
||||
for info in infos:
|
||||
raw_ip = info[4][0]
|
||||
ip = ipaddress.ip_address(raw_ip.split("%")[0])
|
||||
if _is_blocked_ip(ip):
|
||||
raise SsrfBlockedError(f"Resolved address for {hostname} is not a public address: {ip}")
|
||||
|
||||
|
||||
def validate_url(url: str) -> str:
|
||||
"""Raises SsrfBlockedError if `url` is unsafe to fetch. Returns the
|
||||
hostname."""
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in _ALLOWED_SCHEMES:
|
||||
raise SsrfBlockedError(f"Unsupported URL scheme: {parsed.scheme!r}")
|
||||
if not parsed.hostname:
|
||||
raise SsrfBlockedError("URL has no hostname")
|
||||
_resolve_and_validate(parsed.hostname)
|
||||
return parsed.hostname
|
||||
|
||||
|
||||
async def _respect_domain_delay(hostname: str, delay_seconds: float) -> None:
|
||||
if delay_seconds <= 0:
|
||||
return
|
||||
lock = _domain_locks.setdefault(hostname, asyncio.Lock())
|
||||
async with lock:
|
||||
now = time.monotonic()
|
||||
last = _last_request_at.get(hostname)
|
||||
if last is not None:
|
||||
elapsed = now - last
|
||||
if elapsed < delay_seconds:
|
||||
await asyncio.sleep(delay_seconds - elapsed)
|
||||
_last_request_at[hostname] = time.monotonic()
|
||||
|
||||
|
||||
@dataclass
|
||||
class SafeFetchResult:
|
||||
status_code: int
|
||||
text: str
|
||||
content: bytes
|
||||
headers: dict[str, str]
|
||||
final_url: str
|
||||
|
||||
|
||||
async def safe_fetch(
|
||||
url: str,
|
||||
*,
|
||||
settings: Settings | None = None,
|
||||
method: str = "GET",
|
||||
max_redirects: int = _MAX_REDIRECTS,
|
||||
extra_headers: dict[str, str] | None = None,
|
||||
) -> SafeFetchResult:
|
||||
"""Fetch `url` with SSRF validation applied to the initial URL and every
|
||||
redirect hop. Never follows a redirect without re-validating it."""
|
||||
settings = settings or get_settings()
|
||||
current_url = url
|
||||
|
||||
for _ in range(max_redirects + 1):
|
||||
hostname = validate_url(current_url)
|
||||
await _respect_domain_delay(hostname, settings.scraper_domain_delay_seconds)
|
||||
|
||||
headers = {"User-Agent": settings.scraper_user_agent, **(extra_headers or {})}
|
||||
async with httpx.AsyncClient(
|
||||
follow_redirects=False,
|
||||
timeout=settings.scraper_request_timeout_seconds,
|
||||
headers=headers,
|
||||
) as client:
|
||||
response = await client.request(method, current_url)
|
||||
|
||||
if response.status_code in (301, 302, 303, 307, 308) and "location" in response.headers:
|
||||
current_url = urljoin(current_url, response.headers["location"])
|
||||
continue
|
||||
|
||||
return SafeFetchResult(
|
||||
status_code=response.status_code,
|
||||
text=response.text,
|
||||
content=response.content,
|
||||
headers=dict(response.headers),
|
||||
final_url=current_url,
|
||||
)
|
||||
|
||||
raise SsrfBlockedError(f"Too many redirects starting from {url}")
|
||||
|
||||
|
||||
def _is_retryable(exc: BaseException) -> bool:
|
||||
if isinstance(exc, SsrfBlockedError):
|
||||
return False
|
||||
if isinstance(exc, httpx.HTTPError):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
async def fetch_with_retries(
|
||||
url: str,
|
||||
*,
|
||||
settings: Settings | None = None,
|
||||
max_attempts: int = 3,
|
||||
**kwargs,
|
||||
) -> SafeFetchResult:
|
||||
"""`safe_fetch` wrapped with exponential-backoff retry for transient
|
||||
network errors only - SSRF blocks and 4xx responses are not retried."""
|
||||
settings = settings or get_settings()
|
||||
|
||||
async for attempt in tenacity.AsyncRetrying(
|
||||
stop=tenacity.stop_after_attempt(max_attempts),
|
||||
wait=tenacity.wait_exponential(multiplier=1, min=1, max=10),
|
||||
retry=tenacity.retry_if_exception(_is_retryable),
|
||||
reraise=True,
|
||||
):
|
||||
with attempt:
|
||||
result = await safe_fetch(url, settings=settings, **kwargs)
|
||||
if result.status_code >= 500:
|
||||
raise FetchError(f"Server error {result.status_code} fetching {url}")
|
||||
return result
|
||||
|
||||
raise FetchError(f"Exhausted retries fetching {url}") # pragma: no cover
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Structured logging setup with secret redaction, plus a capped live-log
|
||||
feed for the Settings UI.
|
||||
|
||||
Never log raw credentials. Any log event field whose key looks secret-shaped
|
||||
gets its value replaced before it leaves the process.
|
||||
|
||||
Logs also get pushed to a capped Redis list (`app:logs`) so the Settings
|
||||
page can show a live feed of what the application is doing - stdout/Docker
|
||||
logs alone aren't visible from the UI, and this app runs as several
|
||||
separate processes (API, Celery worker, beat), so Redis (already shared
|
||||
infra across all of them) is the simplest common sink. A sink failure here
|
||||
must never break the actual log call or crash the app - every Redis
|
||||
operation is wrapped and swallowed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from datetime import UTC, datetime
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import structlog
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.core.config import Settings
|
||||
|
||||
_SECRET_KEY_PATTERN = re.compile(
|
||||
r"(key|secret|token|password|authorization|credential)", re.IGNORECASE
|
||||
)
|
||||
_REDACTED = "***REDACTED***"
|
||||
|
||||
LOG_STREAM_KEY = "app:logs"
|
||||
LOG_STREAM_MAXLEN = 500
|
||||
|
||||
LOG_CATEGORY_LABELS = {
|
||||
"internal_error": "Internal error",
|
||||
"api_error": "API error",
|
||||
"important": "Important",
|
||||
"normal": "Normal",
|
||||
}
|
||||
|
||||
_NON_CONTEXT_KEYS = {"level", "timestamp", "logger", "event", "important"}
|
||||
|
||||
_sync_redis_client: Any = None
|
||||
|
||||
|
||||
def _redact_secrets(_logger: Any, _method_name: str, event_dict: dict) -> dict:
|
||||
for key in list(event_dict.keys()):
|
||||
if _SECRET_KEY_PATTERN.search(key):
|
||||
event_dict[key] = _REDACTED
|
||||
return event_dict
|
||||
|
||||
|
||||
def _category_for(level: str, event_dict: dict) -> str:
|
||||
if level in ("error", "critical"):
|
||||
return "internal_error"
|
||||
if level == "warning":
|
||||
return "api_error"
|
||||
if level == "info" and event_dict.get("important") is True:
|
||||
return "important"
|
||||
return "normal"
|
||||
|
||||
|
||||
def _get_sync_redis_client(redis_url: str):
|
||||
global _sync_redis_client
|
||||
if _sync_redis_client is None:
|
||||
import redis as redis_sync
|
||||
|
||||
_sync_redis_client = redis_sync.Redis.from_url(redis_url)
|
||||
return _sync_redis_client
|
||||
|
||||
|
||||
def _make_capture_processor(redis_url: str):
|
||||
def _capture_for_ui(_logger: Any, method_name: str, event_dict: dict) -> dict:
|
||||
level = event_dict.get("level", method_name)
|
||||
if level == "debug":
|
||||
return event_dict
|
||||
try:
|
||||
record = {
|
||||
"ts": event_dict.get("timestamp") or datetime.now(UTC).isoformat(),
|
||||
"level": level,
|
||||
"category": _category_for(level, event_dict),
|
||||
"logger": event_dict.get("logger", "app"),
|
||||
"event": str(event_dict.get("event", "")),
|
||||
"context": {k: v for k, v in event_dict.items() if k not in _NON_CONTEXT_KEYS},
|
||||
}
|
||||
client = _get_sync_redis_client(redis_url)
|
||||
pipe = client.pipeline()
|
||||
pipe.lpush(LOG_STREAM_KEY, json.dumps(record, default=str))
|
||||
pipe.ltrim(LOG_STREAM_KEY, 0, LOG_STREAM_MAXLEN - 1)
|
||||
pipe.execute()
|
||||
except Exception: # noqa: BLE001 - a broken log sink must never break the app
|
||||
pass
|
||||
return event_dict
|
||||
|
||||
return _capture_for_ui
|
||||
|
||||
|
||||
def configure_logging(settings: Settings) -> None:
|
||||
logging.basicConfig(level=settings.log_level, format="%(message)s")
|
||||
|
||||
structlog.configure(
|
||||
processors=[
|
||||
structlog.contextvars.merge_contextvars,
|
||||
structlog.stdlib.add_log_level,
|
||||
structlog.processors.TimeStamper(fmt="iso"),
|
||||
_redact_secrets,
|
||||
_make_capture_processor(settings.redis_url),
|
||||
structlog.processors.StackInfoRenderer(),
|
||||
structlog.processors.format_exc_info,
|
||||
structlog.processors.JSONRenderer(),
|
||||
],
|
||||
wrapper_class=structlog.make_filtering_bound_logger(
|
||||
logging.getLevelName(settings.log_level)
|
||||
),
|
||||
context_class=dict,
|
||||
logger_factory=structlog.PrintLoggerFactory(),
|
||||
cache_logger_on_first_use=True,
|
||||
)
|
||||
|
||||
|
||||
def get_logger(name: str) -> structlog.stdlib.BoundLogger:
|
||||
# `.bind(logger=name)` (rather than `structlog.stdlib.add_logger_name`,
|
||||
# which only works with `structlog.stdlib.LoggerFactory`) is what makes
|
||||
# the module name available to `_capture_for_ui` below - this app uses
|
||||
# `PrintLoggerFactory`, whose loggers have no `.name` attribute.
|
||||
return structlog.get_logger().bind(logger=name)
|
||||
|
||||
|
||||
async def get_recent_logs(settings: Settings, limit: int = 100) -> list[dict]:
|
||||
"""Most-recent-first log entries from the capped Redis feed, for the
|
||||
Settings page's Logging box."""
|
||||
import redis.asyncio as redis_asyncio
|
||||
|
||||
client = redis_asyncio.from_url(settings.redis_url)
|
||||
try:
|
||||
raw_entries = await client.lrange(LOG_STREAM_KEY, 0, limit - 1)
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
entries: list[dict] = []
|
||||
for raw in raw_entries:
|
||||
try:
|
||||
entries.append(json.loads(raw))
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
continue
|
||||
return entries
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Shared rate limiter (slowapi / limits, in-memory by default).
|
||||
|
||||
Applied per-route via `@limiter.limit(...)`. Auth endpoints get the
|
||||
tightest limits since they're the classic credential-stuffing target.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import Request
|
||||
from slowapi import Limiter
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.security import get_client_ip
|
||||
|
||||
|
||||
def _client_ip_key(request: Request) -> str:
|
||||
"""Same IP resolution as everything else in the app (is_localhost, the
|
||||
ban/throttle engine) - slowapi's own get_remote_address reads
|
||||
request.client.host directly, which would be Nginx/Cloudflare's own
|
||||
address for every visitor once deployed behind a reverse proxy,
|
||||
collapsing all rate limits into one shared bucket. See
|
||||
app.core.security.get_client_ip / Settings.trusted_proxy_ip_header."""
|
||||
return get_client_ip(request, get_settings())
|
||||
|
||||
|
||||
# Disabled under APP_ENV=test so the many auth calls a test suite makes don't
|
||||
# trip real limits (real limiter behavior is covered by its own test).
|
||||
limiter = Limiter(key_func=_client_ip_key, enabled=get_settings().app_env != "test")
|
||||
@@ -0,0 +1,171 @@
|
||||
"""Password hashing and JWT helpers.
|
||||
|
||||
Password hashing uses Argon2 (via `argon2-cffi`) directly - it's the
|
||||
currently recommended default and needs no extra abstraction layer.
|
||||
JWTs are signed with `JWT_SECRET` (HS256); access tokens are short-lived,
|
||||
refresh tokens are long-lived but stored server-side only as a hash so a
|
||||
leaked DB row can't be replayed as a valid token by itself.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from enum import StrEnum
|
||||
|
||||
import jwt
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerifyMismatchError
|
||||
from fastapi import Request
|
||||
|
||||
from app.core.config import Settings
|
||||
|
||||
_hasher = PasswordHasher()
|
||||
|
||||
|
||||
def get_client_ip(request: Request, settings: Settings) -> str:
|
||||
"""The single source of truth for "what IP is this request from" - used
|
||||
by `is_localhost`, the IP throttle/ban engine, and anywhere else that
|
||||
needs to identify a caller. Today there's no reverse proxy in front of
|
||||
uvicorn in this stack, so the direct TCP peer (`request.client.host`) is
|
||||
the real originating address.
|
||||
|
||||
Once deployed behind Cloudflare (or Nginx), the direct peer becomes the
|
||||
proxy itself, not the visitor - `settings.trusted_proxy_ip_header` (e.g.
|
||||
"CF-Connecting-IP") switches this to read the real address from that
|
||||
header instead. Only ever set this when it's actually known the proxy is
|
||||
in front and stripping/overwriting that header from untrusted clients -
|
||||
otherwise a client could simply forge it to spoof any IP. Left empty by
|
||||
default (trust the direct connection only) - see KNOWN_LIMITATIONS.md."""
|
||||
header_name = settings.trusted_proxy_ip_header
|
||||
if header_name:
|
||||
forwarded = request.headers.get(header_name)
|
||||
if forwarded:
|
||||
return forwarded.strip()
|
||||
return request.client.host if request.client is not None else "unknown"
|
||||
|
||||
|
||||
def is_localhost(request: Request, settings: Settings) -> bool:
|
||||
"""True when the request's resolved client IP (see `get_client_ip`) is
|
||||
this machine's loopback address - not merely "someone on the LAN" - or
|
||||
is explicitly listed in `settings.additional_trusted_local_ips` (empty
|
||||
by default; a narrow, opt-in escape hatch for Docker Desktop's bridge
|
||||
networking, where even host-originated traffic doesn't arrive as
|
||||
literal loopback - see KNOWN_LIMITATIONS.md)."""
|
||||
client_ip = get_client_ip(request, settings)
|
||||
if client_ip in ("127.0.0.1", "::1"):
|
||||
return True
|
||||
extra = {ip.strip() for ip in settings.additional_trusted_local_ips.split(",") if ip.strip()}
|
||||
return client_ip in extra
|
||||
|
||||
|
||||
def hash_password(raw_password: str) -> str:
|
||||
return _hasher.hash(raw_password)
|
||||
|
||||
|
||||
def verify_password(raw_password: str, password_hash: str) -> bool:
|
||||
try:
|
||||
return _hasher.verify(password_hash, raw_password)
|
||||
except VerifyMismatchError:
|
||||
return False
|
||||
|
||||
|
||||
class TokenType(StrEnum):
|
||||
ACCESS = "access"
|
||||
REFRESH = "refresh"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DecodedToken:
|
||||
user_id: uuid.UUID
|
||||
token_type: TokenType
|
||||
jti: str
|
||||
expires_at: datetime
|
||||
|
||||
|
||||
def create_access_token(user_id: uuid.UUID, settings: Settings) -> str:
|
||||
return _encode_token(
|
||||
user_id, TokenType.ACCESS, timedelta(minutes=settings.jwt_access_token_minutes), settings
|
||||
)
|
||||
|
||||
|
||||
def create_refresh_token(user_id: uuid.UUID, settings: Settings) -> tuple[str, str, datetime]:
|
||||
"""Returns (raw_jwt, jti, expires_at). Caller stores a hash of `jti`, not the JWT itself."""
|
||||
expires_at = datetime.now(UTC) + timedelta(days=settings.jwt_refresh_token_days)
|
||||
jti = secrets.token_urlsafe(32)
|
||||
token = _encode_token(
|
||||
user_id,
|
||||
TokenType.REFRESH,
|
||||
timedelta(days=settings.jwt_refresh_token_days),
|
||||
settings,
|
||||
jti=jti,
|
||||
)
|
||||
return token, jti, expires_at
|
||||
|
||||
|
||||
def _encode_token(
|
||||
user_id: uuid.UUID,
|
||||
token_type: TokenType,
|
||||
expires_in: timedelta,
|
||||
settings: Settings,
|
||||
jti: str | None = None,
|
||||
) -> str:
|
||||
now = datetime.now(UTC)
|
||||
payload = {
|
||||
"sub": str(user_id),
|
||||
"type": token_type.value,
|
||||
"iat": now,
|
||||
"exp": now + expires_in,
|
||||
"jti": jti or secrets.token_urlsafe(16),
|
||||
}
|
||||
return jwt.encode(payload, settings.jwt_secret, algorithm="HS256")
|
||||
|
||||
|
||||
class InvalidTokenError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def decode_token(token: str, settings: Settings, expected_type: TokenType) -> DecodedToken:
|
||||
try:
|
||||
payload = jwt.decode(token, settings.jwt_secret, algorithms=["HS256"])
|
||||
except jwt.PyJWTError as exc:
|
||||
raise InvalidTokenError(str(exc)) from exc
|
||||
|
||||
if payload.get("type") != expected_type.value:
|
||||
raise InvalidTokenError(f"Expected a {expected_type.value} token")
|
||||
|
||||
try:
|
||||
user_id = uuid.UUID(payload["sub"])
|
||||
except (KeyError, ValueError) as exc:
|
||||
raise InvalidTokenError("Malformed token subject") from exc
|
||||
|
||||
return DecodedToken(
|
||||
user_id=user_id,
|
||||
token_type=TokenType(payload["type"]),
|
||||
jti=payload["jti"],
|
||||
expires_at=datetime.fromtimestamp(payload["exp"], tz=UTC),
|
||||
)
|
||||
|
||||
|
||||
def hash_token_identifier(jti: str) -> str:
|
||||
"""One-way hash of a refresh token's `jti` for storage/comparison (not the JWT itself)."""
|
||||
import hashlib
|
||||
|
||||
return hashlib.sha256(jti.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def generate_email_code() -> str:
|
||||
"""A 6-digit numeric code for email verification / password reset -
|
||||
short-lived and IP-throttled (app/services/ip_throttle_service.py), so
|
||||
it doesn't need Argon2's cost the way a password does."""
|
||||
return f"{secrets.randbelow(1_000_000):06d}"
|
||||
|
||||
|
||||
def hash_email_code(code: str) -> str:
|
||||
"""One-way hash of an email code for storage/comparison (never the raw
|
||||
code) - same precedent as hash_token_identifier."""
|
||||
import hashlib
|
||||
|
||||
return hashlib.sha256(code.encode("utf-8")).hexdigest()
|
||||
@@ -0,0 +1,13 @@
|
||||
"""Small text utilities with no natural home elsewhere."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
_SLUG_STRIP_RE = re.compile(r"[^a-z0-9]+")
|
||||
|
||||
|
||||
def slugify(value: str) -> str:
|
||||
value = value.strip().lower()
|
||||
value = _SLUG_STRIP_RE.sub("-", value).strip("-")
|
||||
return value or "company"
|
||||
Reference in New Issue
Block a user