Initial commit: CI Agent competitive-intelligence monitoring app

FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring,
source collection, LLM-based change analysis, enrichment, and account
security (Turnstile, escalating lockout, email verification).

apps/api/app/change_detection/extractors.py

@@ -0,0 +1,25 @@
+"""Lightweight, best-effort regex extractors for specific signal types the
+severity model treats specially (pricing, leadership). These are heuristics,
+not NLP - they exist to catch the common "$X/month" and "named a new CEO"
+phrasings, not to parse arbitrary text reliably. Phase 7's LLM extraction
+task is the higher-fidelity version of this; these run cheaply and
+deterministically as part of scoring, without a model call.
+"""
+
+from __future__ import annotations
+
+import re
+
+_PRICE_RE = re.compile(r"\$\s?\d[\d,]*(?:\.\d{2})?\s*(?:/\s*(?:month|mo|year|yr))?")
+_LEADERSHIP_TITLE_RE = re.compile(
+    r"(?i)\b(Chief Executive Officer|CEO|Chief Financial Officer|CFO|Chief Technology Officer|"
+    r"CTO|President|Chairman|Chairwoman|Chairperson)\b"
+)
+
+
+def extract_prices(text: str) -> set[str]:
+    return set(_PRICE_RE.findall(text or ""))
+
+
+def mentions_leadership_title(text: str) -> bool:
+    return bool(_LEADERSHIP_TITLE_RE.search(text or ""))

apps/api/app/change_detection/noise_filters.py

@@ -0,0 +1,32 @@
+"""Strips content that changes on every fetch but carries no meaning, so it
+never counts toward a text diff. Applied before Layer 3 (text diff) - see
+ARCHITECTURE.md and spec section 18.
+"""
+
+from __future__ import annotations
+
+import re
+
+_NOISE_PATTERNS = [
+    # Dynamic timestamps: "Updated: 2026-01-01", "Last modified 01/02/2026 14:30"
+    re.compile(r"(?i)\b(updated|last modified|generated|retrieved)\s*:?\s*[\d/:\-\sTZ]+"),
+    # Session/CSRF-style tokens embedded in visible text (rare, but happens on thin pages)
+    re.compile(r"\b[A-Za-z0-9_-]{24,}\b"),
+    # Cookie/consent banner boilerplate
+    re.compile(r"(?i)we use cookies[^.]*\.?"),
+    re.compile(r"(?i)by (continuing|using this site)[^.]*\.?"),
+    # Copyright year lines, which change every January with no real signal
+    re.compile(r"(?i)copyright\s*(?:©|\(c\))?\s*\d{4}[–\-]?\d{0,4}"),
+    # View/like/share counters
+    re.compile(r"(?i)\b\d[\d,]*\s*(views|likes|shares)\b"),
+]
+
+_WHITESPACE_RUN = re.compile(r"[ \t]{2,}")
+_BLANK_LINES = re.compile(r"\n{3,}")
+
+
+def strip_noise(text: str) -> str:
+    for pattern in _NOISE_PATTERNS:
+        text = pattern.sub(" ", text)
+    text = _WHITESPACE_RUN.sub(" ", text)
+    return _BLANK_LINES.sub("\n\n", text).strip()

apps/api/app/change_detection/scoring.py

@@ -0,0 +1,103 @@
+"""Layer 5: significance scoring + severity classification.
+
+This is the documented, unit-tested formula referenced in ARCHITECTURE.md.
+Deterministic on purpose - severity must be explainable and reproducible
+without a model call. Phase 7's LLM analysis narrates *why* a change
+matters; it does not decide *how much* it matters.
+
+    significance = base_weight
+                 * source_trust_score                    (0.3 - 1.0)
+                 * min(1.0, independent_sources / 2)      # corroboration, caps at 2 sources
+                 * focus_match_multiplier                 (1.3 if it matches the user's stated focus, else 1.0)
+                 * recency_multiplier                     (1.0 if new, 0.5 if a repeat of a recent change)
+
+    confidence = clamp(
+        0.5 * extraction_confidence + 0.4 * source_trust_score + 0.1
+        + (0.15 if independent_sources >= 2 else 0.0),
+        0.0, 1.0
+    )
+
+    severity = bucket(significance * confidence), with a hard floor:
+        CRITICAL requires confidence >= CRITICAL_MIN_CONFIDENCE regardless of score -
+        an uncorroborated single-source signal can never be labeled Critical.
+"""
+
+from __future__ import annotations
+
+from app.models.enums import ChangeType, SeverityLevel
+
+BASE_WEIGHTS: dict[ChangeType, float] = {
+    ChangeType.LEADERSHIP_CHANGE: 0.9,
+    ChangeType.FILING_NEW: 0.85,
+    ChangeType.PRICE_CHANGE: 0.6,
+    ChangeType.NEW_DOCUMENT: 0.5,
+    ChangeType.CONTENT_MODIFIED: 0.3,  # further scaled by diff_ratio - see compute_significance
+    ChangeType.REMOVED_DOCUMENT: 0.3,
+}
+
+# (score_threshold, severity) - first match wins, checked highest first.
+# Calibrated against compute_significance/compute_confidence's actual output
+# range (a single-source signal is already discounted ~2x by the
+# corroboration multiplier) so CRITICAL_MIN_CONFIDENCE below is reachable:
+# with significance capped at 1.0, a score of 0.6 needs confidence >= 0.6,
+# which leaves room for the 0.7 confidence floor to actually bite and
+# downgrade a subset of would-be-CRITICAL cases to HIGH.
+SEVERITY_THRESHOLDS: list[tuple[float, SeverityLevel]] = [
+    (0.6, SeverityLevel.CRITICAL),
+    (0.4, SeverityLevel.HIGH),
+    (0.2, SeverityLevel.MEDIUM),
+    (0.0, SeverityLevel.LOW),
+]
+
+CRITICAL_MIN_CONFIDENCE = 0.7
+
+
+def compute_significance(
+    *,
+    change_type: ChangeType,
+    source_trust_score: float,
+    independent_source_count: int = 1,
+    focus_match: bool = False,
+    is_repeat: bool = False,
+    diff_ratio: float | None = None,
+) -> float:
+    base = BASE_WEIGHTS[change_type]
+    if change_type is ChangeType.CONTENT_MODIFIED and diff_ratio is not None:
+        # A one-line wording tweak and a full page rewrite are both
+        # "content_modified" but shouldn't score the same.
+        base = base + diff_ratio * 0.5
+
+    trust_multiplier = _clamp(source_trust_score, 0.3, 1.0)
+    corroboration_multiplier = min(1.0, independent_source_count / 2)
+    focus_multiplier = 1.3 if focus_match else 1.0
+    recency_multiplier = 0.5 if is_repeat else 1.0
+
+    significance = (
+        base * trust_multiplier * corroboration_multiplier * focus_multiplier * recency_multiplier
+    )
+    return round(_clamp(significance, 0.0, 1.0), 4)
+
+
+def compute_confidence(
+    *,
+    extraction_confidence: float,
+    source_trust_score: float,
+    independent_source_count: int = 1,
+) -> float:
+    corroboration_bonus = 0.15 if independent_source_count >= 2 else 0.0
+    confidence = 0.5 * extraction_confidence + 0.4 * source_trust_score + 0.1 + corroboration_bonus
+    return round(_clamp(confidence, 0.0, 1.0), 4)
+
+
+def classify_severity(significance: float, confidence: float) -> SeverityLevel:
+    score = significance * confidence
+    for threshold, severity in SEVERITY_THRESHOLDS:
+        if score >= threshold:
+            if severity is SeverityLevel.CRITICAL and confidence < CRITICAL_MIN_CONFIDENCE:
+                return SeverityLevel.HIGH
+            return severity
+    return SeverityLevel.LOW  # pragma: no cover - thresholds bottom out at 0.0
+
+
+def _clamp(value: float, lo: float, hi: float) -> float:
+    return max(lo, min(hi, value))

apps/api/app/change_detection/structured_diff.py

@@ -0,0 +1,33 @@
+"""Layer 2: structured field comparison.
+
+Compares the *set* of items (job postings, press releases, filings,
+products - whatever the source's documents represent) between two
+snapshots' `structured_summary["urls"]`/`["titles"]`. This is what catches
+"a new job posting appeared" or "a press release was removed" without
+needing a bespoke parser per source type - collection_service already
+records the full current item set on every run, so this is a plain set
+diff between two runs.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True)
+class StructuredDiff:
+    added: list[str] = field(default_factory=list)
+    removed: list[str] = field(default_factory=list)
+
+    @property
+    def has_changes(self) -> bool:
+        return bool(self.added or self.removed)
+
+
+def diff_item_sets(previous_urls: list[str], current_urls: list[str]) -> StructuredDiff:
+    previous_set = set(previous_urls)
+    current_set = set(current_urls)
+    return StructuredDiff(
+        added=sorted(current_set - previous_set),
+        removed=sorted(previous_set - current_set),
+    )

apps/api/app/change_detection/text_diff.py

@@ -0,0 +1,56 @@
+"""Layer 3: bounded text diff.
+
+Runs on noise-stripped text (see noise_filters.py) so navigation/cookie/
+timestamp churn doesn't register as a change. Bounded: only a capped number
+of added/removed lines are kept, so a full page rewrite doesn't produce an
+unbounded diff blob for storage or LLM consumption later.
+"""
+
+from __future__ import annotations
+
+import difflib
+from dataclasses import dataclass, field
+
+from app.change_detection.noise_filters import strip_noise
+
+_MAX_DIFF_LINES = 40
+
+
+@dataclass(frozen=True)
+class TextDiffResult:
+    diff_ratio: float  # 0.0 = identical, 1.0 = completely different
+    added_lines: list[str] = field(default_factory=list)
+    removed_lines: list[str] = field(default_factory=list)
+
+    @property
+    def is_identical(self) -> bool:
+        return self.diff_ratio == 0.0
+
+
+def bounded_text_diff(previous_text: str, current_text: str) -> TextDiffResult:
+    previous_clean = strip_noise(previous_text or "")
+    current_clean = strip_noise(current_text or "")
+
+    if previous_clean == current_clean:
+        return TextDiffResult(diff_ratio=0.0)
+
+    previous_lines = [line for line in previous_clean.splitlines() if line.strip()]
+    current_lines = [line for line in current_clean.splitlines() if line.strip()]
+
+    matcher = difflib.SequenceMatcher(a=previous_lines, b=current_lines, autojunk=False)
+    similarity = matcher.ratio()
+    diff_ratio = round(1.0 - similarity, 4)
+
+    added: list[str] = []
+    removed: list[str] = []
+    for tag, i1, i2, j1, j2 in matcher.get_opcodes():
+        if tag in ("replace", "delete"):
+            removed.extend(previous_lines[i1:i2])
+        if tag in ("replace", "insert"):
+            added.extend(current_lines[j1:j2])
+
+    return TextDiffResult(
+        diff_ratio=diff_ratio,
+        added_lines=added[:_MAX_DIFF_LINES],
+        removed_lines=removed[:_MAX_DIFF_LINES],
+    )