Initial commit: CI Agent competitive-intelligence monitoring app

FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification).
2026-08-05 10:48:20 -04:00
commit 1a4c80958f
365 changed files with 43541 additions and 0 deletions
@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+from app.collectors.extraction import (
+    canonicalize_url,
+    compute_content_hash,
+    extract_readable_text,
+    extract_title,
+    normalize_whitespace,
+)
+
+
+def test_normalize_whitespace_collapses_blank_lines_and_trims():
+    raw = "  Title  \n\n\n\nBody line one   \n   \nBody line two  "
+    normalized = normalize_whitespace(raw)
+    assert normalized == "Title\n\nBody line one\n\nBody line two"
+
+
+def test_compute_content_hash_is_stable_and_sensitive_to_change():
+    a = compute_content_hash("hello world")
+    b = compute_content_hash("hello world")
+    c = compute_content_hash("hello world!")
+    assert a == b
+    assert a != c
+
+
+def test_canonicalize_url_strips_tracking_params_and_trailing_slash():
+    url = "HTTPS://Example.com/About/?utm_source=x&ref=y"
+    assert canonicalize_url(url) == "https://example.com/About?ref=y"
+
+
+def test_canonicalize_url_normalizes_root_path():
+    assert canonicalize_url("https://example.com") == "https://example.com/"
+
+
+def test_extract_readable_text_prefers_trafilatura_for_article_html():
+    html = """
+    <html><body>
+    <nav>Home | About | Contact</nav>
+    <article>
+      <h1>Acme launches new product</h1>
+      <p>Acme Corp announced a new electric vehicle platform today, expanding its lineup.</p>
+      <p>The company said manufacturing will begin next quarter at its main facility.</p>
+    </article>
+    <footer>Copyright 2026</footer>
+    </body></html>
+    """
+    text, method = extract_readable_text(html, "https://example.com/news/1")
+    assert "Acme launches new product" in text
+    assert "Copyright 2026" not in text
+    assert method in ("trafilatura", "beautifulsoup_fallback")
+
+
+def test_extract_readable_text_falls_back_when_trafilatura_finds_nothing():
+    html = "<html><body></body></html>"
+    text, method = extract_readable_text(html, "https://example.com/thin")
+    assert method == "beautifulsoup_fallback"
+    assert text == ""
+
+
+def test_extract_title_prefers_title_tag():
+    html = "<html><head><title>Acme — About</title></head><body><h1>About</h1></body></html>"
+    assert extract_title(html) == "Acme — About"
+
+
+def test_extract_title_falls_back_to_h1():
+    html = "<html><head></head><body><h1>Fallback Heading</h1></body></html>"
+    assert extract_title(html) == "Fallback Heading"