saksham
1a4c80958f
FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification).
68 lines
2.3 KiB
Python
68 lines
2.3 KiB
Python
from __future__ import annotations
|
|
|
|
from app.collectors.extraction import (
|
|
canonicalize_url,
|
|
compute_content_hash,
|
|
extract_readable_text,
|
|
extract_title,
|
|
normalize_whitespace,
|
|
)
|
|
|
|
|
|
def test_normalize_whitespace_collapses_blank_lines_and_trims():
|
|
raw = " Title \n\n\n\nBody line one \n \nBody line two "
|
|
normalized = normalize_whitespace(raw)
|
|
assert normalized == "Title\n\nBody line one\n\nBody line two"
|
|
|
|
|
|
def test_compute_content_hash_is_stable_and_sensitive_to_change():
|
|
a = compute_content_hash("hello world")
|
|
b = compute_content_hash("hello world")
|
|
c = compute_content_hash("hello world!")
|
|
assert a == b
|
|
assert a != c
|
|
|
|
|
|
def test_canonicalize_url_strips_tracking_params_and_trailing_slash():
|
|
url = "HTTPS://Example.com/About/?utm_source=x&ref=y"
|
|
assert canonicalize_url(url) == "https://example.com/About?ref=y"
|
|
|
|
|
|
def test_canonicalize_url_normalizes_root_path():
|
|
assert canonicalize_url("https://example.com") == "https://example.com/"
|
|
|
|
|
|
def test_extract_readable_text_prefers_trafilatura_for_article_html():
|
|
html = """
|
|
<html><body>
|
|
<nav>Home | About | Contact</nav>
|
|
<article>
|
|
<h1>Acme launches new product</h1>
|
|
<p>Acme Corp announced a new electric vehicle platform today, expanding its lineup.</p>
|
|
<p>The company said manufacturing will begin next quarter at its main facility.</p>
|
|
</article>
|
|
<footer>Copyright 2026</footer>
|
|
</body></html>
|
|
"""
|
|
text, method = extract_readable_text(html, "https://example.com/news/1")
|
|
assert "Acme launches new product" in text
|
|
assert "Copyright 2026" not in text
|
|
assert method in ("trafilatura", "beautifulsoup_fallback")
|
|
|
|
|
|
def test_extract_readable_text_falls_back_when_trafilatura_finds_nothing():
|
|
html = "<html><body></body></html>"
|
|
text, method = extract_readable_text(html, "https://example.com/thin")
|
|
assert method == "beautifulsoup_fallback"
|
|
assert text == ""
|
|
|
|
|
|
def test_extract_title_prefers_title_tag():
|
|
html = "<html><head><title>Acme — About</title></head><body><h1>About</h1></body></html>"
|
|
assert extract_title(html) == "Acme — About"
|
|
|
|
|
|
def test_extract_title_falls_back_to_h1():
|
|
html = "<html><head></head><body><h1>Fallback Heading</h1></body></html>"
|
|
assert extract_title(html) == "Fallback Heading"
|