from __future__ import annotations from app.collectors.extraction import ( canonicalize_url, compute_content_hash, extract_readable_text, extract_title, normalize_whitespace, ) def test_normalize_whitespace_collapses_blank_lines_and_trims(): raw = " Title \n\n\n\nBody line one \n \nBody line two " normalized = normalize_whitespace(raw) assert normalized == "Title\n\nBody line one\n\nBody line two" def test_compute_content_hash_is_stable_and_sensitive_to_change(): a = compute_content_hash("hello world") b = compute_content_hash("hello world") c = compute_content_hash("hello world!") assert a == b assert a != c def test_canonicalize_url_strips_tracking_params_and_trailing_slash(): url = "HTTPS://Example.com/About/?utm_source=x&ref=y" assert canonicalize_url(url) == "https://example.com/About?ref=y" def test_canonicalize_url_normalizes_root_path(): assert canonicalize_url("https://example.com") == "https://example.com/" def test_extract_readable_text_prefers_trafilatura_for_article_html(): html = """

Acme launches new product

Acme Corp announced a new electric vehicle platform today, expanding its lineup.

The company said manufacturing will begin next quarter at its main facility.

""" text, method = extract_readable_text(html, "https://example.com/news/1") assert "Acme launches new product" in text assert "Copyright 2026" not in text assert method in ("trafilatura", "beautifulsoup_fallback") def test_extract_readable_text_falls_back_when_trafilatura_finds_nothing(): html = "" text, method = extract_readable_text(html, "https://example.com/thin") assert method == "beautifulsoup_fallback" assert text == "" def test_extract_title_prefers_title_tag(): html = "Acme — About

About

" assert extract_title(html) == "Acme — About" def test_extract_title_falls_back_to_h1(): html = "

Fallback Heading

" assert extract_title(html) == "Fallback Heading"