saksham
1a4c80958f
FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification).
131 lines
4.4 KiB
Python
131 lines
4.4 KiB
Python
"""Tests for the documented significance/confidence/severity formula in
|
|
app/change_detection/scoring.py (see ARCHITECTURE.md)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from app.change_detection.scoring import classify_severity, compute_confidence, compute_significance
|
|
from app.models.enums import ChangeType, SeverityLevel
|
|
|
|
|
|
def test_significance_scales_with_source_trust():
|
|
high_trust = compute_significance(
|
|
change_type=ChangeType.NEW_DOCUMENT, source_trust_score=1.0, independent_source_count=2
|
|
)
|
|
low_trust = compute_significance(
|
|
change_type=ChangeType.NEW_DOCUMENT, source_trust_score=0.3, independent_source_count=2
|
|
)
|
|
assert high_trust > low_trust
|
|
|
|
|
|
def test_significance_uncorroborated_signal_is_halved():
|
|
corroborated = compute_significance(
|
|
change_type=ChangeType.NEW_DOCUMENT, source_trust_score=1.0, independent_source_count=2
|
|
)
|
|
single_source = compute_significance(
|
|
change_type=ChangeType.NEW_DOCUMENT, source_trust_score=1.0, independent_source_count=1
|
|
)
|
|
assert single_source == round(corroborated / 2, 4)
|
|
|
|
|
|
def test_significance_focus_match_boosts_score():
|
|
matched = compute_significance(
|
|
change_type=ChangeType.NEW_DOCUMENT,
|
|
source_trust_score=1.0,
|
|
independent_source_count=2,
|
|
focus_match=True,
|
|
)
|
|
unmatched = compute_significance(
|
|
change_type=ChangeType.NEW_DOCUMENT,
|
|
source_trust_score=1.0,
|
|
independent_source_count=2,
|
|
focus_match=False,
|
|
)
|
|
assert matched > unmatched
|
|
|
|
|
|
def test_significance_repeat_change_is_dampened():
|
|
fresh = compute_significance(
|
|
change_type=ChangeType.NEW_DOCUMENT, source_trust_score=1.0, is_repeat=False
|
|
)
|
|
repeat = compute_significance(
|
|
change_type=ChangeType.NEW_DOCUMENT, source_trust_score=1.0, is_repeat=True
|
|
)
|
|
assert repeat == round(fresh / 2, 4)
|
|
|
|
|
|
def test_significance_content_modified_scales_with_diff_ratio():
|
|
small_edit = compute_significance(
|
|
change_type=ChangeType.CONTENT_MODIFIED,
|
|
source_trust_score=1.0,
|
|
independent_source_count=2,
|
|
diff_ratio=0.05,
|
|
)
|
|
big_rewrite = compute_significance(
|
|
change_type=ChangeType.CONTENT_MODIFIED,
|
|
source_trust_score=1.0,
|
|
independent_source_count=2,
|
|
diff_ratio=0.9,
|
|
)
|
|
assert big_rewrite > small_edit
|
|
|
|
|
|
def test_significance_never_exceeds_one():
|
|
value = compute_significance(
|
|
change_type=ChangeType.LEADERSHIP_CHANGE,
|
|
source_trust_score=1.0,
|
|
independent_source_count=10,
|
|
focus_match=True,
|
|
)
|
|
assert value <= 1.0
|
|
|
|
|
|
def test_confidence_increases_with_corroboration():
|
|
single = compute_confidence(
|
|
extraction_confidence=0.8, source_trust_score=0.8, independent_source_count=1
|
|
)
|
|
corroborated = compute_confidence(
|
|
extraction_confidence=0.8, source_trust_score=0.8, independent_source_count=2
|
|
)
|
|
assert corroborated > single
|
|
|
|
|
|
def test_confidence_bounded_between_zero_and_one():
|
|
assert (
|
|
0.0
|
|
<= compute_confidence(
|
|
extraction_confidence=0.0, source_trust_score=0.0, independent_source_count=1
|
|
)
|
|
<= 1.0
|
|
)
|
|
assert (
|
|
0.0
|
|
<= compute_confidence(
|
|
extraction_confidence=1.0, source_trust_score=1.0, independent_source_count=5
|
|
)
|
|
<= 1.0
|
|
)
|
|
|
|
|
|
def test_classify_severity_buckets_by_score():
|
|
assert classify_severity(significance=0.9, confidence=0.9) == SeverityLevel.CRITICAL
|
|
assert classify_severity(significance=0.6, confidence=0.9) == SeverityLevel.HIGH
|
|
assert classify_severity(significance=0.3, confidence=0.9) == SeverityLevel.MEDIUM
|
|
assert classify_severity(significance=0.1, confidence=0.9) == SeverityLevel.LOW
|
|
|
|
|
|
def test_classify_severity_critical_requires_high_confidence():
|
|
"""A score that would otherwise land in the Critical bucket (>= 0.6)
|
|
must be downgraded to High when confidence is below the floor - an
|
|
uncorroborated single-source signal can't carry the Critical label."""
|
|
score = 1.0 * 0.65
|
|
assert score >= 0.6 # would be CRITICAL by score alone
|
|
severity = classify_severity(significance=1.0, confidence=0.65)
|
|
assert severity == SeverityLevel.HIGH
|
|
|
|
|
|
def test_classify_severity_high_confidence_allows_critical():
|
|
score = 1.0 * 0.8
|
|
assert score >= 0.6
|
|
severity = classify_severity(significance=1.0, confidence=0.8)
|
|
assert severity == SeverityLevel.CRITICAL
|