saksham
1a4c80958f
FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification).
77 lines
2.3 KiB
Python
77 lines
2.3 KiB
Python
"""Data-retention purge task: only SourceDocument rows older than
|
|
DATA_RETENTION_DAYS are deleted, everything newer is untouched."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import uuid
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
import pytest
|
|
from sqlalchemy import select
|
|
|
|
from app.models.company import Company
|
|
from app.models.enums import SourceType
|
|
from app.models.source import Source
|
|
from app.models.source_document import SourceDocument
|
|
from app.repositories.source_repository import SourceDocumentRepository
|
|
from app.tasks.maintenance import _purge_expired_data_async
|
|
|
|
|
|
async def _make_document(
|
|
db_session, company, source, *, retrieved_date: datetime
|
|
) -> SourceDocument:
|
|
return await SourceDocumentRepository(db_session).create(
|
|
source_id=source.id,
|
|
company_id=company.id,
|
|
url=f"https://acme.example/{uuid.uuid4().hex[:8]}",
|
|
canonical_url=f"https://acme.example/{uuid.uuid4().hex[:8]}",
|
|
title="Doc",
|
|
author=None,
|
|
publication_date=None,
|
|
retrieved_date=retrieved_date,
|
|
content_text="Some content",
|
|
content_hash=uuid.uuid4().hex,
|
|
metadata_json={},
|
|
extraction_method="test",
|
|
trust_score=0.7,
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_purge_deletes_only_documents_older_than_retention_window(db_session, settings):
|
|
company = Company(
|
|
id=uuid.uuid4(),
|
|
user_id=uuid.uuid4(),
|
|
name="Retention Co",
|
|
slug=f"retention-co-{uuid.uuid4().hex[:6]}",
|
|
)
|
|
db_session.add(company)
|
|
await db_session.flush()
|
|
|
|
source = Source(
|
|
company_id=company.id,
|
|
source_type=SourceType.WEBSITE,
|
|
name="Site",
|
|
base_url="https://acme.example",
|
|
)
|
|
db_session.add(source)
|
|
await db_session.flush()
|
|
|
|
now = datetime.now(UTC)
|
|
old_doc = await _make_document(
|
|
db_session,
|
|
company,
|
|
source,
|
|
retrieved_date=now - timedelta(days=settings.data_retention_days + 30),
|
|
)
|
|
recent_doc = await _make_document(
|
|
db_session, company, source, retrieved_date=now - timedelta(days=1)
|
|
)
|
|
await db_session.commit()
|
|
|
|
await _purge_expired_data_async()
|
|
|
|
remaining_ids = set((await db_session.execute(select(SourceDocument.id))).scalars().all())
|
|
assert old_doc.id not in remaining_ids
|
|
assert recent_doc.id in remaining_ids
|