Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

13.0 KB
Raw Blame
"""Sync distinguishes usable content from empty text and empty PDF storage."""

import hashlib

import pytest
from sqlalchemy import event, inspect

from local_deep_research.constants import FILE_PATH_TEXT_ONLY
from local_deep_research.database.models.library import (
    Document,
    DocumentBlob,
    DocumentChunk,
    DocumentCollection,
    DocumentStatus,
)
from local_deep_research.database.models.research import ResearchResource
from local_deep_research.research_library.services.pdf_storage_manager import (
    PDFStorageManager,
)
from local_deep_research.research_library.utils import (
    document_awaits_pdf,
    find_tracker_by_url,
)
from tests.research_library.test_library_sync_contracts import (
    PDF_BYTES,
    REGENERATED_PDF_BYTES,
    SAVED_TEXT,
    StubDownloader,
    _add_chunk,
    _cite_in_another_research,
    _add_database_row_without_pdf,
    db as db,
    filesystem_mode as filesystem_mode,
    make_library_service,
    make_service,
    one_download_session as one_download_session,
    store as store,
    world as world,
)


def add_document(db, world, *, text=None, content=PDF_BYTES, url=None):
    url = url or "https://example.com/empty-content.pdf"
    resource = world.add_resource(url, downloaded=True)
    document = _add_database_row_without_pdf(
        db, world, resource, world.research_id, url, content
    )
    row = db.session.get(Document, document)
    row.text_content = text
    db.session.get(ResearchResource, resource).document_id = document
    tracker = find_tracker_by_url(db.session, url)
    tracker.file_path = "database"
    tracker.file_hash = hashlib.sha256(content).hexdigest()
    db.session.commit()
    return document, resource


@pytest.mark.parametrize(
    "text",
    [
        "\t\n\r ",
        "\u00a0\u2003\u202f",
        "\v\f\x1c\x1d\x1e\x1f\x85\u1680\u2000\u2001\u2002\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u205f\u3000",
    ],
)
def test_whitespace_only_is_removed(db, store, world, monkeypatch, text):
    document, _ = add_document(db, world, text=text)
    stats = make_library_service(
        db, store, monkeypatch
    ).sync_library_with_filesystem()
    with db.fresh() as session:
        assert session.get(Document, document) is None, (
            "whitespace is not usable extracted text"
        )
    assert stats["files_missing"] == 1


@pytest.mark.parametrize("storage", ["filesystem", "database"])
@pytest.mark.parametrize("tracked", [True, False])
def test_empty_pdf_is_refilled_in_same_document(
    db,
    store,
    world,
    one_download_session,
    filesystem_mode,
    monkeypatch,
    storage,
    tracked,
):
    document, resource = add_document(db, world, text=SAVED_TEXT)
    chunk = _add_chunk(db, document)
    row = db.session.get(Document, document)
    row.storage_mode = storage
    if storage == "filesystem":
        row.file_path = "pdfs/empty.pdf"
        (store.root / row.file_path).write_bytes(b"")
    else:
        db.session.add(DocumentBlob(document_id=document, pdf_binary=b""))
    tracker = find_tracker_by_url(db.session, row.original_url)
    tracker.file_path = (row.file_path or "database") if tracked else None
    db.session.commit()
    library = make_library_service(db, store, monkeypatch)
    stats = library.sync_library_with_filesystem()
    assert stats["files_missing"] == 1, (
        "a confirmed empty PDF must be eligible for refill"
    )
    with db.fresh() as session:
        kept = session.get(Document, document)
        assert kept.text_content == SAVED_TEXT
        assert kept.file_path == FILE_PATH_TEXT_ONLY
        assert document_awaits_pdf(kept)
        assert not PDFStorageManager.pdf_exists(store.root, kept, session)
        if storage == "database":
            assert session.get(DocumentBlob, document) is None
        assert session.get(DocumentChunk, chunk) is not None
        assert (
            session.query(DocumentCollection)
            .filter_by(document_id=document)
            .count()
            == 1
        )
        assert session.get(ResearchResource, resource).document_id == document
    downloader = StubDownloader(REGENERATED_PDF_BYTES)
    service = make_service(store, downloader, storage_mode=storage)
    assert service.download_resource(resource) == (True, None)
    with db.fresh() as session:
        kept = session.get(Document, document)
        assert session.query(Document).count() == 1
        assert (
            PDFStorageManager(store.root, storage).load_pdf(kept, session)
            == REGENERATED_PDF_BYTES
        )
        assert session.get(ResearchResource, resource).document_id == document
    assert (
        service.queue_research_downloads(
            world.research_id, "library-sync-contracts"
        )
        == 0
    )
    assert downloader.urls == ["https://example.com/empty-content.pdf"]
    assert library.sync_library_with_filesystem()["files_missing"] == 0


def test_clear_missing_pdf_keeps_a_concurrently_refilled_blob(
    db, store, world, monkeypatch
):
    """A blob that gained real bytes before the delete must survive.

    ``_clear_missing_pdf`` removes the zero-length blob row, but only while
    it is still empty. If a concurrent download writes real bytes into the
    row between the earlier size read and this delete, those bytes must be
    kept. The delete is constrained to empty blobs, so the refilled row is
    preserved.
    """
    document, _ = add_document(db, world, text=SAVED_TEXT)
    row = db.session.get(Document, document)
    row.storage_mode = "database"
    db.session.add(DocumentBlob(document_id=document, pdf_binary=PDF_BYTES))
    tracker = find_tracker_by_url(db.session, row.original_url)
    db.session.commit()
    service = make_library_service(db, store, monkeypatch)
    service._clear_missing_pdf(row, tracker, {"trackers_updated": 0})
    db.session.commit()
    with db.fresh() as session:
        blob = session.get(DocumentBlob, document)
        assert blob is not None, "a refilled (nonempty) blob must be kept"
        assert blob.pdf_binary == PDF_BYTES


@pytest.mark.parametrize(
    "status",
    [DocumentStatus.FAILED, DocumentStatus.PENDING, DocumentStatus.PROCESSING],
)
@pytest.mark.parametrize("holder_first", [True, False])
@pytest.mark.parametrize(
    "blob", [PDF_BYTES, b"", None], ids=["positive", "zero", "absent"]
)
def test_database_tracker_census_includes_noncompleted_holders(
    db, store, world, monkeypatch, status, holder_first, blob
):
    url = "https://example.com/empty-content.pdf"
    empty_resource = world.add_resource(url, downloaded=True)
    holder_research, holder_resource = _cite_in_another_research(db, url)
    tracker = find_tracker_by_url(db.session, url)
    tracker.file_path = "database"
    db.session.commit()

    def holder():
        document = _add_database_row_without_pdf(
            db,
            world,
            holder_resource,
            holder_research,
            url,
            REGENERATED_PDF_BYTES,
        )
        row = db.session.get(Document, document)
        row.status = status
        row.text_content = "retained holder text " * 100_000
        db.session.get(ResearchResource, holder_resource).document_id = document
        if blob is not None:
            db.session.add(
                DocumentBlob(
                    document_id=document,
                    pdf_binary=REGENERATED_PDF_BYTES if blob else b"",
                )
            )
        db.session.commit()
        return document

    if holder_first:
        holder_id = holder()
    empty = _add_database_row_without_pdf(
        db, world, empty_resource, world.research_id, url, PDF_BYTES
    )
    if not holder_first:
        holder_id = holder()
    library = make_library_service(db, store, monkeypatch)
    stats = sync_without_loading_bodies(db, library)
    with db.fresh() as session:
        assert session.get(Document, empty) is None
        assert session.get(Document, holder_id).status == status
        tracker = find_tracker_by_url(
            session, "https://example.com/empty-content.pdf"
        )
        assert (tracker.file_path, tracker.is_downloaded) == (
            ("database", True) if blob else (None, False)
        ), (
            "a noncompleted holder's positive PDF must protect the shared tracker"
        )
    assert stats["trackers_updated"] == (0 if blob else 1)


def sync_without_loading_bodies(db, library):
    statements = []

    def capture(
        connection, cursor, statement, parameters, context, executemany
    ):
        statements.append(statement)

    def check_deferred(session, instance):
        if isinstance(instance, Document):
            assert "text_content" in inspect(instance).unloaded

    event.listen(db.engine, "before_cursor_execute", capture)
    event.listen(type(db.session), "loaded_as_persistent", check_deferred)
    try:
        stats = library.sync_library_with_filesystem()
    finally:
        event.remove(db.engine, "before_cursor_execute", capture)
        event.remove(type(db.session), "loaded_as_persistent", check_deferred)
    # SQL may inspect text or blob length, but must not project either body.
    for statement in statements:
        projection = statement.lower().split("from", 1)[0]
        assert "documents.text_content" not in projection, statement
        assert "document_blobs.pdf_binary" not in projection.replace(
            "length(document_blobs.pdf_binary)", ""
        ), statement
    return stats


@pytest.mark.parametrize(
    "text",
    ["普通の本文", "\t\u00a0 useful text \u2003\n", "x" * 2_000_000, "\u200b"],
    ids=["unicode", "surrounded", "large", "non-whitespace-separator"],
)
def test_text_stays_deferred_and_nonblank_is_retained(
    db, store, world, monkeypatch, text
):
    document, _ = add_document(db, world, text=text)
    stats = sync_without_loading_bodies(
        db, make_library_service(db, store, monkeypatch)
    )
    assert stats["files_missing"] == 0
    with db.fresh() as session:
        assert session.get(Document, document).text_content == text


@pytest.mark.parametrize("storage", ["filesystem", "database"])
@pytest.mark.parametrize("tracked", [True, False])
def test_empty_pdf_without_text_is_deleted(
    db, store, world, monkeypatch, storage, tracked
):
    document, _ = add_document(db, world)
    row = db.session.get(Document, document)
    row.storage_mode = storage
    if storage == "filesystem":
        row.file_path = "pdfs/zero.pdf"
        (store.root / row.file_path).write_bytes(b"")
    else:
        db.session.add(DocumentBlob(document_id=document, pdf_binary=b""))
    tracker = find_tracker_by_url(db.session, row.original_url)
    tracker.file_path = (row.file_path or "database") if tracked else None
    db.session.commit()
    stats = make_library_service(
        db, store, monkeypatch
    ).sync_library_with_filesystem()
    with db.fresh() as session:
        assert session.get(Document, document) is None, (
            "zero bytes cannot preserve an empty document"
        )
    assert stats["files_missing"] == 1


@pytest.mark.parametrize(
    "identity", ["hash", "link", "conflicting-link", "conflicting-hash"]
)
def test_legacy_tracker_identity_is_not_overridden_by_raw_url(
    db, store, world, monkeypatch, identity
):
    from local_deep_research.database.models.download_tracker import (
        DownloadTracker,
    )
    from local_deep_research.research_library.utils import get_legacy_url_hashes

    url = "https://example.com/identity.pdf?utm_source=legacy"
    document, _ = add_document(db, world, url=url)
    canonical = find_tracker_by_url(db.session, url)
    canonical.file_hash = hashlib.sha256(b"newer version").hexdigest()
    canonical_path = canonical.file_path
    legacy = DownloadTracker(
        url=url,
        url_hash=get_legacy_url_hashes(url)[0],
        first_resource_id=canonical.first_resource_id,
        is_downloaded=True,
        file_path="database",
        file_hash=hashlib.sha256(PDF_BYTES).hexdigest()
        if identity != "link"
        else None,
        library_document_id=document
        if identity in ("link", "conflicting-hash")
        else None,
    )
    if identity == "conflicting-link":
        # Point the explicit link at a real, unrelated document.
        other, _ = add_document(
            db,
            world,
            url="https://example.com/unrelated.pdf",
            content=REGENERATED_PDF_BYTES,
        )
        legacy.library_document_id = other
    if identity == "conflicting-hash":
        legacy.file_hash = canonical.file_hash
    db.session.add(legacy)
    db.session.commit()
    canonical_id, legacy_id = canonical.id, legacy.id
    make_library_service(db, store, monkeypatch).sync_library_with_filesystem()
    with db.fresh() as session:
        assert session.get(Document, document) is None
        current = session.get(DownloadTracker, canonical_id)
        assert (current.file_path, current.is_downloaded) == (
            canonical_path,
            True,
        )
        old = session.get(DownloadTracker, legacy_id)
        identified = identity in ("hash", "link")
        assert (old.file_path, old.is_downloaded) == (
            (None, False) if identified else ("database", True)
        )