Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

23.5 KB
Raw Blame
"""Shared library content must outlive any one referencing research."""

from contextlib import contextmanager
from datetime import UTC, datetime
from hashlib import sha256
import sqlite3

import pytest
from sqlalchemy import MetaData, create_engine, event
from sqlalchemy.exc import OperationalError
from sqlalchemy.orm import Session

from local_deep_research.database import research_cascade
from local_deep_research.database.models import (
    Base,
    QueuedResearch,
    ResearchHistory,
    ResearchResource,
)
from local_deep_research.database.models.library import (
    Collection,
    Document,
    DocumentBlob,
    DocumentChunk,
    DocumentCollection,
    EmbeddingProvider,
    RAGIndex,
    RagDocumentStatus,
    SourceType,
)
from local_deep_research.database.research_cascade import (
    SharedDownloadIndex,
    delete_research_with_dependents,
)
from local_deep_research.research_library.deletion.utils.cascade_helper import (
    CascadeHelper,
)


NOW = datetime(2026, 10, 1, tzinfo=UTC)
PDF = b"%PDF-1.4 stored canonical paper"
BODY = "Complete canonical paper text."


@pytest.fixture(
    params=["SET NULL", "CASCADE"], ids=["current", "legacy-resource-fk"]
)
def database(tmp_path, request):
    engine = create_engine(
        f"sqlite:///{tmp_path / 'shared-documents.sqlite'}",
        connect_args={"isolation_level": "", "timeout": 1},
    )

    @event.listens_for(engine, "connect")
    def enable_foreign_keys(connection, _record):
        connection.execute("PRAGMA foreign_keys=ON")

    # Clone metadata rather than mutating the production models or another
    # test's cached cascade plan. Only the historical resource FK differs.
    schema = MetaData()
    for table in Base.metadata.tables.values():
        table.to_metadata(schema)
    for constraint in schema.tables["documents"].foreign_key_constraints:
        if tuple(constraint.column_keys) == ("resource_id",):
            constraint.ondelete = request.param
    with engine.connect() as connection:
        assert (
            connection.exec_driver_sql("PRAGMA journal_mode=WAL").scalar()
            == "wal"
        )
        # Group schema setup; legacy SQLite otherwise commits each DDL alone.
        # The sessions under test still start with their normal driver behavior.
        connection.exec_driver_sql("BEGIN")
        schema.create_all(connection)
        connection.commit()
        resource_fk = next(
            row
            for row in connection.exec_driver_sql(
                "PRAGMA foreign_key_list(documents)"
            )
            if row[3] == "resource_id"
        )
        assert resource_fk[6] == request.param
    try:
        yield engine
    finally:
        engine.dispose()


def seed_family(engine, *, references=True):
    with Session(engine) as session:
        session.add(
            SourceType(
                id="source", name="research_download", display_name="Research"
            )
        )
        for research_id in ("owner", "sibling", "last"):
            session.add(
                ResearchHistory(
                    id=research_id,
                    query="q",
                    mode="quick",
                    status="completed",
                    created_at=NOW.isoformat(),
                )
            )
        session.add_all(
            [
                Collection(id="library", name="Library"),
                Collection(id="selected", name="Selected"),
            ]
        )
        session.flush()
        resources = {}
        for research_id in ("owner", "sibling", "last"):
            resource = ResearchResource(
                research_id=research_id,
                title="Shared paper",
                url=f"https://example.org/{research_id}.pdf",
                source_type="academic",
                created_at=NOW.isoformat(),
            )
            session.add(resource)
            session.flush()
            resources[research_id] = resource.id
        session.add(
            Document(
                id="canonical",
                source_type_id="source",
                resource_id=resources["owner"],
                research_id="owner",
                document_hash=sha256(PDF).hexdigest(),
                title="Shared paper",
                original_url="https://example.org/owner.pdf",
                file_type="pdf",
                file_path="database",
                storage_mode="database",
                file_size=len(PDF),
                text_content=BODY,
                word_count=len(BODY.split()),
                character_count=len(BODY),
                extraction_method="html_extraction",
                extraction_source="arxiv_html",
                extraction_quality="high",
                processed_at=NOW,
            )
        )
        # A document with no surviving reference must retain ordinary cascade
        # behavior; the shared-document exception must not preserve everything.
        session.add(
            Document(
                id="unshared",
                source_type_id="source",
                research_id="owner",
                document_hash="unshared",
                file_size=16,
                file_type="text",
                text_content="Private to owner",
            )
        )
        session.flush()
        session.add(
            DocumentBlob(
                document_id="canonical",
                pdf_binary=PDF,
                blob_hash=sha256(PDF).hexdigest(),
            )
        )
        if references:
            for research_id in ("sibling", "last"):
                session.get(
                    ResearchResource, resources[research_id]
                ).document_id = "canonical"
        for collection_id in ("library", "selected"):
            index = RAGIndex(
                collection_name=f"collection_{collection_id}",
                embedding_model="test",
                embedding_model_type=EmbeddingProvider.SENTENCE_TRANSFORMERS,
                embedding_dimension=2,
                index_path=f"/{collection_id}.faiss",
                index_hash=collection_id,
                chunk_size=100,
                chunk_overlap=0,
            )
            session.add(index)
            session.flush()
            session.add_all(
                [
                    DocumentCollection(
                        document_id="canonical",
                        collection_id=collection_id,
                        indexed=True,
                        chunk_count=1,
                        last_indexed_at=NOW,
                    ),
                    RagDocumentStatus(
                        document_id="canonical",
                        collection_id=collection_id,
                        rag_index_id=index.id,
                        chunk_count=1,
                        indexed_at=NOW,
                    ),
                    DocumentChunk(
                        chunk_hash=collection_id,
                        source_type="document",
                        source_id="canonical",
                        collection_name=f"collection_{collection_id}",
                        chunk_text=BODY,
                        chunk_index=0,
                        start_char=0,
                        end_char=len(BODY),
                        word_count=len(BODY.split()),
                        embedding_id=collection_id,
                        embedding_model="test",
                        embedding_model_type=EmbeddingProvider.SENTENCE_TRANSFORMERS,
                    ),
                ]
            )
        session.commit()
    return resources


def assert_preserved(session, research_id, resource_id):
    session.expire_all()
    doc = session.get(Document, "canonical")
    assert doc is not None
    assert (doc.research_id, doc.resource_id) == (research_id, resource_id)
    assert (
        session.get(ResearchResource, resource_id).research_id
        == doc.research_id
    )
    assert doc.text_content == BODY
    assert doc.document_hash == sha256(PDF).hexdigest()
    assert doc.extraction_source == "arxiv_html"
    assert doc.extraction_quality == "high"
    assert doc.original_url == "https://example.org/owner.pdf"
    assert (doc.file_path, doc.storage_mode, doc.file_size) == (
        "database",
        "database",
        len(PDF),
    )
    assert session.get(DocumentBlob, doc.id).pdf_binary == PDF
    links = (
        session.query(DocumentCollection).filter_by(document_id=doc.id).all()
    )
    assert {
        (
            link.collection_id,
            link.indexed,
            link.chunk_count,
            link.last_indexed_at,
        )
        for link in links
    } == {(collection, True, 1, NOW) for collection in ("library", "selected")}
    assert (
        session.query(RagDocumentStatus).filter_by(document_id=doc.id).count()
        == 2
    )
    assert {
        (row.embedding_id, row.chunk_text)
        for row in session.query(DocumentChunk).filter_by(source_id=doc.id)
    } == {("library", BODY), ("selected", BODY)}
    assert (
        session.connection().exec_driver_sql("PRAGMA foreign_key_check").all()
        == []
    )


@pytest.mark.parametrize("deleted", ["owner", "sibling"])
def test_other_research_keeps_document_and_all_stored_content(
    database, deleted
):
    resources = seed_family(database)
    with Session(database) as session:
        assert delete_research_with_dependents(session, deleted)
        session.commit()
        remaining_owner = "sibling" if deleted == "owner" else "owner"
        assert_preserved(session, remaining_owner, resources[remaining_owner])
        assert session.get(ResearchHistory, deleted) is None
        assert (
            session.get(ResearchResource, resources["last"]).document_id
            == "canonical"
        )
        if deleted == "owner":
            assert session.get(Document, "unshared") is None


@pytest.mark.parametrize("order", [("owner", "sibling"), ("sibling", "owner")])
def test_clear_history_transfers_to_surviving_research(database, order):
    resources = seed_family(database)
    with Session(database) as session:
        index = SharedDownloadIndex.build(session, order)
        for research_id in order:
            assert delete_research_with_dependents(
                session, research_id, shared_downloads=index
            )
        session.commit()
        assert_preserved(session, "last", resources["last"])
        assert (
            session.get(ResearchResource, resources["last"]).document_id
            == "canonical"
        )


def test_last_reference_keeps_existing_document_cascade(database):
    seed_family(database)
    with Session(database) as session:
        index = SharedDownloadIndex.build(session, ["owner", "sibling", "last"])
        for research_id in ("owner", "sibling", "last"):
            assert delete_research_with_dependents(
                session, research_id, shared_downloads=index
            )
        session.commit()
        for model in (
            ResearchHistory,
            ResearchResource,
            Document,
            DocumentBlob,
            DocumentCollection,
            RagDocumentStatus,
        ):
            assert session.query(model).count() == 0


@pytest.mark.parametrize("guard", ["status", "queue"])
def test_guard_refusal_restores_ownership_and_content(database, guard):
    resources = seed_family(database)
    with Session(database) as session:
        if guard == "status":
            session.get(ResearchHistory, "owner").status = "in_progress"
        else:
            session.add(
                QueuedResearch(
                    username="test",
                    research_id="owner",
                    query="q",
                    mode="quick",
                    position=1,
                    is_processing=True,
                    created_at=NOW,
                )
            )
        session.commit()
        assert not delete_research_with_dependents(session, "owner")
        session.commit()
        assert_preserved(session, "owner", resources["owner"])
        assert session.get(Document, "unshared") is not None
        assert session.query(ResearchHistory).count() == 3


def test_caller_rollback_restores_transferred_owner(database):
    resources = seed_family(database)
    with Session(database) as session:
        index = SharedDownloadIndex.build(session, ["owner", "sibling"])
        for research_id in ("owner", "sibling"):
            assert delete_research_with_dependents(
                session, research_id, shared_downloads=index
            )
        session.rollback()
    with Session(database) as verification:
        assert_preserved(verification, "owner", resources["owner"])
        assert verification.query(ResearchHistory).count() == 3
        assert (
            verification.get(ResearchResource, resources["sibling"]).document_id
            == "canonical"
        )


@pytest.mark.parametrize("failure_stage", ["transfer", "parent-delete"])
def test_failed_transfer_rolls_back_deletion(database, failure_stage):
    resources = seed_family(database)

    def fail_owner_change(_conn, _cursor, statement, *_args):
        if (
            failure_stage == "transfer"
            and statement.startswith("UPDATE documents SET")
            and "research_id=" in statement
        ) or (
            failure_stage == "parent-delete"
            and statement.startswith("DELETE FROM research_history")
        ):
            raise RuntimeError("controlled ownership write failure")

    event.listen(database, "before_cursor_execute", fail_owner_change)
    try:
        with Session(database) as session:
            with pytest.raises(
                RuntimeError, match="controlled ownership write failure"
            ):
                delete_research_with_dependents(session, "owner")
            session.rollback()
    finally:
        event.remove(database, "before_cursor_execute", fail_owner_change)
    with Session(database) as verification:
        assert_preserved(verification, "owner", resources["owner"])
        assert verification.query(ResearchHistory).count() == 3


def test_explicit_document_database_delete_remains_global(database):
    resources = seed_family(database)
    with Session(database) as session:
        assert CascadeHelper.delete_document_completely(session, "canonical")
        session.commit()
        assert session.get(Document, "canonical") is None
        assert session.get(DocumentBlob, "canonical") is None
        assert session.query(ResearchHistory).count() == 3
        for research_id in ("sibling", "last"):
            assert (
                session.get(
                    ResearchResource, resources[research_id]
                ).document_id
                is None
            )


def test_batch_retains_document_with_protected_successor(database):
    resources = seed_family(database)
    with Session(database) as session:
        session.get(ResearchHistory, "sibling").status = "in_progress"
        session.commit()
        index = SharedDownloadIndex.build(session, ["owner", "sibling", "last"])
        assert delete_research_with_dependents(
            session, "owner", shared_downloads=index
        )
        assert not delete_research_with_dependents(
            session, "sibling", shared_downloads=index
        )
        assert delete_research_with_dependents(
            session, "last", shared_downloads=index
        )
        session.commit()
        assert_preserved(session, "sibling", resources["sibling"])
        assert session.query(ResearchHistory).count() == 1


@pytest.mark.parametrize("batch", [False, True], ids=["single", "batch"])
@pytest.mark.parametrize("change", ["add", "remove"])
def test_concurrent_references_require_fresh_deletion_snapshot(
    database, monkeypatch, batch, change
):
    resources = seed_family(database, references=change == "remove")
    lookup = research_cascade._shared_document_replacements
    changed = False

    def change_after_lookup(*args):
        nonlocal changed
        result = lookup(*args)
        if not changed:
            changed = True
            with Session(database) as writer:
                for research_id in ("sibling", "last"):
                    writer.get(
                        ResearchResource, resources[research_id]
                    ).document_id = "canonical" if change == "add" else None
                writer.commit()
        return result

    monkeypatch.setattr(
        research_cascade, "_shared_document_replacements", change_after_lookup
    )
    with Session(database) as deletion:
        index = (
            SharedDownloadIndex.build(deletion, ["owner"]) if batch else None
        )
        with pytest.raises(OperationalError, match="database is locked"):
            delete_research_with_dependents(
                deletion, "owner", shared_downloads=index
            )
        deletion.rollback()
        assert changed
        with Session(database) as verification:
            assert_preserved(verification, "owner", resources["owner"])
            assert verification.get(ResearchHistory, "owner") is not None
            expected = "canonical" if change == "add" else None
            assert (
                verification.get(
                    ResearchResource, resources["sibling"]
                ).document_id
                == expected
            )

        # The retry must follow the committed references rather than its old
        # retention decision, in either direction and with a fresh batch index.
        index = (
            SharedDownloadIndex.build(deletion, ["owner"]) if batch else None
        )
        assert delete_research_with_dependents(
            deletion, "owner", shared_downloads=index
        )
        deletion.commit()
        if change == "add":
            assert_preserved(deletion, "sibling", resources["sibling"])
        else:
            assert deletion.get(Document, "canonical") is None
            assert deletion.get(DocumentBlob, "canonical") is None
        assert deletion.get(ResearchHistory, "owner") is None
        assert deletion.get(ResearchHistory, "sibling") is not None


@pytest.fixture
def route_session(database, monkeypatch):
    from local_deep_research.web.routers import research

    # Match the production context: a thread reuses its Session, but a failed
    # outer scope rolls back before another operation may use it.
    with Session(database) as session:

        @contextmanager
        def context(_username):
            try:
                yield session
            finally:
                if session.in_transaction():
                    session.rollback()

        monkeypatch.setattr(research, "get_user_db_session", context)
        monkeypatch.setattr(
            research, "get_active_research_ids", lambda: {"sibling", "last"}
        )
        yield session


def call_delete_route(batch):
    from local_deep_research.web.routers import research

    if batch:
        return research.clear_history(request=None, username="test-user")
    return research.delete_research(
        request=None, research_id="owner", username="test-user"
    )


@pytest.mark.parametrize("batch", [False, True], ids=["single", "batch"])
@pytest.mark.parametrize("change", ["add", "remove"])
def test_routes_retry_changed_document_references(
    database, route_session, monkeypatch, batch, change
):
    resources = seed_family(database, references=change == "remove")
    lookup = research_cascade._shared_document_replacements
    lookups = 0

    def change_after_lookup(*args):
        nonlocal lookups
        result = lookup(*args)
        lookups += 1
        if lookups == 1:
            with Session(database) as writer:
                for research_id in ("sibling", "last"):
                    writer.get(
                        ResearchResource, resources[research_id]
                    ).document_id = "canonical" if change == "add" else None
                writer.commit()
        return result

    monkeypatch.setattr(
        research_cascade, "_shared_document_replacements", change_after_lookup
    )
    assert call_delete_route(batch) == {"status": "success"}
    assert lookups == 2
    assert not route_session.in_transaction()
    with Session(database) as verification:
        assert verification.get(ResearchHistory, "owner") is None
        assert verification.get(ResearchHistory, "sibling") is not None
        if change == "add":
            assert_preserved(verification, "sibling", resources["sibling"])
        else:
            assert verification.get(Document, "canonical") is None
            assert verification.get(DocumentBlob, "canonical") is None


@pytest.mark.parametrize("batch", [False, True], ids=["single", "batch"])
def test_retry_rechecks_a_research_claimed_by_another_writer(
    database, route_session, monkeypatch, batch
):
    resources = seed_family(database)
    lookup = research_cascade._shared_document_replacements
    changed = False

    def claim_after_lookup(*args):
        nonlocal changed
        result = lookup(*args)
        if not changed:
            changed = True
            with Session(database) as writer:
                writer.get(ResearchHistory, "owner").status = "in_progress"
                writer.commit()
        return result

    monkeypatch.setattr(
        research_cascade, "_shared_document_replacements", claim_after_lookup
    )
    response = call_delete_route(batch)
    if batch:
        assert response == {"status": "success"}
    else:
        assert response.status_code == 400
    assert changed
    with Session(database) as verification:
        assert (
            verification.get(ResearchHistory, "owner").status == "in_progress"
        )
        assert_preserved(verification, "owner", resources["owner"])


@pytest.mark.parametrize("batch", [False, True], ids=["single", "batch"])
@pytest.mark.parametrize("failure", ["transient-lock", "persistent-lock", "io"])
def test_report_file_survives_until_deletion_commits(
    database, route_session, monkeypatch, tmp_path, batch, failure
):
    from local_deep_research.config import paths

    resources = seed_family(database)
    report = tmp_path / "research-report.md"
    report.write_text("Retain the report when its database deletion fails.")
    monkeypatch.setattr(
        paths, "get_research_outputs_directory", lambda: tmp_path
    )
    with Session(database) as writer:
        writer.get(ResearchHistory, "owner").report_path = str(report)
        writer.commit()

    real_commit = route_session.commit
    attempts = 0

    def commit():
        nonlocal attempts
        attempts += 1
        # This is a real file: even an ultimately successful retry must not
        # remove it before the successful database commit.
        assert report.is_file()
        if failure == "persistent-lock" or attempts == 1:
            message = (
                "disk I/O error" if failure == "io" else "database is locked"
            )
            raise OperationalError(
                "COMMIT", None, sqlite3.OperationalError(message)
            )
        real_commit()

    monkeypatch.setattr(route_session, "commit", commit)
    response = call_delete_route(batch)
    if failure == "transient-lock":
        assert response == {"status": "success"}
        assert attempts == 2
        assert not report.exists()
        with Session(database) as verification:
            assert verification.get(ResearchHistory, "owner") is None
    else:
        assert response.status_code == 500
        assert attempts == (3 if failure == "persistent-lock" else 1)
        assert (
            report.read_text()
            == "Retain the report when its database deletion fails."
        )
        with Session(database) as verification:
            assert verification.get(ResearchHistory, "owner") is not None
            assert_preserved(verification, "owner", resources["owner"])