Skip to content

A community-supported supercharged document management system: scan, index and archive all your documents

42.3 KB
Raw Blame
import inspect
import sqlite3
from collections.abc import Callable
from collections.abc import Generator
from pathlib import Path

import pytest
import sqlite_vec
from llama_index.core.schema import TextNode
from llama_index.core.vector_stores.types import FilterOperator
from llama_index.core.vector_stores.types import MetadataFilter
from llama_index.core.vector_stores.types import MetadataFilters
from llama_index.core.vector_stores.types import VectorStoreQuery
from pytest_mock import MockerFixture

from paperless_ai.migrations import MIGRATIONS
from paperless_ai.migrations import Migration
from paperless_ai.migrations import m0001_v1_to_v2
from paperless_ai.tables import DocumentChunksTable
from paperless_ai.tables import DocumentMetaTable
from paperless_ai.vector_store import _MAX_IN_VALUES
from paperless_ai.vector_store import DB_FILENAME
from paperless_ai.vector_store import DEFAULT_TABLE_NAME
from paperless_ai.vector_store import SCHEMA_VERSION
from paperless_ai.vector_store import PaperlessSqliteVecVectorStore
from paperless_ai.vector_store import _build_where
from paperless_ai.vector_store import _pack

DIM = 16


def make_node(
    node_id: str,
    document_id: int,
    *,
    modified: str = "2026-06-10T00:00:00",
    seed: float = 0.0,
    text: str = "some text",
) -> TextNode:
    node = TextNode(
        id_=node_id,
        text=text,
        metadata={"document_id": document_id, "modified": modified},
    )
    node.relationships = {}
    node.embedding = [seed + i / 100 for i in range(DIM)]
    return node


@pytest.fixture
def store(tmp_path: Path) -> Generator[PaperlessSqliteVecVectorStore, None, None]:
    with PaperlessSqliteVecVectorStore(uri=str(tmp_path)) as store:
        yield store


def _query(
    store: PaperlessSqliteVecVectorStore,
    embedding: list[float],
    top_k: int = 5,
    filters=None,
):
    return store.query(
        VectorStoreQuery(
            query_embedding=embedding,
            similarity_top_k=top_k,
            filters=filters,
        ),
    )


def _eq_filter(key: str, value: int):
    return MetadataFilters(
        filters=[MetadataFilter(key=key, operator=FilterOperator.EQ, value=value)],
    )


def _in_filter(document_ids: list[int]):
    return MetadataFilters(
        filters=[
            MetadataFilter(
                key="document_id",
                operator=FilterOperator.IN,
                value=document_ids,
            ),
        ],
    )


def _ne_filter(document_id: int):
    return MetadataFilters(
        filters=[
            MetadataFilter(
                key="document_id",
                operator=FilterOperator.NE,
                value=document_id,
            ),
        ],
    )


def _nin_filter(document_ids: list[int]):
    return MetadataFilters(
        filters=[
            MetadataFilter(
                key="document_id",
                operator=FilterOperator.NIN,
                value=document_ids,
            ),
        ],
    )


class TestCrud:
    def test_add_then_query_returns_node(self, store) -> None:
        node = make_node("n1", 1)
        assert store.add([node]) == ["n1"]
        result = _query(store, node.embedding, top_k=1)
        assert result.ids == ["n1"]
        assert result.nodes[0].metadata["document_id"] == 1
        # cosine distance of the identical vector is 0 -> similarity 1
        assert result.similarities[0] == pytest.approx(1.0)

    def test_query_empty_store_returns_empty_no_raise(self, store) -> None:
        result = _query(store, [0.0] * DIM)
        assert result.ids == [] and result.nodes == [] and result.similarities == []

    def test_add_empty_list_is_noop(self, store) -> None:
        assert store.add([]) == []
        assert not store.table_exists()

    def test_delete_removes_all_chunks_of_document(self, store) -> None:
        store.add([make_node("a1", 1), make_node("a2", 1), make_node("b1", 2)])
        store.delete(1)
        result = _query(store, [0.0] * DIM, top_k=10)
        assert result.ids == ["b1"]

    def test_query_with_in_filter_scopes_results(self, store) -> None:
        store.add(
            [
                make_node("a1", 1, seed=0.0),
                make_node("b1", 2, seed=1.0),
                make_node("c1", 3, seed=2.0),
            ],
        )
        result = _query(store, [0.0] * DIM, top_k=10, filters=_in_filter([2, 3]))
        assert sorted(result.ids) == ["b1", "c1"]

    def test_query_respects_top_k_with_filter(self, store) -> None:
        # k semantics: global top-k even with IN filters (document_id is a
        # metadata column, not a partition key -- see design doc).
        store.add(
            [make_node(f"n{i}", i % 4, seed=float(i)) for i in range(12)],
        )
        result = _query(
            store,
            [0.0] * DIM,
            top_k=3,
            filters=_in_filter([0, 1, 2, 3]),
        )
        assert len(result.ids) == 3
        assert result.similarities == sorted(result.similarities, reverse=True)

    def test_get_nodes_filter_and_empty_paths(self, store) -> None:
        assert store.get_nodes(filters=_in_filter([1])) == []  # no table yet
        store.add([make_node("a1", 1), make_node("b1", 2)])
        nodes = store.get_nodes(filters=_in_filter([1]))
        assert [n.node_id for n in nodes] == ["a1"]
        assert nodes[0].embedding is not None
        assert store.get_nodes(filters=_in_filter([999])) == []

    def test_query_with_eq_filter_scopes_results(self, store) -> None:
        store.add(
            [
                make_node("a1", 1, seed=0.0),
                make_node("b1", 2, seed=1.0),
                make_node("c1", 3, seed=2.0),
            ],
        )
        result = _query(
            store,
            [0.0] * DIM,
            top_k=10,
            filters=_eq_filter("document_id", 2),
        )
        assert result.ids == ["b1"]

    def test_get_nodes_node_ids_not_implemented(self, store) -> None:
        with pytest.raises(NotImplementedError):
            store.get_nodes(node_ids=["x"])

    def test_fresh_instance_sees_existing_table(self, store, tmp_path: Path) -> None:
        store.add([make_node("a1", 1)])
        with PaperlessSqliteVecVectorStore(uri=str(tmp_path)) as reopened:
            assert reopened.table_exists()
            assert reopened.vector_dim() == DIM
            assert _query(reopened, [0.0] * DIM, top_k=1).ids == ["a1"]

    def test_table_exists_and_drop(self, store) -> None:
        assert not store.table_exists()
        store.add([make_node("a1", 1)])
        assert store.table_exists()
        store.drop_table()
        assert not store.table_exists()
        assert store.vector_dim() is None

    def test_document_id_stored_as_integer_in_vec0(
        self,
        store: PaperlessSqliteVecVectorStore,
    ) -> None:
        """
        GIVEN:
            - An empty vector store
        WHEN:
            - A node is added with an int document_id
        THEN:
            - vec0's own document_id column holds an INTEGER, not TEXT
        """
        store.add([make_node("a1", 1)])
        row = store.client.execute(
            "SELECT document_id FROM documents WHERE id = 'a1'",
        ).fetchone()
        assert isinstance(row["document_id"], int)

    def test_drop_table_clears_modified_times(
        self,
        store: PaperlessSqliteVecVectorStore,
    ) -> None:
        """
        GIVEN:
            - A store with a tracked document's modified time
        WHEN:
            - drop_table() is called
        THEN:
            - document_meta and document_chunks are both cleared directly
              (asserted against the tables themselves, not via
              get_modified_times()/table_exists() -- those short-circuit on
              the vec0 table being gone, which drop_table() does first, so
              they would pass even if DocumentMetaTable.delete_all()/
              DocumentChunksTable.delete_all() were never called)
        """
        store.add([make_node("a1", 1)])
        store.drop_table()
        assert (
            store.client.execute(
                "SELECT count(*) FROM document_meta",
            ).fetchone()[0]
            == 0
        )
        assert (
            store.client.execute(
                "SELECT count(*) FROM document_chunks",
            ).fetchone()[0]
            == 0
        )

    def test_upsert_document_checks_table_exists_once(
        self,
        store: PaperlessSqliteVecVectorStore,
        mocker: MockerFixture,
    ) -> None:
        """
        GIVEN:
            - An existing store with one document already indexed
        WHEN:
            - upsert_document() replaces that document's chunks
        THEN:
            - table_exists() is queried at most once per call, not twice
              (previously: once via _ensure_table(), once via the separate
              `if self.table_exists():` delete-chunks guard)
        """
        store.add([make_node("a1", 1)])
        # store is a pydantic model, whose __setattr__/__delattr__ reject
        # arbitrary instance attributes ("object has no attribute
        # 'table_exists'"), so mocker.spy(store, "table_exists") can't
        # shadow the method on the instance. Spying on the class works
        # (bound method lookup on the instance still resolves through it).
        exists_spy = mocker.spy(PaperlessSqliteVecVectorStore, "table_exists")
        store.upsert_document(1, [make_node("a2", 1)])
        assert exists_spy.call_count == 1


class TestBuildWhere:
    def test_ne_filter_translates_to_not_equal_clause(self) -> None:
        where, params = _build_where(_ne_filter(1))
        assert where == "(document_id != ?)"
        assert params == [1]

    def test_query_with_ne_filter_excludes_matching_document(self, store) -> None:
        store.add([make_node("a1", 1), make_node("b1", 2)])
        assert sorted(
            _query(store, [0.0] * DIM, top_k=5, filters=_ne_filter(1)).ids,
        ) == [
            "b1",
        ]

    def test_nin_filter_translates_to_not_in_clause(self) -> None:
        where, params = _build_where(_nin_filter([1, 2]))
        assert where == "(document_id NOT IN (?,?))"
        assert params == [1, 2]

    def test_query_with_nin_filter_excludes_matching_documents(self, store) -> None:
        store.add([make_node("a1", 1), make_node("b1", 2), make_node("c1", 3)])
        assert sorted(
            _query(store, [0.0] * DIM, top_k=5, filters=_nin_filter([1, 2])).ids,
        ) == ["c1"]

    def test_empty_in_filter_excludes_everything(self) -> None:
        """
        GIVEN:
            - An IN filter with an empty value list
        WHEN:
            - _build_where() translates it to SQL
        THEN:
            - It excludes everything (the opposite of an empty NOT IN
              filter) -- an empty inclusion list must never widen results
        """
        where, params = _build_where(_in_filter([]))
        assert where == "(1 = 0)"
        assert params == []

    def test_empty_nin_filter_excludes_nothing(self) -> None:
        """
        GIVEN:
            - A NOT IN filter with an empty value list -- e.g. an
              unrestricted chat caller when nothing is currently trashed
        WHEN:
            - _build_where() translates it to SQL
        THEN:
            - It excludes nothing (unlike an empty IN filter, which
              excludes everything) -- an empty exclusion list must never
              narrow results
        """
        where, params = _build_where(_nin_filter([]))
        assert where == "(1 = 1)"
        assert params == []

    def test_fails_closed_when_no_filter_is_translatable(self) -> None:
        # A nested MetadataFilters is not a MetadataFilter, so it is skipped.
        # With no translatable clauses, the function must fail closed rather
        # than emit "()" (invalid SQL) and never widen document access.
        nested = MetadataFilters(
            filters=[
                MetadataFilter(
                    key="document_id",
                    operator=FilterOperator.EQ,
                    value=1,
                ),
            ],
        )
        where, params = _build_where(MetadataFilters(filters=[nested]))
        assert where == "1 = 0"
        assert params == []

    @pytest.mark.parametrize(
        "build_filter",
        [_in_filter, _nin_filter],
        ids=["in", "nin"],
    )
    def test_fails_closed_when_filter_exceeds_max_values(
        self,
        build_filter: Callable[[list[str]], MetadataFilters],
        caplog: pytest.LogCaptureFixture,
    ) -> None:
        """
        GIVEN:
            - An IN or NOT IN filter with more values than _MAX_IN_VALUES
              (SQLite's own bound-parameter limit is 32766; this guard sits
              below that with headroom for the query's other bound parameters)
        WHEN:
            - _build_where() translates it to SQL
        THEN:
            - It fails closed ("1 = 0", no params) instead of building a
              clause SQLite would reject, and logs a warning -- this filter
              scopes document access, so refusing to build it must never
              widen the scope to "everything" by accident. Failing open on
              a NOT IN would surface exactly the excluded rows
        """
        oversized = build_filter([str(i) for i in range(_MAX_IN_VALUES + 1)])

        with caplog.at_level("WARNING"):
            where, params = _build_where(oversized)

        assert where == "(1 = 0)"
        assert params == []
        assert "document_id" in caplog.text

    def test_query_with_untranslatable_filter_returns_no_rows(
        self,
        store: PaperlessSqliteVecVectorStore,
    ) -> None:
        """
        GIVEN:
            - A store with one chunk each for documents 1 and 2
        WHEN:
            - A query and a get_nodes() call are issued with a filter whose
              only clause is an untranslatable nested MetadataFilters
        THEN:
            - Neither call raises, and both fail closed by returning no rows
              (rather than emitting invalid or unfiltered SQL)
        """
        store.add([make_node("a1", "1"), make_node("b1", "2")])
        nested = MetadataFilters(
            filters=[
                MetadataFilter(
                    key="document_id",
                    operator=FilterOperator.EQ,
                    value=1,
                ),
            ],
        )
        filters = MetadataFilters(filters=[nested])
        # Must not raise (no "WHERE ()") and must return nothing (fail closed).
        assert _query(store, [0.0] * DIM, top_k=5, filters=filters).ids == []
        assert store.get_nodes(filters=filters) == []


class TestUpsert:
    def test_upsert_replaces_and_prunes_stale_chunks(self, store) -> None:
        store.add(
            [make_node("d1c1", 1), make_node("d1c2", 1), make_node("d2c1", 2)],
        )
        store.upsert_document(1, [make_node("d1new", 1)])
        result = _query(store, [0.0] * DIM, top_k=10)
        assert sorted(result.ids) == ["d1new", "d2c1"]

    def test_upsert_creates_table_when_missing(self, store) -> None:
        store.upsert_document(1, [make_node("a1", 1)])
        assert _query(store, [0.0] * DIM, top_k=1).ids == ["a1"]

    def test_upsert_empty_nodes_removes_document(self, store) -> None:
        store.add([make_node("a1", 1), make_node("b1", 2)])
        store.upsert_document(1, [])
        assert _query(store, [0.0] * DIM, top_k=10).ids == ["b1"]

    def test_upsert_is_atomic_for_concurrent_readers(
        self,
        store,
        tmp_path: Path,
    ) -> None:
        """A second connection must never observe document 1 half-replaced."""
        store.add([make_node("a1", 1), make_node("a2", 1)])
        with PaperlessSqliteVecVectorStore(uri=str(tmp_path)) as reader:
            store.upsert_document(1, [make_node("a3", 1)])
            ids = [n.node_id for n in reader.get_nodes(filters=_in_filter([1]))]
            assert ids == ["a3"]


class TestMetadataCoercion:
    def test_none_metadata_values_become_empty_strings(self, store) -> None:
        node = make_node("a1", 1)
        node.metadata["modified"] = None
        store.add([node])  # must not raise (vec0 rejects NULL metadata)
        assert store.get_modified_times() == {"1": ""}


class TestModelNameTracking:
    def test_stored_model_name_none_without_table(self, tmp_path: Path) -> None:
        with PaperlessSqliteVecVectorStore(
            uri=str(tmp_path),
            embed_model_name="model-a",
        ) as store:
            assert store.stored_model_name() is None

    def test_model_name_stored_after_add_and_persists(self, tmp_path: Path) -> None:
        with PaperlessSqliteVecVectorStore(
            uri=str(tmp_path),
            embed_model_name="model-a",
        ) as store:
            store.add([make_node("a1", 1)])
            assert store.stored_model_name() == "model-a"
        with PaperlessSqliteVecVectorStore(uri=str(tmp_path)) as reopened:
            assert reopened.stored_model_name() == "model-a"

    def test_config_mismatch_semantics(self, tmp_path: Path) -> None:
        with PaperlessSqliteVecVectorStore(
            uri=str(tmp_path),
            embed_model_name="model-a",
        ) as store:
            assert not store.config_mismatch("anything")  # no table yet
            store.add([make_node("a1", 1)])
            assert not store.config_mismatch("model-a")
            assert store.config_mismatch("model-b")

    def test_config_mismatch_false_when_table_predates_tracking(
        self,
        tmp_path: Path,
    ) -> None:
        with PaperlessSqliteVecVectorStore(uri=str(tmp_path)) as store:  # no model name
            store.add([make_node("a1", 1)])
            assert not store.config_mismatch("model-a")


class TestGetModifiedTimes:
    def test_empty_store_returns_empty_dict(self, store) -> None:
        assert store.get_modified_times() == {}

    def test_returns_one_entry_per_document(self, store) -> None:
        store.add(
            [
                make_node("a1", 1, modified="2026-01-01T00:00:00"),
                make_node("a2", 1, modified="2026-01-01T00:00:00"),
                make_node("b1", 2, modified="2026-02-02T00:00:00"),
            ],
        )
        assert store.get_modified_times() == {
            "1": "2026-01-01T00:00:00",
            "2": "2026-02-02T00:00:00",
        }


class TestCompact:
    def _bloat_ratio(self, store) -> float:
        live = store.client.execute(
            "SELECT count(*) FROM documents",
        ).fetchone()[0]
        # vec0 0.1.9 does not accumulate deleted rows in the _rowids shadow
        # table, so we track cumulative inserts in index_meta instead.
        row = store.client.execute(
            "SELECT value FROM index_meta WHERE key = 'total_inserts'",
        ).fetchone()
        total = int(row["value"]) if row else live
        return total / max(live, 1)

    def _churn(self, store, cycles: int) -> None:
        for i in range(cycles):
            store.upsert_document(
                1,
                [make_node(f"gen{i}-{j}", 1, seed=float(j)) for j in range(20)],
            )

    def test_compact_noop_below_threshold(self, store) -> None:
        store.add([make_node("a1", 1)])
        store.compact()
        assert _query(store, [0.0] * DIM, top_k=1).ids == ["a1"]

    def test_force_compact_preserves_rows_and_metadata(self, store) -> None:
        store.add([make_node("a1", 1), make_node("b1", 2, seed=3.0)])
        self._churn(store, 5)
        before = {
            n.node_id: n.metadata for n in store.get_nodes(filters=_in_filter([1, 2]))
        }
        store.compact(force=True)
        after = {
            n.node_id: n.metadata for n in store.get_nodes(filters=_in_filter([1, 2]))
        }
        assert after == before
        assert self._bloat_ratio(store) == pytest.approx(1.0)
        # store remains fully usable after the rebuild; use a seed far from all
        # existing nodes (gen4-0..gen4-19 have seeds 0..19) so cosine KNN is
        # unambiguous at top_k=1.
        store.upsert_document(3, [make_node("c1", 3, seed=100.0)])
        assert "c1" in _query(store, [100.0] * DIM, top_k=1).ids

    def test_auto_compact_triggers_on_churn(self, store) -> None:
        store.add([make_node(f"s{j}", 1, seed=float(j)) for j in range(20)])
        self._churn(store, 5)
        assert self._bloat_ratio(store) > 2
        store.compact()
        assert self._bloat_ratio(store) == pytest.approx(1.0)

    def test_compact_on_missing_table_is_noop(self, store) -> None:
        store.compact()
        store.compact(force=True)

    def test_failed_compact_removes_temp_wal_and_shm(
        self,
        store,
        tmp_path: Path,
        monkeypatch,
    ) -> None:
        """A compact() that raises mid-rebuild must leave no .compact* files.

        Normally the sole connection's close() checkpoints the temp WAL away,
        but a concurrent reader keeps -wal/-shm alive, so the cleanup must
        unlink them explicitly (as the structural-migration path does).
        """
        store.add([make_node("a1", 1)])
        compact_path = str(tmp_path / DB_FILENAME) + ".compact"
        held: list[sqlite3.Connection] = []

        def boom(conn: sqlite3.Connection, dim: int) -> None:
            # Hold an extra connection so close() of the rebuild connection is
            # not the last one -> the temp -wal/-shm survive the checkpoint.
            extra = sqlite3.connect(compact_path)
            extra.execute("SELECT 1").fetchall()
            held.append(extra)
            raise RuntimeError("boom")

        monkeypatch.setattr(
            PaperlessSqliteVecVectorStore,
            "_create_vec_table",
            staticmethod(boom),
        )
        try:
            with pytest.raises(RuntimeError):
                store.compact(force=True)
            assert sorted(p.name for p in tmp_path.glob("*.compact*")) == []
        finally:
            for c in held:
                c.close()

    def test_force_compact_streams_rows_across_batches(
        self,
        store,
        monkeypatch,
    ) -> None:
        """Rebuild must preserve every row when rows span multiple batches.

        A tiny batch size forces several fetchmany()/executemany() cycles so a
        regression in the streaming loop (dropped tail, off-by-one) surfaces.
        """
        monkeypatch.setattr("paperless_ai.vector_store.BATCH_SIZE", 3)
        store.add([make_node(f"n{i}", 1, seed=float(i)) for i in range(10)])
        store.compact(force=True)
        ids = {n.node_id for n in store.get_nodes(filters=_in_filter([1]))}
        assert ids == {f"n{i}" for i in range(10)}
        assert self._bloat_ratio(store) == pytest.approx(1.0)

    def test_force_compact_preserves_modified_times(
        self,
        store: PaperlessSqliteVecVectorStore,
    ) -> None:
        """
        GIVEN:
            - A store with documents whose modified times are tracked
        WHEN:
            - compact(force=True) rebuilds the database file
        THEN:
            - get_modified_times() still returns every document's value
              (document_meta must be copied across the file-swap, not just
              the vec0 rows)
        """
        store.add(
            [
                make_node("a1", 1, modified="2026-01-01T00:00:00"),
                make_node("b1", 2, modified="2026-02-02T00:00:00"),
            ],
        )
        before = store.get_modified_times()
        store.compact(force=True)
        assert store.get_modified_times() == before

    def test_compact_on_unmigrated_store_is_noop(
        self,
        store: PaperlessSqliteVecVectorStore,
        mocker: MockerFixture,
    ) -> None:
        """
        GIVEN:
            - A store whose schema_version has been forced behind
              SCHEMA_VERSION (has_pending_migration() is True)
        WHEN:
            - compact(force=True) is called directly, without the caller
              having run check_and_run_migrations() first
        THEN:
            - compact() is a safe no-op: no file-swap rebuild is attempted
              at all (asserted via a spy on _rebuild_into, since schema_version
              alone is not a reliable signal -- a rebuild would otherwise
              copy the stale schema_version across unchanged, making a
              before/after equality check pass even when a rebuild *did*
              happen). Rebuilding an unmigrated store would silently lose
              document_meta and leave the swapped-in file claiming the old
              schema_version -- see the class docstring rationale.
        """
        store.add([make_node("a1", 1)])
        store.client.execute(
            "UPDATE index_meta SET value = '0' WHERE key = 'schema_version'",
        )
        assert store.has_pending_migration() is True
        rebuild_spy = mocker.spy(PaperlessSqliteVecVectorStore, "_rebuild_into")
        store.compact(force=True)
        rebuild_spy.assert_not_called()
        row = store.client.execute(
            "SELECT value FROM index_meta WHERE key = 'schema_version'",
        ).fetchone()
        assert int(row["value"]) == 0


class TestDbFile:
    def test_single_db_file_in_index_dir(self, store, tmp_path: Path) -> None:
        store.add([make_node("a1", 1)])
        assert (tmp_path / DB_FILENAME).exists()

    def test_wal_mode_enabled(self, store) -> None:
        assert (
            store.client.execute("PRAGMA journal_mode").fetchone()[0].lower() == "wal"
        )


class TestMigrations:
    """Tests for the schema migration machinery.

    These tests exercise check_and_run_migrations()'s generic dispatch logic
    (structural vs. re-embed, version-boundary stopping) using ad hoc test
    migrations layered on top of SCHEMA_VERSION -- distinct from
    TestV1ToV2Migration, which exercises the real, frozen m0001_v1_to_v2
    migration. Test migrations use version numbers starting at
    SCHEMA_VERSION (2) and above so they never collide with the real
    from_version=1/to_version=2 migration already registered in MIGRATIONS.

    The fake structural migrations' apply() fixtures (see
    test_structural_migration_copies_rows_and_updates_version and
    test_stop_at_reembed_boundary below) only populate the rebuilt vec0
    table itself -- they never insert into document_chunks/document_meta on
    the destination connection. That's fine here: these tests exist to
    verify the generic dispatch mechanism (version bookkeeping, structural-
    vs-reembed branching), not full schema correctness of a rebuilt store;
    the real migration's data completeness is covered separately by
    TestV1ToV2Migration.
    """

    def _schema_version(self, store: PaperlessSqliteVecVectorStore) -> int | None:
        row = store.client.execute(
            "SELECT value FROM index_meta WHERE key = 'schema_version'",
        ).fetchone()
        return int(row[0]) if row else None

    def test_new_table_records_schema_version(self, store) -> None:
        store.add([make_node("a1", 1)])
        assert self._schema_version(store) == SCHEMA_VERSION

    def test_check_migrations_no_table_returns_false(self, store) -> None:
        assert store.check_and_run_migrations() is False

    def test_check_migrations_current_version_returns_false(self, store) -> None:
        store.add([make_node("a1", 1)])
        assert store.check_and_run_migrations() is False

    def test_reembed_migration_returns_true(self, store, tmp_path: Path) -> None:
        store.add([make_node("a1", 1)])
        migration = Migration(
            from_version=SCHEMA_VERSION,
            to_version=SCHEMA_VERSION + 1,
            kind="re-embed",
            description="test re-embed",
        )
        MIGRATIONS.append(migration)
        try:
            from paperless_ai import vector_store as vs_mod

            original = vs_mod.SCHEMA_VERSION
            vs_mod.SCHEMA_VERSION = SCHEMA_VERSION + 1
            result = store.check_and_run_migrations()
        finally:
            MIGRATIONS.remove(migration)
            vs_mod.SCHEMA_VERSION = original
        assert result is True

    def test_structural_migration_copies_rows_and_updates_version(
        self,
        store,
        tmp_path: Path,
    ) -> None:
        store.add([make_node("a1", 1), make_node("b1", 2)])

        def apply(
            src: sqlite3.Connection,
            dst: sqlite3.Connection,
            dim: int,
        ) -> None:
            dst.execute(  # nosemgrep
                f"CREATE VIRTUAL TABLE {DEFAULT_TABLE_NAME} USING vec0("
                "id TEXT PRIMARY KEY, document_id TEXT, modified TEXT,"
                f" +node_content TEXT, embedding float[{dim}] distance_metric=cosine"
                ")",
            )
            dst.execute(
                "INSERT INTO index_meta (key, value) VALUES ('dim', ?) "
                "ON CONFLICT(key) DO UPDATE SET value = excluded.value",
                (str(dim),),
            )
            rows = src.execute(
                "SELECT id, document_id, node_content, embedding "
                f"FROM {DEFAULT_TABLE_NAME}",
            ).fetchall()
            dst.execute("BEGIN IMMEDIATE")
            dst.executemany(
                f"INSERT INTO {DEFAULT_TABLE_NAME} "
                "(id, document_id, modified, node_content, embedding) "
                "VALUES (?, ?, ?, ?, ?)",
                [
                    (
                        r["id"],
                        str(r["document_id"]),
                        "",
                        r["node_content"],
                        bytes(r["embedding"]),
                    )
                    for r in rows
                ],
            )
            dst.execute(
                "INSERT INTO index_meta (key, value) VALUES ('total_inserts', ?) "
                "ON CONFLICT(key) DO UPDATE SET value = excluded.value",
                (str(len(rows)),),
            )
            dst.execute("COMMIT")

        migration = Migration(
            from_version=SCHEMA_VERSION,
            to_version=SCHEMA_VERSION + 1,
            kind="structural",
            description="test structural",
            apply=apply,
        )
        MIGRATIONS.append(migration)
        try:
            from paperless_ai import vector_store as vs_mod

            original = vs_mod.SCHEMA_VERSION
            vs_mod.SCHEMA_VERSION = SCHEMA_VERSION + 1
            result = store.check_and_run_migrations()
        finally:
            MIGRATIONS.remove(migration)
            vs_mod.SCHEMA_VERSION = original

        assert result is False
        assert self._schema_version(store) == SCHEMA_VERSION + 1
        ids = {n.node_id for n in store.get_nodes()}
        assert ids == {"a1", "b1"}

    def test_compact_preserves_schema_version(self, store) -> None:
        store.add([make_node("a1", 1)])
        assert self._schema_version(store) == SCHEMA_VERSION
        store.compact(force=True)
        assert self._schema_version(store) == SCHEMA_VERSION

    def test_stop_at_reembed_boundary(self, store) -> None:
        # Registry: structural v(N+1), re-embed v(N+2), structural v(N+3),
        # where N = SCHEMA_VERSION. Only v(N+1) should apply; the re-embed
        # boundary must stop execution before v(N+3) runs, and the stored
        # version must stay at N+1.
        store.add([make_node("a1", 1), make_node("b1", 2)])

        def copy_apply(
            src: sqlite3.Connection,
            dst: sqlite3.Connection,
            dim: int,
        ) -> None:
            dst.execute(  # nosemgrep
                f"CREATE VIRTUAL TABLE {DEFAULT_TABLE_NAME} USING vec0("
                "id TEXT PRIMARY KEY, document_id TEXT, modified TEXT,"
                f" +node_content TEXT, embedding float[{dim}] distance_metric=cosine"
                ")",
            )
            dst.execute(
                "INSERT INTO index_meta (key, value) VALUES ('dim', ?) "
                "ON CONFLICT(key) DO UPDATE SET value = excluded.value",
                (str(dim),),
            )
            rows = src.execute(
                "SELECT id, document_id, node_content, embedding "
                f"FROM {DEFAULT_TABLE_NAME}",
            ).fetchall()
            dst.execute("BEGIN IMMEDIATE")
            dst.executemany(
                f"INSERT INTO {DEFAULT_TABLE_NAME} "
                "(id, document_id, modified, node_content, embedding) "
                "VALUES (?, ?, ?, ?, ?)",
                [
                    (
                        r["id"],
                        str(r["document_id"]),
                        "",
                        r["node_content"],
                        bytes(r["embedding"]),
                    )
                    for r in rows
                ],
            )
            dst.execute("COMMIT")

        migrations = [
            Migration(
                from_version=SCHEMA_VERSION,
                to_version=SCHEMA_VERSION + 1,
                kind="structural",
                description="v(N+1) structural",
                apply=copy_apply,
            ),
            Migration(
                from_version=SCHEMA_VERSION + 1,
                to_version=SCHEMA_VERSION + 2,
                kind="re-embed",
                description="v(N+2) re-embed boundary",
            ),
            Migration(
                from_version=SCHEMA_VERSION + 2,
                to_version=SCHEMA_VERSION + 3,
                kind="structural",
                description="v(N+3) structural - must not run",
                apply=copy_apply,
            ),
        ]
        MIGRATIONS.extend(migrations)
        try:
            from paperless_ai import vector_store as vs_mod

            original = vs_mod.SCHEMA_VERSION
            vs_mod.SCHEMA_VERSION = SCHEMA_VERSION + 3
            result = store.check_and_run_migrations()
        finally:
            for m in migrations:
                MIGRATIONS.remove(m)
            vs_mod.SCHEMA_VERSION = original

        assert result is True
        assert self._schema_version(store) == SCHEMA_VERSION + 1

    def test_has_pending_migration_false_when_no_table(
        self,
        store: PaperlessSqliteVecVectorStore,
    ) -> None:
        """
        GIVEN:
            - A vector store with no table created yet
        WHEN:
            - has_pending_migration() is checked
        THEN:
            - False is returned (nothing to migrate before anything exists)
        """
        assert store.has_pending_migration() is False

    def test_has_pending_migration_false_at_current_version(
        self,
        store: PaperlessSqliteVecVectorStore,
    ) -> None:
        """
        GIVEN:
            - A store at the current SCHEMA_VERSION
        WHEN:
            - has_pending_migration() is checked
        THEN:
            - False is returned
        """
        store.add([make_node("a1", 1)])
        assert store.has_pending_migration() is False

    def test_has_pending_migration_true_when_behind(
        self,
        store: PaperlessSqliteVecVectorStore,
    ) -> None:
        """
        GIVEN:
            - A store whose schema_version has been forced behind SCHEMA_VERSION
        WHEN:
            - has_pending_migration() is checked
        THEN:
            - True is returned
        """
        store.add([make_node("a1", 1)])
        store.client.execute(
            "UPDATE index_meta SET value = '0' WHERE key = 'schema_version'",
        )
        assert store.has_pending_migration() is True


class TestV1ToV2Migration:
    """m0001_v1_to_v2 migrates a real, historically-shaped v1 store. The
    fixture below is a literal, hardcoded v1 DDL string -- NOT derived from
    any current code -- so this test keeps testing the actual historical
    shape even if vector_store.py's "current" schema changes again later.
    """

    def _build_v1_store(self, db_path: str, dim: int) -> None:
        conn = sqlite3.connect(db_path)
        conn.row_factory = sqlite3.Row
        conn.enable_load_extension(True)  # noqa: FBT003
        sqlite_vec.load(conn)
        conn.enable_load_extension(False)  # noqa: FBT003
        conn.execute("PRAGMA journal_mode=WAL")
        conn.execute("PRAGMA synchronous=NORMAL")
        conn.execute(
            "CREATE TABLE IF NOT EXISTS index_meta (key TEXT PRIMARY KEY, value TEXT)",
        )
        conn.execute(  # nosemgrep
            "CREATE VIRTUAL TABLE documents USING vec0("
            "id TEXT PRIMARY KEY, document_id TEXT, modified TEXT,"
            f" +node_content TEXT, embedding float[{dim}] distance_metric=cosine"
            ")",
        )
        conn.execute(
            "INSERT INTO index_meta (key, value) VALUES ('dim', ?)",
            (str(dim),),
        )
        conn.execute(
            "INSERT INTO index_meta (key, value) VALUES ('schema_version', '1')",
        )
        conn.execute(
            "INSERT INTO index_meta (key, value) VALUES ('embed_model', 'model-a')",
        )
        rows = [
            ("c1", "1", "2026-01-01T00:00:00", '{"text": "a"}', _pack([0.1] * dim)),
            ("c2", "1", "2026-01-01T00:00:00", '{"text": "b"}', _pack([0.2] * dim)),
            ("c3", "2", "2026-02-02T00:00:00", '{"text": "c"}', _pack([0.3] * dim)),
        ]
        conn.executemany(
            "INSERT INTO documents (id, document_id, modified, node_content, embedding)"
            " VALUES (?, ?, ?, ?, ?)",
            rows,
        )
        conn.execute(
            "INSERT INTO index_meta (key, value) VALUES ('total_inserts', '3')",
        )
        conn.commit()
        conn.close()

    def test_migration_converts_v1_store_to_v2(self, tmp_path: Path) -> None:
        """
        GIVEN:
            - A real v1-shaped store (TEXT document_id, modified inline in
              vec0, no document_chunks/document_meta) built from a literal,
              hardcoded historical DDL
        WHEN:
            - A PaperlessSqliteVecVectorStore is opened against it
        THEN:
            - schema_version becomes 2, document_id values become int,
              document_chunks/document_meta are backfilled once per chunk/
              document respectively, and dim/embed_model survive
        """
        db_dir = tmp_path
        self._build_v1_store(str(db_dir / DB_FILENAME), dim=16)
        with PaperlessSqliteVecVectorStore(uri=str(db_dir)) as store:
            assert store.check_and_run_migrations() is False
            row = store.client.execute(
                "SELECT value FROM index_meta WHERE key = 'schema_version'",
            ).fetchone()
            assert int(row["value"]) == 2
            doc_id_row = store.client.execute(
                "SELECT document_id FROM documents WHERE id = 'c1'",
            ).fetchone()
            assert isinstance(doc_id_row["document_id"], int)
            assert doc_id_row["document_id"] == 1
            chunk_ids = sorted(
                r["chunk_id"]
                for r in store.client.execute(
                    "SELECT chunk_id FROM document_chunks",
                ).fetchall()
            )
            assert chunk_ids == ["c1", "c2", "c3"]
            assert store.get_modified_times() == {
                "1": "2026-01-01T00:00:00",
                "2": "2026-02-02T00:00:00",
            }
            assert store.stored_model_name() == "model-a"
            assert store.vector_dim() == 16

    def test_migration_raises_on_malformed_document_id(
        self,
        tmp_path: Path,
    ) -> None:
        """
        GIVEN:
            - A v1-shaped store with a corrupted, non-integer document_id
              value on one row
        WHEN:
            - The migration runs
        THEN:
            - A ValueError is raised (fail loudly, no silent data loss) --
              this matches the rest of vector_store.py, which has no
              precedent for silently skipping malformed rows
        """
        db_dir = tmp_path
        self._build_v1_store(str(db_dir / DB_FILENAME), dim=16)
        conn = sqlite3.connect(str(db_dir / DB_FILENAME))
        conn.enable_load_extension(True)  # noqa: FBT003
        sqlite_vec.load(conn)
        conn.enable_load_extension(False)  # noqa: FBT003
        conn.execute(
            "UPDATE documents SET document_id = 'not-an-int' WHERE id = 'c1'",
        )
        conn.commit()
        conn.close()
        with (
            pytest.raises(ValueError),
            PaperlessSqliteVecVectorStore(uri=str(db_dir)) as store,
        ):
            store.check_and_run_migrations()

    def test_migration_never_delegates_to_current_schema_helpers(
        self,
        tmp_path: Path,
        mocker: MockerFixture,
    ) -> None:
        """
        GIVEN:
            - A real v1-shaped store
        WHEN:
            - The migration runs, with DocumentChunksTable.create/
              DocumentMetaTable.create/_create_vec_table spied on
        THEN:
            - None of those "current schema" helpers are ever called during
              the migration -- it must freeze its own historical DDL, per
              the DDL-freezing rule (see spec), so a future schema bump
              can't silently corrupt this migration's output
        """
        db_dir = tmp_path
        self._build_v1_store(str(db_dir / DB_FILENAME), dim=16)
        chunks_create_spy = mocker.spy(DocumentChunksTable, "create")
        meta_create_spy = mocker.spy(DocumentMetaTable, "create")
        vec_table_spy = mocker.spy(
            PaperlessSqliteVecVectorStore,
            "_create_vec_table",
        )
        with PaperlessSqliteVecVectorStore(uri=str(db_dir)) as store:
            store.check_and_run_migrations()
        # _open_connection() legitimately calls create() three times across
        # a structural migration: once for the store's own live connection
        # (construction), once for the migration's temp rebuild file
        # (_rebuild_file), and once more when _swap_in_compact() reopens the
        # swapped-in file as self._conn. What matters is that
        # m0001_v1_to_v2's apply() itself never calls these directly, which
        # a source-text check alone can't prove (it can't tell "mentioned in
        # a comment" from "actually called", and is trivially evadable by
        # importing a symbol under an alias). Asserting the exact call count
        # instead: exactly 3 calls to each create() (all from
        # _open_connection, never a 4th from inside apply()), and zero calls
        # to _create_vec_table (neither _open_connection nor apply() calls
        # it -- apply() freezes its own literal CREATE VIRTUAL TABLE DDL
        # instead).
        assert chunks_create_spy.call_count == 3
        assert meta_create_spy.call_count == 3
        assert vec_table_spy.call_count == 0
        # Cheap secondary signal, kept alongside the spy assertions above
        # (not in place of them): the migration module's source should never
        # even mention these "current schema" helpers by name.
        source = inspect.getsource(m0001_v1_to_v2)
        assert "DocumentChunksTable.create" not in source
        assert "DocumentMetaTable.create" not in source
        assert "_create_vec_table(" not in source