Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

56.3 KB
Raw Blame
"""Unit tests for ``FaissVectorStore`` (``vector_stores/implementations/faiss_store.py``).

No DB is needed here — this module is DB-agnostic (ids + vectors only, see
the "SECURITY INVARIANT" block in ``vector_stores/base.py``). Persistence
resources (path / lock / integrity hooks) are injected fakes: a ``tmp_path``,
a real ``threading.Lock()``, and no-op integrity callables.
"""

import threading

import faiss
import numpy as np
import pytest

from local_deep_research.vector_stores.implementations.faiss_store import (
    FaissVectorStore,
)


# --------------------------------------------------------------------------
# Fakes / helpers
# --------------------------------------------------------------------------
def _noop_record(_path):
    return None


def _ok_verify(_path):
    return (True, None)


def _make_store(
    tmp_path,
    *,
    dimension=4,
    index_type="flat",
    metric="cosine",
    normalize=True,
    bind=True,
    integrity_record=_noop_record,
    integrity_verify=_ok_verify,
    lock=None,
):
    kwargs = dict(
        dimension=dimension,
        index_type=index_type,
        metric=metric,
        normalize=normalize,
    )
    if bind:
        kwargs.update(
            path=tmp_path / "coll.faiss",
            lock=lock or threading.Lock(),
            integrity_record=integrity_record,
            integrity_verify=integrity_verify,
        )
    return FaissVectorStore.create(**kwargs)


def _vecs(n, dim=4, seed=0):
    rng = np.random.default_rng(seed)
    return rng.random((n, dim)).astype("float32")


# --------------------------------------------------------------------------
# create / add / count / live_ids
# --------------------------------------------------------------------------
class TestCreateAddCount:
    def test_create_empty_store(self, tmp_path):
        store = _make_store(tmp_path)
        assert store.count() == 0
        assert store.live_ids() == []

    def test_add_increases_count_and_live_ids(self, tmp_path):
        store = _make_store(tmp_path)
        store.add([10, 20, 30], _vecs(3))
        assert store.count() == 3
        assert set(store.live_ids()) == {10, 20, 30}

    def test_add_empty_ids_is_noop(self, tmp_path):
        store = _make_store(tmp_path)
        store.add([], np.empty((0, 4), dtype="float32"))
        assert store.count() == 0

    def test_add_ids_vectors_length_mismatch_raises(self, tmp_path):
        store = _make_store(tmp_path)
        with pytest.raises(ValueError, match="length mismatch"):
            store.add([1, 2, 3], _vecs(2))

    def test_add_dimension_mismatch_raises(self, tmp_path):
        """Guards the python -O silent-corruption case: faiss's own C++
        assert on dimension is stripped under -O, so the store must
        validate explicitly (see the comment in ``_prepare``)."""
        store = _make_store(tmp_path, dimension=4)
        with pytest.raises(ValueError, match="vector dimension"):
            store.add([1], _vecs(1, dim=8))

    def test_int64_ids_beyond_int32(self, tmp_path):
        big_id = 5_000_000_000  # > 2**31, would overflow int32
        store = _make_store(tmp_path)
        store.add([big_id], _vecs(1))
        assert store.live_ids() == [big_id]
        hits = store.search(_vecs(1)[0], 1)
        assert hits[0][0] == big_id


# --------------------------------------------------------------------------
# search: cosine vs l2 ordering
# --------------------------------------------------------------------------
class TestSearchOrdering:
    def test_cosine_orders_by_descending_similarity(self, tmp_path):
        store = _make_store(
            tmp_path, dimension=2, metric="cosine", normalize=True
        )
        # query points along +x axis. id 1 is near-parallel (high cosine
        # sim), id 2 is near-orthogonal (low), id 3 points away (negative).
        store.add(
            [1, 2, 3],
            np.array([[1.0, 0.05], [0.05, 1.0], [-1.0, 0.05]], dtype="float32"),
        )
        hits = store.search(np.array([1.0, 0.0], dtype="float32"), 3)
        ids = [i for i, _ in hits]
        assert ids == [1, 2, 3]
        # inner product on normalized vectors: higher is more similar,
        # so scores must be descending.
        scores = [d for _, d in hits]
        assert scores == sorted(scores, reverse=True)

    def test_l2_orders_by_ascending_distance(self, tmp_path):
        store = _make_store(tmp_path, dimension=2, metric="l2", normalize=False)
        store.add(
            [1, 2, 3],
            np.array([[0.1, 0.1], [5.0, 5.0], [1.0, 1.0]], dtype="float32"),
        )
        hits = store.search(np.array([0.0, 0.0], dtype="float32"), 3)
        ids = [i for i, _ in hits]
        assert ids == [1, 3, 2]  # nearest (0.1,0.1) -> (1,1) -> (5,5)
        dists = [d for _, d in hits]
        assert dists == sorted(dists)

    def test_search_k_le_zero_returns_empty(self, tmp_path):
        store = _make_store(tmp_path)
        store.add([1], _vecs(1))
        assert store.search(_vecs(1)[0], 0) == []
        assert store.search(_vecs(1)[0], -1) == []

    def test_search_empty_index_returns_empty(self, tmp_path):
        store = _make_store(tmp_path)
        assert store.search(_vecs(1)[0], 5) == []

    def test_search_empty_store_rejects_wrong_dimension_query(self, tmp_path):
        store = _make_store(tmp_path, dimension=4)
        query = _vecs(1, dim=8)[0]

        with pytest.raises(ValueError):
            store.search(query, 1)

    def test_search_populated_store_rejects_wrong_dimension_query(
        self, tmp_path
    ):
        # NOTE: this does NOT discriminate the `_prepare`-before-`ntotal==0`
        # reorder in `search()` — a POPULATED index never takes the early
        # `return []` under either ordering, so this still passes with that
        # reorder fully reverted. The real pin for the reorder is the
        # EMPTY-store sibling above
        # (test_search_empty_store_rejects_wrong_dimension_query): before
        # the reorder, an empty store returned `[]` for a wrong-dimension
        # query instead of raising. Keep both — don't delete the empty-store
        # test as a "duplicate" of this one.
        store = _make_store(tmp_path, dimension=4)
        store.add([1], _vecs(1, dim=4))
        query = _vecs(1, dim=8)[0]

        with pytest.raises(ValueError):
            store.search(query, 1)

    def test_search_fewer_hits_than_k_drops_empty_slots(self, tmp_path):
        """faiss returns -1 for empty slots when fewer than k neighbors
        exist; those must be filtered out, not surfaced as id -1."""
        store = _make_store(tmp_path)
        store.add([7], _vecs(1))
        hits = store.search(_vecs(1)[0], 5)
        assert len(hits) == 1
        assert hits[0][0] == 7

    def test_search_k_less_than_total_returns_nearest_k_in_order(
        self, tmp_path
    ):
        # Existing ordering tests use k>=ntotal; this covers k strictly less
        # than ntotal, where faiss must still return only the k CLOSEST hits
        # (not an arbitrary k-subset) in ranked order.
        store = _make_store(tmp_path, dimension=2, metric="l2", normalize=False)
        store.add(
            [1, 2, 3, 4, 5],
            np.array(
                [
                    [0.1, 0.1],
                    [5.0, 5.0],
                    [1.0, 1.0],
                    [2.0, 2.0],
                    [10.0, 10.0],
                ],
                dtype="float32",
            ),
        )
        hits = store.search(np.array([0.0, 0.0], dtype="float32"), 2)
        assert len(hits) == 2
        ids = [i for i, _ in hits]
        # Two nearest to the origin: (0.1,0.1) then (1,1).
        assert ids == [1, 3]
        dists = [d for _, d in hits]
        assert dists == sorted(dists)


# --------------------------------------------------------------------------
# delete / live_ids / count / reconstruct
# --------------------------------------------------------------------------
class TestDeleteLiveIdsReconstruct:
    def test_delete_removes_and_returns_count(self, tmp_path):
        store = _make_store(tmp_path)
        store.add([1, 2, 3], _vecs(3))
        removed = store.delete([2, 3])
        assert removed == 2
        assert store.live_ids() == [1]
        assert store.count() == 1

    def test_delete_empty_list_is_noop(self, tmp_path):
        store = _make_store(tmp_path)
        store.add([1], _vecs(1))
        assert store.delete([]) == 0
        assert store.count() == 1

    def test_delete_none_entries_filtered(self, tmp_path):
        store = _make_store(tmp_path)
        store.add([1], _vecs(1))
        assert store.delete([None, None]) == 0

    def test_reconstruct_missing_id_returns_none(self, tmp_path):
        store = _make_store(tmp_path)
        store.add([1], _vecs(1))
        assert store.reconstruct(999) is None

    def test_reconstruct_existing_id_returns_vector(self, tmp_path):
        store = _make_store(tmp_path, dimension=3, metric="l2", normalize=False)
        vec = np.array([[1.0, 2.0, 3.0]], dtype="float32")
        store.add([5], vec)
        out = store.reconstruct(5)
        assert out is not None
        np.testing.assert_allclose(np.asarray(out), vec[0], atol=1e-5)

    def test_reconstruct_after_delete_returns_none(self, tmp_path):
        # Distinct from a never-added id: this id WAS live, then removed via
        # remove_ids -- the Optional contract _rebuild_dropping/integrity
        # re-embed rely on must hold for a removed id too, not just an absent one.
        store = _make_store(tmp_path, dimension=3, metric="l2", normalize=False)
        store.add([1, 2], _vecs(2, dim=3))
        removed = store.delete([1])
        assert removed == 1
        assert store.reconstruct(1) is None
        assert store.reconstruct(999) is None  # never-added id, for contrast
        assert (
            store.reconstruct(2) is not None
        )  # untouched id still reconstructs

    def test_supports_delete_false_for_hnsw(self, tmp_path):
        store = _make_store(tmp_path, index_type="hnsw")
        assert store.supports_delete is False

    def test_hnsw_delete_raises_clear_runtime_error(self, tmp_path):
        store = _make_store(tmp_path, index_type="hnsw")
        store.add([1], _vecs(1))
        with pytest.raises(RuntimeError, match="hnsw"):
            store.delete([1])

    def test_hnsw_apply_remove_ids_rebuilds(self, tmp_path):
        # apply() must NOT raise for HNSW removals: faiss HNSW has no
        # remove_ids, so apply() drops-by-rebuild (_rebuild_dropping) instead.
        store = _make_store(tmp_path, index_type="hnsw")
        store.apply(add_ids=[1, 2, 3], add_vectors=_vecs(3))
        stats = store.apply(add_ids=[], add_vectors=None, remove_ids=[2])
        assert stats["removed"] == 1
        assert set(store.live_ids()) == {1, 3}
        assert store.count() == 2

    def test_hnsw_apply_replace_same_id_rebuilds(self, tmp_path):
        # remove + add of the SAME id (a "replace", used by re-index) must
        # succeed for HNSW via the rebuild path — the original crash repro.
        store = _make_store(tmp_path, index_type="hnsw")
        store.apply(add_ids=[1, 2], add_vectors=_vecs(2))
        newvec = _vecs(1, seed=99)
        store.apply(add_ids=[1], add_vectors=newvec, remove_ids=[1])
        assert set(store.live_ids()) == {1, 2}

    def test_hnsw_apply_remove_all_and_absent(self, tmp_path):
        store = _make_store(tmp_path, index_type="hnsw")
        store.apply(add_ids=[1, 2, 3], add_vectors=_vecs(3))
        # Absent id: leniently ignored, nothing removed.
        assert (
            store.apply(add_ids=[], add_vectors=None, remove_ids=[999])[
                "removed"
            ]
            == 0
        )
        # Remove every id: empties to 0 without crashing.
        stats = store.apply(add_ids=[], add_vectors=None, remove_ids=[1, 2, 3])
        assert stats["removed"] == 3
        assert store.count() == 0

    def test_hnsw_rebuild_survives_reload(self, tmp_path):
        # The rebuilt index must persist and reload as a valid IndexIDMap2.
        store = _make_store(tmp_path, index_type="hnsw")
        store.apply(add_ids=[1, 2, 3, 4], add_vectors=_vecs(4))
        store.apply(add_ids=[], add_vectors=None, remove_ids=[2, 4])
        reloaded = FaissVectorStore.load(
            tmp_path / "coll.faiss",
            dimension=4,
            index_type="hnsw",
            metric="cosine",
            normalize=True,
        )
        assert set(reloaded.live_ids()) == {1, 3}


# --------------------------------------------------------------------------
# persist / load
# --------------------------------------------------------------------------
class TestPersistLoad:
    def test_persist_is_atomic_no_tmp_left(self, tmp_path):
        store = _make_store(tmp_path)
        store.add([1, 2], _vecs(2))
        path = tmp_path / "out.faiss"
        store.persist(path)
        assert path.exists()
        leftover_tmp = list(tmp_path.glob("*.tmp"))
        assert leftover_tmp == []

    def test_persist_write_index_failure_leaves_original_file_untouched_and_no_tmp(
        self, tmp_path, monkeypatch
    ):
        """A failure INSIDE the real write (the module-level ``write_index``
        imported ~line 37) must never corrupt or replace the last-good
        durable file: persist()'s temp+rename means ``write_index`` raising
        happens strictly before ``os.replace``, so the original bytes must
        be untouched, and the temp file must be cleaned up by persist()'s
        own ``finally`` block rather than left dangling as ``*.tmp`` litter.

        Existing persist-failure tests mock the whole ``store.persist``
        method; this exercises the real mid-write sequence, which those
        leave uncovered.
        """
        store = _make_store(tmp_path)
        store.add([1, 2], _vecs(2))
        path = tmp_path / "out.faiss"
        store.persist(path)
        assert path.exists()
        original_bytes = path.read_bytes()

        def _boom(index, filename):
            raise RuntimeError("simulated write_index failure")

        monkeypatch.setattr(
            "local_deep_research.vector_stores.implementations."
            "faiss_store.write_index",
            _boom,
        )

        # Mutate the in-memory index further so a torn overwrite (if it
        # happened) would be detectably different from the captured bytes.
        store.add([3], _vecs(1, seed=5))

        with pytest.raises(RuntimeError, match="simulated write_index failure"):
            store.persist(path)

        assert path.read_bytes() == original_bytes  # no torn overwrite
        assert list(tmp_path.glob("*.tmp")) == []  # temp cleaned up

    def test_persist_writes_no_pkl_companion(self, tmp_path):
        store = _make_store(tmp_path)
        store.add([1], _vecs(1))
        path = tmp_path / "out.faiss"
        store.persist(path)
        assert list(tmp_path.glob("*.pkl")) == []

    def test_apply_reload_rejects_dimension_mismatch(self, tmp_path):
        # A store bound to a path whose on-disk index has a DIFFERENT dimension
        # (embedding model/config changed) must fail closed on apply()'s
        # reload-under-lock, not silently add a wrong-width vector (which faiss
        # would only catch with an assert stripped under `python -O`).
        first = _make_store(tmp_path, dimension=4)
        first.apply(add_ids=[1], add_vectors=_vecs(1, dim=4))

        mismatched = _make_store(tmp_path, dimension=8)  # same default path
        with pytest.raises(RuntimeError, match="dimension"):
            mismatched.apply(add_ids=[2], add_vectors=_vecs(1, dim=8))

    @pytest.mark.skipif(
        __import__("sys").platform == "win32",
        reason="chmod is not supported on Windows",
    )
    def test_persist_chmods_parent_private(self, tmp_path):
        store = _make_store(tmp_path)
        store.add([1], _vecs(1))
        d = tmp_path / "sub" / "out.faiss"
        store.persist(d)
        mode = d.parent.stat().st_mode & 0o777
        assert mode == 0o700

    def test_load_round_trip(self, tmp_path):
        store = _make_store(
            tmp_path, dimension=4, metric="cosine", normalize=True
        )
        store.add([1, 2, 3], _vecs(3))
        path = tmp_path / "rt.faiss"
        store.persist(path)

        loaded = FaissVectorStore.load(
            path,
            dimension=4,
            index_type="flat",
            metric="cosine",
            normalize=True,
        )
        assert set(loaded.live_ids()) == {1, 2, 3}
        assert loaded.count() == 3

    def test_load_raises_on_non_idmap2_file(self, tmp_path):
        """post-migration every on-disk index must be IndexIDMap2; a raw
        base index means pre-migration or corruption and must surface."""
        path = tmp_path / "raw.faiss"
        raw = faiss.IndexFlatL2(4)
        faiss.write_index(raw, str(path))

        with pytest.raises(ValueError, match="not IndexIDMap2"):
            FaissVectorStore.load(
                path,
                dimension=4,
                index_type="flat",
                metric="l2",
                normalize=False,
            )

    def test_load_raises_on_dimension_mismatch(self, tmp_path):
        store = _make_store(tmp_path, dimension=4)
        store.add([1], _vecs(1, dim=4))
        path = tmp_path / "dim.faiss"
        store.persist(path)

        with pytest.raises(ValueError, match="dimension"):
            FaissVectorStore.load(
                path,
                dimension=8,
                index_type="flat",
                metric="cosine",
                normalize=True,
            )


# --------------------------------------------------------------------------
# apply(): add / dedup / replace / purge / empty
# --------------------------------------------------------------------------
class TestApply:
    def test_apply_requires_persistence_binding(self, tmp_path):
        store = _make_store(tmp_path, bind=False)
        with pytest.raises(RuntimeError, match="persistence binding"):
            store.apply(add_ids=[1], add_vectors=_vecs(1))

    def test_apply_add_ids_without_vectors_raises(self, tmp_path):
        store = _make_store(tmp_path)
        with pytest.raises(ValueError, match="add_vectors"):
            store.apply(add_ids=[1], add_vectors=None)

    def test_apply_basic_add(self, tmp_path):
        store = _make_store(tmp_path)
        stats = store.apply(add_ids=[1, 2], add_vectors=_vecs(2))
        assert stats == {"added": 2, "removed": 0}
        assert set(store.live_ids()) == {1, 2}

    def test_apply_dedup_within_batch(self, tmp_path):
        store = _make_store(tmp_path)
        # id 1 repeated in the same batch -> only the first occurrence kept.
        stats = store.apply(add_ids=[1, 1, 2], add_vectors=_vecs(3))
        assert stats["added"] == 2
        assert set(store.live_ids()) == {1, 2}

    def test_apply_replaces_live_id_not_skips(self, tmp_path):
        # An id already live is REPLACED (removed then re-added), NOT skipped:
        # the new vector must win (reused-id safety — SQLite recycles
        # AUTOINCREMENT ids on ROLLBACK, so a live id may carry a stale orphan).
        store = _make_store(tmp_path)
        store.apply(add_ids=[1], add_vectors=_vecs(1))
        stats = store.apply(add_ids=[1, 2], add_vectors=_vecs(2))
        assert stats["added"] == 2  # id 1 replaced + id 2 added
        assert stats["removed"] == 1  # stale id 1 removed first
        assert set(store.live_ids()) == {1, 2}
        assert store.count() == 2  # no duplicate id 1

    def test_apply_replace_on_collision_is_unconditional(self, tmp_path):
        """Replace-on-collision runs even with dedup=False: a live id is
        removed before re-add, so IndexIDMap2 never accrues a duplicate key
        (which would cause phantom hits and the reused-id text leak)."""
        store = _make_store(tmp_path)
        store.apply(add_ids=[1], add_vectors=_vecs(1))
        store.apply(add_ids=[1], add_vectors=_vecs(1), dedup=False)
        assert store.count() == 1  # replaced, not duplicated

    def test_apply_replace_same_id(self, tmp_path):
        store = _make_store(tmp_path, dimension=3, metric="l2", normalize=False)
        v1 = np.array([[1.0, 0.0, 0.0]], dtype="float32")
        store.apply(add_ids=[1], add_vectors=v1)

        v2 = np.array([[9.0, 9.0, 9.0]], dtype="float32")
        stats = store.apply(add_ids=[1], add_vectors=v2, remove_ids=[1])
        assert stats == {"added": 1, "removed": 1}
        assert store.live_ids() == [1]
        np.testing.assert_allclose(
            np.asarray(store.reconstruct(1)), v2[0], atol=1e-5
        )

    def test_apply_purge(self, tmp_path):
        store = _make_store(tmp_path)
        store.apply(add_ids=[1, 2, 3], add_vectors=_vecs(3))
        stats = store.apply(add_ids=[], add_vectors=None, remove_ids=[1, 2, 3])
        assert stats["removed"] == 3
        assert store.count() == 0

    def test_apply_empty_is_noop_but_persists(self, tmp_path):
        store = _make_store(tmp_path)
        stats = store.apply(add_ids=[], add_vectors=None, remove_ids=[])
        assert stats == {"added": 0, "removed": 0}
        assert store._path.exists()
        assert store.count() == 0


# --------------------------------------------------------------------------
# Cross-instance reload-under-lock (#4200 guarantee)
# --------------------------------------------------------------------------
class TestCrossInstanceReloadUnderLock:
    def test_second_instance_absorbs_first_instances_write(self, tmp_path):
        """Two stores share the same path + lock (as the service layer's
        cached-instance-per-collection model does). Instance A adds+persists
        first; instance B was constructed independently (its in-memory index
        does NOT yet know about A's write). B.apply() must reload the
        on-disk file under the lock and merge onto it — losing A's write
        here would be exactly the #4200 regression."""
        path = tmp_path / "shared.faiss"
        lock = threading.Lock()
        dim = 4

        store_a = FaissVectorStore.create(
            dimension=dim,
            index_type="flat",
            metric="cosine",
            normalize=True,
            path=path,
            lock=lock,
            integrity_record=_noop_record,
            integrity_verify=_ok_verify,
        )
        store_b = FaissVectorStore.create(
            dimension=dim,
            index_type="flat",
            metric="cosine",
            normalize=True,
            path=path,
            lock=lock,
            integrity_record=_noop_record,
            integrity_verify=_ok_verify,
        )

        store_a.apply(add_ids=[1, 2], add_vectors=_vecs(2, dim=dim))
        assert path.exists()

        store_b.apply(add_ids=[3], add_vectors=_vecs(1, dim=dim, seed=99))

        assert set(store_b.live_ids()) == {1, 2, 3}
        reloaded = FaissVectorStore.load(
            path,
            dimension=dim,
            index_type="flat",
            metric="cosine",
            normalize=True,
        )
        assert set(reloaded.live_ids()) == {1, 2, 3}

    def test_reload_fails_closed_on_bad_integrity(self, tmp_path):
        """If the on-disk file exists but integrity_verify says it can't be
        trusted, apply() must refuse to overwrite it (fail closed) rather
        than silently clobbering a concurrent writer's committed save."""
        path = tmp_path / "shared.faiss"
        lock = threading.Lock()
        dim = 4

        store_a = FaissVectorStore.create(
            dimension=dim,
            index_type="flat",
            metric="cosine",
            normalize=True,
            path=path,
            lock=lock,
            integrity_record=_noop_record,
            integrity_verify=_ok_verify,
        )
        store_a.apply(add_ids=[1], add_vectors=_vecs(1, dim=dim))

        def _bad_verify(_path):
            return (False, "checksum mismatch")

        store_b = FaissVectorStore.create(
            dimension=dim,
            index_type="flat",
            metric="cosine",
            normalize=True,
            path=path,
            lock=lock,
            integrity_record=_noop_record,
            integrity_verify=_bad_verify,
        )
        with pytest.raises(RuntimeError, match="Refusing to overwrite"):
            store_b.apply(add_ids=[2], add_vectors=_vecs(1, dim=dim))

    def test_apply_reload_fails_closed_on_unreadable_ondisk_file_and_does_not_clobber(
        self, tmp_path
    ):
        """If the on-disk file exists, integrity_verify says it's fine, but
        read_index itself fails (corruption), apply()'s reload-under-lock must
        fail closed (raise) rather than silently overwrite the corrupt/foreign
        bytes with our in-memory copy -- that would clobber whatever a
        concurrent writer had committed there."""
        path = tmp_path / "shared.faiss"
        lock = threading.Lock()
        dim = 4

        store_a = FaissVectorStore.create(
            dimension=dim,
            index_type="flat",
            metric="cosine",
            normalize=True,
            path=path,
            lock=lock,
            integrity_record=_noop_record,
            integrity_verify=_ok_verify,  # isolates the read_index branch
        )
        store_a.apply(add_ids=[1], add_vectors=_vecs(1, dim=dim))

        # Corrupt the on-disk bytes so read_index() itself raises.
        path.write_bytes(b"not a valid faiss index" * 8)
        corrupted_bytes = path.read_bytes()

        store_b = FaissVectorStore.create(
            dimension=dim,
            index_type="flat",
            metric="cosine",
            normalize=True,
            path=path,
            lock=lock,
            integrity_record=_noop_record,
            integrity_verify=_ok_verify,
        )
        with pytest.raises(RuntimeError, match="failed to reload"):
            store_b.apply(add_ids=[2], add_vectors=_vecs(1, dim=dim, seed=2))

        # Fail-closed: the corrupt bytes must be untouched, not overwritten.
        assert path.read_bytes() == corrupted_bytes


# --------------------------------------------------------------------------
# _read_guard: poisoned instance refuses reads
# --------------------------------------------------------------------------
class TestReadGuardPoisoned:
    def test_poisoned_unlocked_instance_refuses_count(self, tmp_path):
        store = _make_store(tmp_path, bind=False)
        store._poisoned = True
        with pytest.raises(RuntimeError, match="poisoned"):
            store.count()

    def test_poisoned_locked_instance_refuses_search(self, tmp_path):
        store = _make_store(tmp_path)
        store._poisoned = True
        with pytest.raises(RuntimeError, match="poisoned"):
            store.search(_vecs(1)[0], 1)

    def test_poisoned_refuses_live_ids_and_reconstruct(self, tmp_path):
        store = _make_store(tmp_path)
        store._poisoned = True
        with pytest.raises(RuntimeError, match="poisoned"):
            store.live_ids()
        with pytest.raises(RuntimeError, match="poisoned"):
            store.reconstruct(1)


# --------------------------------------------------------------------------
# integrity_record: retry then raise
# --------------------------------------------------------------------------
class TestIntegrityRecordRetry:
    def test_record_integrity_retries_then_raises(self, tmp_path):
        calls = []

        def _always_fails(_path):
            calls.append(1)
            raise OSError("db hiccup")

        store = _make_store(
            tmp_path,
            integrity_record=_always_fails,
            integrity_verify=_ok_verify,
        )
        with pytest.raises(RuntimeError, match="failed to record"):
            store.apply(add_ids=[1], add_vectors=_vecs(1))

        assert len(calls) == 3  # default attempts=3
        # The vectors WERE persisted (persist() ran before the integrity
        # failure); a resync from disk keeps the in-memory state honest
        # rather than poisoning the instance over a checksum-recording bug.
        assert store.count() == 1
        assert store._poisoned is False

    def test_record_integrity_succeeds_on_second_attempt(self, tmp_path):
        calls = []

        def _fails_once(_path):
            calls.append(1)
            if len(calls) < 2:
                raise OSError("transient")

        store = _make_store(
            tmp_path, integrity_record=_fails_once, integrity_verify=_ok_verify
        )
        store.apply(add_ids=[1], add_vectors=_vecs(1))
        assert len(calls) == 2
        assert store.count() == 1

    def test_no_integrity_record_hook_is_noop(self, tmp_path):
        store = _make_store(tmp_path, integrity_record=None)
        # Must not raise despite no integrity hook.
        store.apply(add_ids=[1], add_vectors=_vecs(1))
        assert store.count() == 1


class TestNonFiniteVectorRejection:
    def test_nan_vector_rejected(self, tmp_path):
        store = _make_store(tmp_path)
        bad = np.full((1, 4), np.nan, dtype="float32")
        with pytest.raises(ValueError, match="non-finite"):
            store.add([1], bad)

    def test_inf_vector_rejected(self, tmp_path):
        store = _make_store(tmp_path)
        bad = np.array([[np.inf, 0.0, 0.0, 0.0]], dtype="float32")
        with pytest.raises(ValueError, match="non-finite"):
            store.add([1], bad)

    def test_zero_vector_rejected_under_normalize(self, tmp_path):
        # faiss normalize_L2 maps a zero-magnitude vector to all-zeros (not NaN),
        # so it would silently be indexed as a degenerate, unsearchable vector.
        # _prepare rejects it explicitly instead.
        store = _make_store(tmp_path, normalize=True)
        with pytest.raises(ValueError, match="zero-magnitude"):
            store.add([1], np.zeros((1, 4), dtype="float32"))
        assert store.count() == 0

    def test_zero_vector_allowed_without_normalize(self, tmp_path):
        # Without L2 normalization a zero vector is a valid (if degenerate)
        # entry — no NaN, no silent unsearchability from normalize — so it is
        # accepted rather than rejected.
        store = _make_store(tmp_path, normalize=False)
        store.add([1], np.zeros((1, 4), dtype="float32"))
        assert store.count() == 1

    def test_nan_query_rejected(self, tmp_path):
        store = _make_store(tmp_path)
        store.add([1], _vecs(1))
        with pytest.raises(ValueError, match="non-finite"):
            store.search(np.full((1, 4), np.nan, dtype="float32"), 1)


class TestReplaceOnIdCollision:
    """A reused DocumentChunk.id (SQLite recycles AUTOINCREMENT ids on ROLLBACK)
    must get the NEW vector, never silently keep a stale orphan vector under
    that id — otherwise search rehydrates the WRONG document's text."""

    def test_flat_collision_new_vector_wins(self, tmp_path):
        store = _make_store(tmp_path, dimension=3, metric="l2", normalize=False)
        store.apply(
            add_ids=[7],
            add_vectors=np.array([[1.0, 0.0, 0.0]], dtype="float32"),
        )
        new = np.array([[0.0, 1.0, 0.0]], dtype="float32")
        store.apply(add_ids=[7], add_vectors=new)  # same id, different vector
        assert store.count() == 1  # not duplicated
        np.testing.assert_allclose(
            np.asarray(store.reconstruct(7)), new[0], atol=1e-5
        )

    def test_hnsw_collision_new_vector_wins(self, tmp_path):
        store = _make_store(
            tmp_path,
            dimension=3,
            index_type="hnsw",
            metric="l2",
            normalize=False,
        )
        store.apply(
            add_ids=[7, 8],
            add_vectors=np.array(
                [[1.0, 0.0, 0.0], [0.0, 0.0, 1.0]], dtype="float32"
            ),
        )
        new = np.array([[0.0, 1.0, 0.0]], dtype="float32")
        store.apply(add_ids=[7], add_vectors=new)
        assert set(store.live_ids()) == {7, 8}
        np.testing.assert_allclose(
            np.asarray(store.reconstruct(7)), new[0], atol=1e-5
        )


class TestHnswOffLockRebuild:
    """The HNSW rebuild builds the new graph OFF the write lock, so a
    concurrent search() is not blocked for the (expensive) O(collection)
    rebuild — only for the fast snapshot + the final swap."""

    def test_offlock_rebuild_correctness(self, tmp_path):
        store = _make_store(
            tmp_path,
            index_type="hnsw",
            dimension=3,
            metric="l2",
            normalize=False,
        )
        store.apply(
            add_ids=[1, 2, 3],
            add_vectors=np.array(
                [[1.0, 0, 0], [0, 1.0, 0], [0, 0, 1.0]], dtype="float32"
            ),
        )
        # Remove id 2 AND replace id 1 with a new vector in one apply().
        newv = np.array([[9.0, 9.0, 9.0]], dtype="float32")
        stats = store.apply(add_ids=[1], add_vectors=newv, remove_ids=[2])
        assert stats["removed"] >= 1
        assert set(store.live_ids()) == {1, 3}
        np.testing.assert_allclose(
            np.asarray(store.reconstruct(1)), newv[0], atol=1e-5
        )
        reloaded = FaissVectorStore.load(
            tmp_path / "coll.faiss",
            dimension=3,
            index_type="hnsw",
            metric="l2",
            normalize=False,
        )
        assert set(reloaded.live_ids()) == {1, 3}

    def test_search_not_blocked_during_rebuild(self, tmp_path):
        import threading
        import time

        store = _make_store(
            tmp_path,
            index_type="hnsw",
            dimension=4,
            metric="l2",
            normalize=False,
        )
        store.apply(add_ids=[1, 2, 3, 4], add_vectors=_vecs(4, dim=4))

        build_started = threading.Event()
        reader_done = threading.Event()
        real_build = store._build_base_index  # staticmethod -> plain function

        def slow_build(dimension, index_type, metric):
            # Phase 2 runs OFF the lock; block here until the reader finishes.
            build_started.set()
            reader_done.wait(timeout=5)
            return real_build(dimension, index_type, metric)

        store._build_base_index = slow_build  # instance-shadow the staticmethod

        results = {}

        def do_rebuild():
            results["r"] = store.apply(
                add_ids=[], add_vectors=None, remove_ids=[2]
            )

        t = threading.Thread(target=do_rebuild)
        t.start()
        assert build_started.wait(timeout=5), "rebuild never reached Phase 2"

        # The off-lock build is blocked. A search must return WITHOUT waiting on
        # it (under the old under-lock design this would deadlock on the lock,
        # timing out the join below).
        t0 = time.monotonic()
        hits = store.search(_vecs(1, dim=4)[0], 4)
        elapsed = time.monotonic() - t0
        assert elapsed < 1.0, f"search blocked {elapsed:.2f}s during rebuild"
        # search saw the pre-swap (old) index, which still had id 2.
        assert 2 in {i for i, _ in hits}

        reader_done.set()
        t.join(timeout=5)
        assert not t.is_alive()
        assert results["r"]["removed"] == 1
        assert set(store.live_ids()) == {1, 3, 4}


class TestHnswOffLockRace:
    """A concurrent write during the off-lock rebuild bumps the version; the
    rebuild's swap must detect that and fall back to a fully-locked rebuild
    rather than clobbering the concurrent write (no lost update)."""

    def test_concurrent_add_during_rebuild_not_lost(self, tmp_path):
        import threading

        store = _make_store(
            tmp_path,
            index_type="hnsw",
            dimension=4,
            metric="l2",
            normalize=False,
        )
        store.apply(add_ids=[1, 2, 3], add_vectors=_vecs(3, dim=4))

        build_started = threading.Event()
        concurrent_done = threading.Event()
        real_build = store._build_base_index

        def slow_build(dimension, index_type, metric):
            build_started.set()
            concurrent_done.wait(timeout=5)
            return real_build(dimension, index_type, metric)

        store._build_base_index = slow_build

        def do_rebuild():
            store.apply(add_ids=[], add_vectors=None, remove_ids=[2])

        t = threading.Thread(target=do_rebuild)
        t.start()
        assert build_started.wait(timeout=5)

        # Concurrent pure-add of id 9 while the rebuild is off-lock building.
        # (Pure add takes no rebuild path, so it does not hit slow_build.)
        store.apply(add_ids=[9], add_vectors=_vecs(1, dim=4, seed=9))
        concurrent_done.set()
        t.join(timeout=5)
        assert not t.is_alive()

        # The rebuild removed 2; the concurrent add of 9 survived -> {1,3,9}.
        assert set(store.live_ids()) == {1, 3, 9}
        reloaded = FaissVectorStore.load(
            tmp_path / "coll.faiss",
            dimension=4,
            index_type="hnsw",
            metric="l2",
            normalize=False,
        )
        assert set(reloaded.live_ids()) == {1, 3, 9}


class TestHnswOffLockRetryExhaustion:
    """The off-lock rebuild retries a bounded number of times on a stale
    snapshot, then gives up and falls back to a fully-locked rebuild. Guards
    both the >1 retry-count guard (an off-by-one would either give up
    prematurely or recurse forever) and the fallback call itself, which
    re-acquires ``self._lock`` -- a non-reentrant ``threading.Lock`` -- from
    OUTSIDE any held lock, so it must not deadlock."""

    def test_hnsw_apply_gives_up_after_retries_and_falls_back_to_locked_rebuild(
        self, tmp_path
    ):
        store = _make_store(tmp_path, index_type="hnsw", dimension=4)
        store.apply(add_ids=[1, 2, 3], add_vectors=_vecs(3, dim=4))

        # Force the staleness check to ALWAYS see a mismatch: an ever-changing
        # fingerprint differs between the Phase-1 snapshot and the Phase-3
        # recheck on every attempt, no matter how many retries run.
        counter = [0]

        def always_changing_fingerprint():
            counter[0] += 1
            return (counter[0],)

        store._file_fingerprint = always_changing_fingerprint

        # Spy on the fallback so we can confirm it actually ran (and exactly
        # once), not just that the final result happens to look right.
        orig_apply_locked = store._apply_locked
        fallback_calls = []

        def spy_apply_locked(*args, **kwargs):
            fallback_calls.append(1)
            return orig_apply_locked(*args, **kwargs)

        store._apply_locked = spy_apply_locked

        stats = store.apply(add_ids=[], add_vectors=None, remove_ids=[2])

        assert fallback_calls == [
            1
        ]  # fell back to the locked rebuild exactly once
        assert stats["removed"] == 1
        assert stats["added"] == 0
        assert set(store.live_ids()) == {1, 3}
        assert store.count() == 2

    def test_rejected_survivor_buffers_are_released_before_retry(
        self, tmp_path, monkeypatch
    ):
        import itertools
        import weakref

        store = _make_store(tmp_path, index_type="hnsw", dimension=4)
        store.apply(add_ids=[1, 2, 3], add_vectors=_vecs(3, dim=4))
        generations = itertools.count()
        monkeypatch.setattr(
            store, "_file_fingerprint", lambda: (next(generations),)
        )
        original = np.empty
        buffers = []

        def allocate(shape, *args, **kwargs):
            if shape == (2, 4):
                assert all(reference() is None for reference in buffers)
            result = original(shape, *args, **kwargs)
            if shape == (2, 4):
                buffers.append(weakref.ref(result))
            return result

        monkeypatch.setattr(np, "empty", allocate)
        store.apply(add_ids=[], add_vectors=None, remove_ids=[2])
        assert len(buffers) == 4  # Three rejected attempts and the fallback.
        assert set(store.live_ids()) == {1, 3}


class TestApplyFailureRecovery:
    """apply()'s failure paths (_restore_from_disk / _reload_under_lock's
    poison-discard branch) must never surface a partial in-memory mutation as
    if it were durable, and must never dangle in a rebuilt-but-unpersisted
    state. Failures here are driven through REAL code paths (a stubbed
    ``persist()`` that genuinely raises) rather than by poking ``_poisoned``
    by hand, so the assertions exercise the actual recovery logic."""

    def test_first_ever_apply_failure_poisons_then_next_apply_discards_stale_mutation(
        self, tmp_path
    ):
        store = _make_store(tmp_path, dimension=4)

        # Simulate a genuine write failure (e.g. disk full) AFTER add() has
        # already mutated the in-memory index but BEFORE anything durable is
        # written -- no .faiss file exists yet at all.
        def failing_persist(_path):
            raise OSError("simulated disk failure on first-ever write")

        store.persist = failing_persist

        with pytest.raises(OSError, match="simulated disk failure"):
            store.apply(add_ids=[1], add_vectors=_vecs(1, dim=4))

        assert store._poisoned is True
        assert not store._path.exists()  # first write never landed

        # Restore real persistence and drive a second, unrelated successful
        # apply(). The never-persisted id 1 must NOT resurrect here.
        store.persist = FaissVectorStore.persist.__get__(store)
        stats = store.apply(add_ids=[2], add_vectors=_vecs(1, dim=4, seed=2))

        assert stats == {"added": 1, "removed": 0}
        assert store.live_ids() == [2]
        assert store._poisoned is False

    def test_restore_from_disk_poisons_store_when_disk_unreadable_after_failed_write(
        self, tmp_path
    ):
        store = _make_store(tmp_path, dimension=4)
        store.apply(
            add_ids=[1], add_vectors=_vecs(1, dim=4)
        )  # real durable file

        # Simulate a torn/corrupting write: the persist call itself corrupts
        # the on-disk bytes before failing, so _restore_from_disk()'s own
        # re-read of disk (not a hand-set flag) is what discovers it can't
        # recover and poisons the store.
        path = store._path

        def corrupt_then_fail(_path):
            path.write_bytes(b"corrupted mid-write garbage")
            raise OSError("simulated write failure mid-persist")

        store.persist = corrupt_then_fail

        with pytest.raises(
            OSError, match="simulated write failure mid-persist"
        ):
            store.apply(add_ids=[2], add_vectors=_vecs(1, dim=4, seed=2))

        assert store._poisoned is True

    def test_hnsw_rebuild_persist_failure_reverts_to_disk_not_dangling_graph(
        self, tmp_path
    ):
        store = _make_store(tmp_path, index_type="hnsw", dimension=4)
        store.apply(add_ids=[1, 2, 3], add_vectors=_vecs(3, dim=4))
        pre_rebuild_ids = set(store.live_ids())

        # Fail persist() ITSELF, right after Phase 3 has already swapped
        # self._index to the rebuilt (post-removal) graph but before the new
        # graph reaches disk -- disk still holds the pre-rebuild file.
        def failing_persist(_path):
            raise OSError("simulated persist failure mid hnsw rebuild")

        store.persist = failing_persist

        with pytest.raises(
            OSError, match="simulated persist failure mid hnsw rebuild"
        ):
            store.apply(add_ids=[], add_vectors=None, remove_ids=[2])

        # Reverted to the durable (pre-rebuild) state, not left dangling on
        # the new, unpersisted, in-memory-only graph.
        assert set(store.live_ids()) == pre_rebuild_ids
        assert store._poisoned is False

    def test_reload_under_lock_file_exists_branch_clears_poison_on_adopt(
        self, tmp_path
    ):
        """REGRESSION for fix B (daa5e2ebb): the file-exists branch of
        ``_reload_under_lock`` must clear ``self._poisoned`` (~line 678)
        after adopting the verified on-disk index -- otherwise a store that
        was poisoned by some earlier, unrelated failure stays permanently
        poisoned even after it has successfully reloaded a known-good file,
        and every subsequent read keeps raising the "poisoned" RuntimeError
        forever, with no way to recover short of constructing a fresh
        instance.

        CRITICAL (tautology trap): this calls ``_reload_under_lock()``
        DIRECTLY rather than going through ``apply()`` -- ``_apply_locked``
        and ``_apply_hnsw`` ALSO clear ``_poisoned`` at their own tail
        (~lines 865/926/980), so asserting on the result of a full apply()
        would pass even with fix B reverted and prove nothing.
        """
        store = _make_store(tmp_path, dimension=4)
        store.apply(add_ids=[1, 2], add_vectors=_vecs(2, dim=4))
        assert store._path.exists()  # a verified, matching-config file

        # Poison the instance directly (the same acceptable-fallback
        # approach the existing read-guard tests use) to isolate the
        # file-exists branch of _reload_under_lock from however the
        # poisoning originally happened.
        store._poisoned = True

        store._reload_under_lock()

        assert store._poisoned is False
        # Reads must actually succeed now (not raise the poisoned
        # RuntimeError) and reflect the adopted on-disk state.
        assert store.count() == 2
        assert set(store.live_ids()) == {1, 2}
        hits = store.search(_vecs(1, dim=4)[0], 2)
        assert {i for i, _ in hits} == {1, 2}


class TestPersistLoadRoundtripSearch:
    """A real persist() -> load() cycle must be invisible to a caller: search
    results (ids, order, scores) must be identical before and after, and for
    HNSW, reconstruct() of every id must match what was actually indexed."""

    @pytest.mark.parametrize("index_type", ["flat", "hnsw"])
    def test_persist_load_roundtrip_preserves_search_ranking(
        self, tmp_path, index_type
    ):
        store = _make_store(
            tmp_path,
            dimension=4,
            index_type=index_type,
            metric="cosine",
            normalize=True,
        )
        ids = [10, 20, 30, 40, 50]
        vecs = _vecs(5, dim=4, seed=7)
        store.add(ids, vecs)
        query = _vecs(1, dim=4, seed=99)[0]
        before = store.search(query, 3)
        assert before  # sanity: got hits before persisting at all

        path = tmp_path / f"rt_{index_type}.faiss"
        store.persist(path)
        loaded = FaissVectorStore.load(
            path,
            dimension=4,
            index_type=index_type,
            metric="cosine",
            normalize=True,
        )
        after = loaded.search(query, 3)

        assert [i for i, _ in after] == [i for i, _ in before]
        for (id_before, score_before), (id_after, score_after) in zip(
            before, after
        ):
            assert id_after == id_before
            assert score_after == pytest.approx(score_before, abs=1e-5)

        if index_type == "hnsw":
            normalized = vecs.copy()
            faiss.normalize_L2(normalized)
            for vid, expected in zip(ids, normalized):
                recon = loaded.reconstruct(vid)
                assert recon is not None
                np.testing.assert_allclose(
                    np.asarray(recon), expected, atol=1e-4
                )


class TestMetricIndexTypeCanonicalization:
    """index_type/metric are canonicalized (strip + lower) so a caller who
    passes mixed case or stray whitespace still gets the SAME physical index
    every construction path builds -- otherwise 'COSINE' would build inner
    product but a later 'cosine' comparison could mismatch it as L2."""

    def test_uppercase_and_whitespace_metric_persist_reload_roundtrip(
        self, tmp_path
    ):
        store = _make_store(
            tmp_path,
            dimension=4,
            index_type=" Flat ",
            metric=" COSINE ",
            normalize=True,
        )
        # Canonicalized immediately at construction time.
        assert store.metric == "cosine"
        assert store.index_type == "flat"

        store.add([1, 2], _vecs(2, dim=4))
        path = tmp_path / "canon.faiss"
        store.persist(path)

        # load() with the canonical spelling must NOT raise: create() and
        # _physical_config_reason() must canonicalize identically.
        loaded = FaissVectorStore.load(
            path,
            dimension=4,
            index_type="flat",
            metric="cosine",
            normalize=True,
        )
        assert loaded.count() == 2
        assert loaded.metric == "cosine"
        assert loaded.index_type == "flat"

        # The physical index really is inner-product -- non-canonical input
        # did not silently fall through to the L2 default.
        base = faiss.downcast_index(loaded._index.index)
        assert base.metric_type == faiss.METRIC_INNER_PRODUCT


class TestPersistTempSweep:
    def test_persist_sweeps_stale_temp_files(self, tmp_path):
        import os
        import time

        store = _make_store(tmp_path)
        store.add([1], _vecs(1))
        path = tmp_path / "out.faiss"
        # A stale temp left by a hard-killed prior write, backdated well past
        # the sweep age threshold so it is recognized as a genuine orphan.
        stale = tmp_path / "out.faiss.deadbeef.tmp"
        stale.write_bytes(b"garbage")
        old = time.time() - 4000
        os.utime(stale, (old, old))
        assert stale.exists()
        store.persist(path)
        assert path.exists()
        assert not stale.exists()  # swept
        assert list(tmp_path.glob("*.tmp")) == []

    def test_persist_leaves_recent_temp_untouched(self, tmp_path):
        # A just-created temp may belong to a concurrent (multi-process) writer
        # for the SAME path whose rename hasn't landed yet — the age gate must
        # not sweep it, or that writer crashes with FileNotFoundError.
        store = _make_store(tmp_path)
        store.add([1], _vecs(1))
        path = tmp_path / "out.faiss"
        recent = tmp_path / "out.faiss.inflight.tmp"
        recent.write_bytes(b"another writer's in-flight bytes")
        store.persist(path)
        assert path.exists()
        assert recent.exists()  # too recent to be treated as an orphan

    def test_persist_leaves_sibling_index_temp_untouched(self, tmp_path):
        store = _make_store(tmp_path)
        store.add([1], _vecs(1))
        path = tmp_path / "out.faiss"
        sibling = tmp_path / "other.faiss.abc.tmp"  # a different index's temp
        sibling.write_bytes(b"x")
        store.persist(path)
        assert sibling.exists()  # scoped to path.name, sibling untouched


class TestHnswOffLockRaceMultiInstance:
    """The off-lock rebuild's staleness check must be PER-FILE, not
    per-instance: production builds a FRESH FaissVectorStore per operation and
    shares only the write lock, so a concurrent writer is a different Python
    object. A per-instance counter would never see it; the on-disk fingerprint
    does."""

    def test_concurrent_write_from_other_instance_not_lost(self, tmp_path):
        import threading

        lock = threading.Lock()
        path = tmp_path / "coll.faiss"
        # Instance A seeds the index.
        a = FaissVectorStore.create(
            dimension=4,
            index_type="hnsw",
            metric="l2",
            normalize=False,
            path=path,
            lock=lock,
            integrity_record=_noop_record,
            integrity_verify=_ok_verify,
        )
        a.apply(add_ids=[1, 2, 3], add_vectors=_vecs(3, dim=4))

        # Instance B — a SEPARATE object loaded from the same file, same lock.
        b = FaissVectorStore.load(
            path,
            dimension=4,
            index_type="hnsw",
            metric="l2",
            normalize=False,
            lock=lock,
            integrity_record=_noop_record,
            integrity_verify=_ok_verify,
        )

        build_started = threading.Event()
        concurrent_done = threading.Event()
        real_build = b._build_base_index

        def slow_build(dimension, index_type, metric):
            build_started.set()
            concurrent_done.wait(timeout=5)
            return real_build(dimension, index_type, metric)

        b._build_base_index = slow_build

        def b_rebuild():
            b.apply(add_ids=[], add_vectors=None, remove_ids=[2])

        t = threading.Thread(target=b_rebuild)
        t.start()
        assert build_started.wait(timeout=5)

        # Instance A (different object) persists a concurrent add of id 9 while
        # B is mid off-lock rebuild. B must detect the file changed underneath
        # (fingerprint), fall back, and preserve id 9.
        a.apply(add_ids=[9], add_vectors=_vecs(1, dim=4, seed=9))
        concurrent_done.set()
        t.join(timeout=6)
        assert not t.is_alive()

        final = FaissVectorStore.load(
            path,
            dimension=4,
            index_type="hnsw",
            metric="l2",
            normalize=False,
        )
        # id 9 (A's write) survived; id 2 (B's removal) is gone. A per-instance
        # counter would have let B clobber A -> {1,3}, losing id 9.
        assert set(final.live_ids()) == {1, 3, 9}


class TestLoadConfigValidation:
    """load() must verify the PHYSICAL index matches the caller-declared
    index_type/metric, not trust the label — a stale/foreign file adopted under
    a mismatched config either crashes .delete() or silently rebuilds the file.
    """

    def _persist(self, tmp_path, *, index_type, metric, normalize):
        store = _make_store(
            tmp_path,
            dimension=4,
            index_type=index_type,
            metric=metric,
            normalize=normalize,
        )
        # apply() persists under the lock, leaving a real .faiss on disk.
        store.apply(add_ids=[1, 2], add_vectors=_vecs(2, dim=4), remove_ids=[])
        return tmp_path / "coll.faiss"

    def _load(self, path, *, index_type, metric, normalize):
        return FaissVectorStore.load(
            path,
            dimension=4,
            index_type=index_type,
            metric=metric,
            normalize=normalize,
            lock=threading.Lock(),
            integrity_record=_noop_record,
            integrity_verify=_ok_verify,
        )

    def test_hnsw_file_loaded_as_flat_raises(self, tmp_path):
        path = self._persist(
            tmp_path, index_type="hnsw", metric="l2", normalize=False
        )
        with pytest.raises(ValueError, match="index_type"):
            self._load(path, index_type="flat", metric="l2", normalize=False)

    def test_flat_file_loaded_as_hnsw_raises(self, tmp_path):
        path = self._persist(
            tmp_path, index_type="flat", metric="cosine", normalize=True
        )
        with pytest.raises(ValueError, match="index_type"):
            self._load(path, index_type="hnsw", metric="cosine", normalize=True)

    def test_metric_family_mismatch_raises(self, tmp_path):
        # cosine builds METRIC_INNER_PRODUCT; loading as l2 (METRIC_L2) mismatches.
        path = self._persist(
            tmp_path, index_type="flat", metric="cosine", normalize=True
        )
        with pytest.raises(ValueError, match="metric"):
            self._load(path, index_type="flat", metric="l2", normalize=False)

    def test_matching_config_loads(self, tmp_path):
        path = self._persist(
            tmp_path, index_type="flat", metric="cosine", normalize=True
        )
        store = self._load(
            path, index_type="flat", metric="cosine", normalize=True
        )
        assert store.count() == 2

    def test_cosine_and_dot_product_are_interchangeable(self, tmp_path):
        # Both map to METRIC_INNER_PRODUCT — same physical index, so loading a
        # cosine file as dot_product must NOT raise (compare metric FAMILY).
        path = self._persist(
            tmp_path, index_type="flat", metric="cosine", normalize=True
        )
        store = self._load(
            path, index_type="flat", metric="dot_product", normalize=True
        )
        assert store.count() == 2

    def test_reload_rejects_wrong_type_file(self, tmp_path):
        """apply()'s reload-under-lock must enforce the SAME physical-type check
        as load(): a wrong-typed file swapped in at the path (a concurrent
        writer / stale leftover) must be refused, not silently adopted — which
        would misroute the delete (HNSW-as-flat raises; flat-as-hnsw rebuilds)."""
        import faiss

        store = _make_store(
            tmp_path,
            dimension=4,
            index_type="flat",
            metric="l2",
            normalize=False,
        )
        store.apply(add_ids=[1], add_vectors=_vecs(1, dim=4), remove_ids=[])

        # Swap the on-disk file for an HNSW IndexIDMap2 of the same dimension.
        hnsw = faiss.IndexIDMap2(faiss.IndexHNSWFlat(4, 32, faiss.METRIC_L2))
        hnsw.add_with_ids(_vecs(2, dim=4), np.array([10, 11], dtype="int64"))
        faiss.write_index(hnsw, str(tmp_path / "coll.faiss"))

        # The next apply() reloads under the lock and must refuse the wrong type.
        with pytest.raises(RuntimeError, match="physical index"):
            store.apply(add_ids=[2], add_vectors=_vecs(1, dim=4), remove_ids=[])