~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Unit tests for ``FaissVectorStore`` (``vector_stores/implementations/faiss_store.py``).
No DB is needed here — this module is DB-agnostic (ids + vectors only, see
the "SECURITY INVARIANT" block in ``vector_stores/base.py``). Persistence
resources (path / lock / integrity hooks) are injected fakes: a ``tmp_path``,
a real ``threading.Lock()``, and no-op integrity callables.
"""
import threading
import faiss
import numpy as np
import pytest
from local_deep_research.vector_stores.implementations.faiss_store import (
FaissVectorStore,
)
# --------------------------------------------------------------------------
# Fakes / helpers
# --------------------------------------------------------------------------
def _noop_record(_path):
return None
def _ok_verify(_path):
return (True, None)
def _make_store(
tmp_path,
*,
dimension=4,
index_type="flat",
metric="cosine",
normalize=True,
bind=True,
integrity_record=_noop_record,
integrity_verify=_ok_verify,
lock=None,
):
kwargs = dict(
dimension=dimension,
index_type=index_type,
metric=metric,
normalize=normalize,
)
if bind:
kwargs.update(
path=tmp_path / "coll.faiss",
lock=lock or threading.Lock(),
integrity_record=integrity_record,
integrity_verify=integrity_verify,
)
return FaissVectorStore.create(**kwargs)
def _vecs(n, dim=4, seed=0):
rng = np.random.default_rng(seed)
return rng.random((n, dim)).astype("float32")
# --------------------------------------------------------------------------
# create / add / count / live_ids
# --------------------------------------------------------------------------
class TestCreateAddCount:
def test_create_empty_store(self, tmp_path):
store = _make_store(tmp_path)
assert store.count() == 0
assert store.live_ids() == []
def test_add_increases_count_and_live_ids(self, tmp_path):
store = _make_store(tmp_path)
store.add([10, 20, 30], _vecs(3))
assert store.count() == 3
assert set(store.live_ids()) == {10, 20, 30}
def test_add_empty_ids_is_noop(self, tmp_path):
store = _make_store(tmp_path)
store.add([], np.empty((0, 4), dtype="float32"))
assert store.count() == 0
def test_add_ids_vectors_length_mismatch_raises(self, tmp_path):
store = _make_store(tmp_path)
with pytest.raises(ValueError, match="length mismatch"):
store.add([1, 2, 3], _vecs(2))
def test_add_dimension_mismatch_raises(self, tmp_path):
"""Guards the python -O silent-corruption case: faiss's own C++
assert on dimension is stripped under -O, so the store must
validate explicitly (see the comment in ``_prepare``)."""
store = _make_store(tmp_path, dimension=4)
with pytest.raises(ValueError, match="vector dimension"):
store.add([1], _vecs(1, dim=8))
def test_int64_ids_beyond_int32(self, tmp_path):
big_id = 5_000_000_000 # > 2**31, would overflow int32
store = _make_store(tmp_path)
store.add([big_id], _vecs(1))
assert store.live_ids() == [big_id]
hits = store.search(_vecs(1)[0], 1)
assert hits[0][0] == big_id
# --------------------------------------------------------------------------
# search: cosine vs l2 ordering
# --------------------------------------------------------------------------
class TestSearchOrdering:
def test_cosine_orders_by_descending_similarity(self, tmp_path):
store = _make_store(
tmp_path, dimension=2, metric="cosine", normalize=True
)
# query points along +x axis. id 1 is near-parallel (high cosine
# sim), id 2 is near-orthogonal (low), id 3 points away (negative).
store.add(
[1, 2, 3],
np.array([[1.0, 0.05], [0.05, 1.0], [-1.0, 0.05]], dtype="float32"),
)
hits = store.search(np.array([1.0, 0.0], dtype="float32"), 3)
ids = [i for i, _ in hits]
assert ids == [1, 2, 3]
# inner product on normalized vectors: higher is more similar,
# so scores must be descending.
scores = [d for _, d in hits]
assert scores == sorted(scores, reverse=True)
def test_l2_orders_by_ascending_distance(self, tmp_path):
store = _make_store(tmp_path, dimension=2, metric="l2", normalize=False)
store.add(
[1, 2, 3],
np.array([[0.1, 0.1], [5.0, 5.0], [1.0, 1.0]], dtype="float32"),
)
hits = store.search(np.array([0.0, 0.0], dtype="float32"), 3)
ids = [i for i, _ in hits]
assert ids == [1, 3, 2] # nearest (0.1,0.1) -> (1,1) -> (5,5)
dists = [d for _, d in hits]
assert dists == sorted(dists)
def test_search_k_le_zero_returns_empty(self, tmp_path):
store = _make_store(tmp_path)
store.add([1], _vecs(1))
assert store.search(_vecs(1)[0], 0) == []
assert store.search(_vecs(1)[0], -1) == []
def test_search_empty_index_returns_empty(self, tmp_path):
store = _make_store(tmp_path)
assert store.search(_vecs(1)[0], 5) == []
def test_search_empty_store_rejects_wrong_dimension_query(self, tmp_path):
store = _make_store(tmp_path, dimension=4)
query = _vecs(1, dim=8)[0]
with pytest.raises(ValueError):
store.search(query, 1)
def test_search_populated_store_rejects_wrong_dimension_query(
self, tmp_path
):
# NOTE: this does NOT discriminate the `_prepare`-before-`ntotal==0`
# reorder in `search()` — a POPULATED index never takes the early
# `return []` under either ordering, so this still passes with that
# reorder fully reverted. The real pin for the reorder is the
# EMPTY-store sibling above
# (test_search_empty_store_rejects_wrong_dimension_query): before
# the reorder, an empty store returned `[]` for a wrong-dimension
# query instead of raising. Keep both — don't delete the empty-store
# test as a "duplicate" of this one.
store = _make_store(tmp_path, dimension=4)
store.add([1], _vecs(1, dim=4))
query = _vecs(1, dim=8)[0]
with pytest.raises(ValueError):
store.search(query, 1)
def test_search_fewer_hits_than_k_drops_empty_slots(self, tmp_path):
"""faiss returns -1 for empty slots when fewer than k neighbors
exist; those must be filtered out, not surfaced as id -1."""
store = _make_store(tmp_path)
store.add([7], _vecs(1))
hits = store.search(_vecs(1)[0], 5)
assert len(hits) == 1
assert hits[0][0] == 7
def test_search_k_less_than_total_returns_nearest_k_in_order(
self, tmp_path
):
# Existing ordering tests use k>=ntotal; this covers k strictly less
# than ntotal, where faiss must still return only the k CLOSEST hits
# (not an arbitrary k-subset) in ranked order.
store = _make_store(tmp_path, dimension=2, metric="l2", normalize=False)
store.add(
[1, 2, 3, 4, 5],
np.array(
[
[0.1, 0.1],
[5.0, 5.0],
[1.0, 1.0],
[2.0, 2.0],
[10.0, 10.0],
],
dtype="float32",
),
)
hits = store.search(np.array([0.0, 0.0], dtype="float32"), 2)
assert len(hits) == 2
ids = [i for i, _ in hits]
# Two nearest to the origin: (0.1,0.1) then (1,1).
assert ids == [1, 3]
dists = [d for _, d in hits]
assert dists == sorted(dists)
# --------------------------------------------------------------------------
# delete / live_ids / count / reconstruct
# --------------------------------------------------------------------------
class TestDeleteLiveIdsReconstruct:
def test_delete_removes_and_returns_count(self, tmp_path):
store = _make_store(tmp_path)
store.add([1, 2, 3], _vecs(3))
removed = store.delete([2, 3])
assert removed == 2
assert store.live_ids() == [1]
assert store.count() == 1
def test_delete_empty_list_is_noop(self, tmp_path):
store = _make_store(tmp_path)
store.add([1], _vecs(1))
assert store.delete([]) == 0
assert store.count() == 1
def test_delete_none_entries_filtered(self, tmp_path):
store = _make_store(tmp_path)
store.add([1], _vecs(1))
assert store.delete([None, None]) == 0
def test_reconstruct_missing_id_returns_none(self, tmp_path):
store = _make_store(tmp_path)
store.add([1], _vecs(1))
assert store.reconstruct(999) is None
def test_reconstruct_existing_id_returns_vector(self, tmp_path):
store = _make_store(tmp_path, dimension=3, metric="l2", normalize=False)
vec = np.array([[1.0, 2.0, 3.0]], dtype="float32")
store.add([5], vec)
out = store.reconstruct(5)
assert out is not None
np.testing.assert_allclose(np.asarray(out), vec[0], atol=1e-5)
def test_reconstruct_after_delete_returns_none(self, tmp_path):
# Distinct from a never-added id: this id WAS live, then removed via
# remove_ids -- the Optional contract _rebuild_dropping/integrity
# re-embed rely on must hold for a removed id too, not just an absent one.
store = _make_store(tmp_path, dimension=3, metric="l2", normalize=False)
store.add([1, 2], _vecs(2, dim=3))
removed = store.delete([1])
assert removed == 1
assert store.reconstruct(1) is None
assert store.reconstruct(999) is None # never-added id, for contrast
assert (
store.reconstruct(2) is not None
) # untouched id still reconstructs
def test_supports_delete_false_for_hnsw(self, tmp_path):
store = _make_store(tmp_path, index_type="hnsw")
assert store.supports_delete is False
def test_hnsw_delete_raises_clear_runtime_error(self, tmp_path):
store = _make_store(tmp_path, index_type="hnsw")
store.add([1], _vecs(1))
with pytest.raises(RuntimeError, match="hnsw"):
store.delete([1])
def test_hnsw_apply_remove_ids_rebuilds(self, tmp_path):
# apply() must NOT raise for HNSW removals: faiss HNSW has no
# remove_ids, so apply() drops-by-rebuild (_rebuild_dropping) instead.
store = _make_store(tmp_path, index_type="hnsw")
store.apply(add_ids=[1, 2, 3], add_vectors=_vecs(3))
stats = store.apply(add_ids=[], add_vectors=None, remove_ids=[2])
assert stats["removed"] == 1
assert set(store.live_ids()) == {1, 3}
assert store.count() == 2
def test_hnsw_apply_replace_same_id_rebuilds(self, tmp_path):
# remove + add of the SAME id (a "replace", used by re-index) must
# succeed for HNSW via the rebuild path — the original crash repro.
store = _make_store(tmp_path, index_type="hnsw")
store.apply(add_ids=[1, 2], add_vectors=_vecs(2))
newvec = _vecs(1, seed=99)
store.apply(add_ids=[1], add_vectors=newvec, remove_ids=[1])
assert set(store.live_ids()) == {1, 2}
def test_hnsw_apply_remove_all_and_absent(self, tmp_path):
store = _make_store(tmp_path, index_type="hnsw")
store.apply(add_ids=[1, 2, 3], add_vectors=_vecs(3))
# Absent id: leniently ignored, nothing removed.
assert (
store.apply(add_ids=[], add_vectors=None, remove_ids=[999])[
"removed"
]
== 0
)
# Remove every id: empties to 0 without crashing.
stats = store.apply(add_ids=[], add_vectors=None, remove_ids=[1, 2, 3])
assert stats["removed"] == 3
assert store.count() == 0
def test_hnsw_rebuild_survives_reload(self, tmp_path):
# The rebuilt index must persist and reload as a valid IndexIDMap2.
store = _make_store(tmp_path, index_type="hnsw")
store.apply(add_ids=[1, 2, 3, 4], add_vectors=_vecs(4))
store.apply(add_ids=[], add_vectors=None, remove_ids=[2, 4])
reloaded = FaissVectorStore.load(
tmp_path / "coll.faiss",
dimension=4,
index_type="hnsw",
metric="cosine",
normalize=True,
)
assert set(reloaded.live_ids()) == {1, 3}
# --------------------------------------------------------------------------
# persist / load
# --------------------------------------------------------------------------
class TestPersistLoad:
def test_persist_is_atomic_no_tmp_left(self, tmp_path):
store = _make_store(tmp_path)
store.add([1, 2], _vecs(2))
path = tmp_path / "out.faiss"
store.persist(path)
assert path.exists()
leftover_tmp = list(tmp_path.glob("*.tmp"))
assert leftover_tmp == []
def test_persist_write_index_failure_leaves_original_file_untouched_and_no_tmp(
self, tmp_path, monkeypatch
):
"""A failure INSIDE the real write (the module-level ``write_index``
imported ~line 37) must never corrupt or replace the last-good
durable file: persist()'s temp+rename means ``write_index`` raising
happens strictly before ``os.replace``, so the original bytes must
be untouched, and the temp file must be cleaned up by persist()'s
own ``finally`` block rather than left dangling as ``*.tmp`` litter.
Existing persist-failure tests mock the whole ``store.persist``
method; this exercises the real mid-write sequence, which those
leave uncovered.
"""
store = _make_store(tmp_path)
store.add([1, 2], _vecs(2))
path = tmp_path / "out.faiss"
store.persist(path)
assert path.exists()
original_bytes = path.read_bytes()
def _boom(index, filename):
raise RuntimeError("simulated write_index failure")
monkeypatch.setattr(
"local_deep_research.vector_stores.implementations."
"faiss_store.write_index",
_boom,
)
# Mutate the in-memory index further so a torn overwrite (if it
# happened) would be detectably different from the captured bytes.
store.add([3], _vecs(1, seed=5))
with pytest.raises(RuntimeError, match="simulated write_index failure"):
store.persist(path)
assert path.read_bytes() == original_bytes # no torn overwrite
assert list(tmp_path.glob("*.tmp")) == [] # temp cleaned up
def test_persist_writes_no_pkl_companion(self, tmp_path):
store = _make_store(tmp_path)
store.add([1], _vecs(1))
path = tmp_path / "out.faiss"
store.persist(path)
assert list(tmp_path.glob("*.pkl")) == []
def test_apply_reload_rejects_dimension_mismatch(self, tmp_path):
# A store bound to a path whose on-disk index has a DIFFERENT dimension
# (embedding model/config changed) must fail closed on apply()'s
# reload-under-lock, not silently add a wrong-width vector (which faiss
# would only catch with an assert stripped under `python -O`).
first = _make_store(tmp_path, dimension=4)
first.apply(add_ids=[1], add_vectors=_vecs(1, dim=4))
mismatched = _make_store(tmp_path, dimension=8) # same default path
with pytest.raises(RuntimeError, match="dimension"):
mismatched.apply(add_ids=[2], add_vectors=_vecs(1, dim=8))
@pytest.mark.skipif(
__import__("sys").platform == "win32",
reason="chmod is not supported on Windows",
)
def test_persist_chmods_parent_private(self, tmp_path):
store = _make_store(tmp_path)
store.add([1], _vecs(1))
d = tmp_path / "sub" / "out.faiss"
store.persist(d)
mode = d.parent.stat().st_mode & 0o777
assert mode == 0o700
def test_load_round_trip(self, tmp_path):
store = _make_store(
tmp_path, dimension=4, metric="cosine", normalize=True
)
store.add([1, 2, 3], _vecs(3))
path = tmp_path / "rt.faiss"
store.persist(path)
loaded = FaissVectorStore.load(
path,
dimension=4,
index_type="flat",
metric="cosine",
normalize=True,
)
assert set(loaded.live_ids()) == {1, 2, 3}
assert loaded.count() == 3
def test_load_raises_on_non_idmap2_file(self, tmp_path):
"""post-migration every on-disk index must be IndexIDMap2; a raw
base index means pre-migration or corruption and must surface."""
path = tmp_path / "raw.faiss"
raw = faiss.IndexFlatL2(4)
faiss.write_index(raw, str(path))
with pytest.raises(ValueError, match="not IndexIDMap2"):
FaissVectorStore.load(
path,
dimension=4,
index_type="flat",
metric="l2",
normalize=False,
)
def test_load_raises_on_dimension_mismatch(self, tmp_path):
store = _make_store(tmp_path, dimension=4)
store.add([1], _vecs(1, dim=4))
path = tmp_path / "dim.faiss"
store.persist(path)
with pytest.raises(ValueError, match="dimension"):
FaissVectorStore.load(
path,
dimension=8,
index_type="flat",
metric="cosine",
normalize=True,
)
# --------------------------------------------------------------------------
# apply(): add / dedup / replace / purge / empty
# --------------------------------------------------------------------------
class TestApply:
def test_apply_requires_persistence_binding(self, tmp_path):
store = _make_store(tmp_path, bind=False)
with pytest.raises(RuntimeError, match="persistence binding"):
store.apply(add_ids=[1], add_vectors=_vecs(1))
def test_apply_add_ids_without_vectors_raises(self, tmp_path):
store = _make_store(tmp_path)
with pytest.raises(ValueError, match="add_vectors"):
store.apply(add_ids=[1], add_vectors=None)
def test_apply_basic_add(self, tmp_path):
store = _make_store(tmp_path)
stats = store.apply(add_ids=[1, 2], add_vectors=_vecs(2))
assert stats == {"added": 2, "removed": 0}
assert set(store.live_ids()) == {1, 2}
def test_apply_dedup_within_batch(self, tmp_path):
store = _make_store(tmp_path)
# id 1 repeated in the same batch -> only the first occurrence kept.
stats = store.apply(add_ids=[1, 1, 2], add_vectors=_vecs(3))
assert stats["added"] == 2
assert set(store.live_ids()) == {1, 2}
def test_apply_replaces_live_id_not_skips(self, tmp_path):
# An id already live is REPLACED (removed then re-added), NOT skipped:
# the new vector must win (reused-id safety — SQLite recycles
# AUTOINCREMENT ids on ROLLBACK, so a live id may carry a stale orphan).
store = _make_store(tmp_path)
store.apply(add_ids=[1], add_vectors=_vecs(1))
stats = store.apply(add_ids=[1, 2], add_vectors=_vecs(2))
assert stats["added"] == 2 # id 1 replaced + id 2 added
assert stats["removed"] == 1 # stale id 1 removed first
assert set(store.live_ids()) == {1, 2}
assert store.count() == 2 # no duplicate id 1
def test_apply_replace_on_collision_is_unconditional(self, tmp_path):
"""Replace-on-collision runs even with dedup=False: a live id is
removed before re-add, so IndexIDMap2 never accrues a duplicate key
(which would cause phantom hits and the reused-id text leak)."""
store = _make_store(tmp_path)
store.apply(add_ids=[1], add_vectors=_vecs(1))
store.apply(add_ids=[1], add_vectors=_vecs(1), dedup=False)
assert store.count() == 1 # replaced, not duplicated
def test_apply_replace_same_id(self, tmp_path):
store = _make_store(tmp_path, dimension=3, metric="l2", normalize=False)
v1 = np.array([[1.0, 0.0, 0.0]], dtype="float32")
store.apply(add_ids=[1], add_vectors=v1)
v2 = np.array([[9.0, 9.0, 9.0]], dtype="float32")
stats = store.apply(add_ids=[1], add_vectors=v2, remove_ids=[1])
assert stats == {"added": 1, "removed": 1}
assert store.live_ids() == [1]
np.testing.assert_allclose(
np.asarray(store.reconstruct(1)), v2[0], atol=1e-5
)
def test_apply_purge(self, tmp_path):
store = _make_store(tmp_path)
store.apply(add_ids=[1, 2, 3], add_vectors=_vecs(3))
stats = store.apply(add_ids=[], add_vectors=None, remove_ids=[1, 2, 3])
assert stats["removed"] == 3
assert store.count() == 0
def test_apply_empty_is_noop_but_persists(self, tmp_path):
store = _make_store(tmp_path)
stats = store.apply(add_ids=[], add_vectors=None, remove_ids=[])
assert stats == {"added": 0, "removed": 0}
assert store._path.exists()
assert store.count() == 0
# --------------------------------------------------------------------------
# Cross-instance reload-under-lock (#4200 guarantee)
# --------------------------------------------------------------------------
class TestCrossInstanceReloadUnderLock:
def test_second_instance_absorbs_first_instances_write(self, tmp_path):
"""Two stores share the same path + lock (as the service layer's
cached-instance-per-collection model does). Instance A adds+persists
first; instance B was constructed independently (its in-memory index
does NOT yet know about A's write). B.apply() must reload the
on-disk file under the lock and merge onto it — losing A's write
here would be exactly the #4200 regression."""
path = tmp_path / "shared.faiss"
lock = threading.Lock()
dim = 4
store_a = FaissVectorStore.create(
dimension=dim,
index_type="flat",
metric="cosine",
normalize=True,
path=path,
lock=lock,
integrity_record=_noop_record,
integrity_verify=_ok_verify,
)
store_b = FaissVectorStore.create(
dimension=dim,
index_type="flat",
metric="cosine",
normalize=True,
path=path,
lock=lock,
integrity_record=_noop_record,
integrity_verify=_ok_verify,
)
store_a.apply(add_ids=[1, 2], add_vectors=_vecs(2, dim=dim))
assert path.exists()
store_b.apply(add_ids=[3], add_vectors=_vecs(1, dim=dim, seed=99))
assert set(store_b.live_ids()) == {1, 2, 3}
reloaded = FaissVectorStore.load(
path,
dimension=dim,
index_type="flat",
metric="cosine",
normalize=True,
)
assert set(reloaded.live_ids()) == {1, 2, 3}
def test_reload_fails_closed_on_bad_integrity(self, tmp_path):
"""If the on-disk file exists but integrity_verify says it can't be
trusted, apply() must refuse to overwrite it (fail closed) rather
than silently clobbering a concurrent writer's committed save."""
path = tmp_path / "shared.faiss"
lock = threading.Lock()
dim = 4
store_a = FaissVectorStore.create(
dimension=dim,
index_type="flat",
metric="cosine",
normalize=True,
path=path,
lock=lock,
integrity_record=_noop_record,
integrity_verify=_ok_verify,
)
store_a.apply(add_ids=[1], add_vectors=_vecs(1, dim=dim))
def _bad_verify(_path):
return (False, "checksum mismatch")
store_b = FaissVectorStore.create(
dimension=dim,
index_type="flat",
metric="cosine",
normalize=True,
path=path,
lock=lock,
integrity_record=_noop_record,
integrity_verify=_bad_verify,
)
with pytest.raises(RuntimeError, match="Refusing to overwrite"):
store_b.apply(add_ids=[2], add_vectors=_vecs(1, dim=dim))
def test_apply_reload_fails_closed_on_unreadable_ondisk_file_and_does_not_clobber(
self, tmp_path
):
"""If the on-disk file exists, integrity_verify says it's fine, but
read_index itself fails (corruption), apply()'s reload-under-lock must
fail closed (raise) rather than silently overwrite the corrupt/foreign
bytes with our in-memory copy -- that would clobber whatever a
concurrent writer had committed there."""
path = tmp_path / "shared.faiss"
lock = threading.Lock()
dim = 4
store_a = FaissVectorStore.create(
dimension=dim,
index_type="flat",
metric="cosine",
normalize=True,
path=path,
lock=lock,
integrity_record=_noop_record,
integrity_verify=_ok_verify, # isolates the read_index branch
)
store_a.apply(add_ids=[1], add_vectors=_vecs(1, dim=dim))
# Corrupt the on-disk bytes so read_index() itself raises.
path.write_bytes(b"not a valid faiss index" * 8)
corrupted_bytes = path.read_bytes()
store_b = FaissVectorStore.create(
dimension=dim,
index_type="flat",
metric="cosine",
normalize=True,
path=path,
lock=lock,
integrity_record=_noop_record,
integrity_verify=_ok_verify,
)
with pytest.raises(RuntimeError, match="failed to reload"):
store_b.apply(add_ids=[2], add_vectors=_vecs(1, dim=dim, seed=2))
# Fail-closed: the corrupt bytes must be untouched, not overwritten.
assert path.read_bytes() == corrupted_bytes
# --------------------------------------------------------------------------
# _read_guard: poisoned instance refuses reads
# --------------------------------------------------------------------------
class TestReadGuardPoisoned:
def test_poisoned_unlocked_instance_refuses_count(self, tmp_path):
store = _make_store(tmp_path, bind=False)
store._poisoned = True
with pytest.raises(RuntimeError, match="poisoned"):
store.count()
def test_poisoned_locked_instance_refuses_search(self, tmp_path):
store = _make_store(tmp_path)
store._poisoned = True
with pytest.raises(RuntimeError, match="poisoned"):
store.search(_vecs(1)[0], 1)
def test_poisoned_refuses_live_ids_and_reconstruct(self, tmp_path):
store = _make_store(tmp_path)
store._poisoned = True
with pytest.raises(RuntimeError, match="poisoned"):
store.live_ids()
with pytest.raises(RuntimeError, match="poisoned"):
store.reconstruct(1)
# --------------------------------------------------------------------------
# integrity_record: retry then raise
# --------------------------------------------------------------------------
class TestIntegrityRecordRetry:
def test_record_integrity_retries_then_raises(self, tmp_path):
calls = []
def _always_fails(_path):
calls.append(1)
raise OSError("db hiccup")
store = _make_store(
tmp_path,
integrity_record=_always_fails,
integrity_verify=_ok_verify,
)
with pytest.raises(RuntimeError, match="failed to record"):
store.apply(add_ids=[1], add_vectors=_vecs(1))
assert len(calls) == 3 # default attempts=3
# The vectors WERE persisted (persist() ran before the integrity
# failure); a resync from disk keeps the in-memory state honest
# rather than poisoning the instance over a checksum-recording bug.
assert store.count() == 1
assert store._poisoned is False
def test_record_integrity_succeeds_on_second_attempt(self, tmp_path):
calls = []
def _fails_once(_path):
calls.append(1)
if len(calls) < 2:
raise OSError("transient")
store = _make_store(
tmp_path, integrity_record=_fails_once, integrity_verify=_ok_verify
)
store.apply(add_ids=[1], add_vectors=_vecs(1))
assert len(calls) == 2
assert store.count() == 1
def test_no_integrity_record_hook_is_noop(self, tmp_path):
store = _make_store(tmp_path, integrity_record=None)
# Must not raise despite no integrity hook.
store.apply(add_ids=[1], add_vectors=_vecs(1))
assert store.count() == 1
class TestNonFiniteVectorRejection:
def test_nan_vector_rejected(self, tmp_path):
store = _make_store(tmp_path)
bad = np.full((1, 4), np.nan, dtype="float32")
with pytest.raises(ValueError, match="non-finite"):
store.add([1], bad)
def test_inf_vector_rejected(self, tmp_path):
store = _make_store(tmp_path)
bad = np.array([[np.inf, 0.0, 0.0, 0.0]], dtype="float32")
with pytest.raises(ValueError, match="non-finite"):
store.add([1], bad)
def test_zero_vector_rejected_under_normalize(self, tmp_path):
# faiss normalize_L2 maps a zero-magnitude vector to all-zeros (not NaN),
# so it would silently be indexed as a degenerate, unsearchable vector.
# _prepare rejects it explicitly instead.
store = _make_store(tmp_path, normalize=True)
with pytest.raises(ValueError, match="zero-magnitude"):
store.add([1], np.zeros((1, 4), dtype="float32"))
assert store.count() == 0
def test_zero_vector_allowed_without_normalize(self, tmp_path):
# Without L2 normalization a zero vector is a valid (if degenerate)
# entry — no NaN, no silent unsearchability from normalize — so it is
# accepted rather than rejected.
store = _make_store(tmp_path, normalize=False)
store.add([1], np.zeros((1, 4), dtype="float32"))
assert store.count() == 1
def test_nan_query_rejected(self, tmp_path):
store = _make_store(tmp_path)
store.add([1], _vecs(1))
with pytest.raises(ValueError, match="non-finite"):
store.search(np.full((1, 4), np.nan, dtype="float32"), 1)
class TestReplaceOnIdCollision:
"""A reused DocumentChunk.id (SQLite recycles AUTOINCREMENT ids on ROLLBACK)
must get the NEW vector, never silently keep a stale orphan vector under
that id — otherwise search rehydrates the WRONG document's text."""
def test_flat_collision_new_vector_wins(self, tmp_path):
store = _make_store(tmp_path, dimension=3, metric="l2", normalize=False)
store.apply(
add_ids=[7],
add_vectors=np.array([[1.0, 0.0, 0.0]], dtype="float32"),
)
new = np.array([[0.0, 1.0, 0.0]], dtype="float32")
store.apply(add_ids=[7], add_vectors=new) # same id, different vector
assert store.count() == 1 # not duplicated
np.testing.assert_allclose(
np.asarray(store.reconstruct(7)), new[0], atol=1e-5
)
def test_hnsw_collision_new_vector_wins(self, tmp_path):
store = _make_store(
tmp_path,
dimension=3,
index_type="hnsw",
metric="l2",
normalize=False,
)
store.apply(
add_ids=[7, 8],
add_vectors=np.array(
[[1.0, 0.0, 0.0], [0.0, 0.0, 1.0]], dtype="float32"
),
)
new = np.array([[0.0, 1.0, 0.0]], dtype="float32")
store.apply(add_ids=[7], add_vectors=new)
assert set(store.live_ids()) == {7, 8}
np.testing.assert_allclose(
np.asarray(store.reconstruct(7)), new[0], atol=1e-5
)
class TestHnswOffLockRebuild:
"""The HNSW rebuild builds the new graph OFF the write lock, so a
concurrent search() is not blocked for the (expensive) O(collection)
rebuild — only for the fast snapshot + the final swap."""
def test_offlock_rebuild_correctness(self, tmp_path):
store = _make_store(
tmp_path,
index_type="hnsw",
dimension=3,
metric="l2",
normalize=False,
)
store.apply(
add_ids=[1, 2, 3],
add_vectors=np.array(
[[1.0, 0, 0], [0, 1.0, 0], [0, 0, 1.0]], dtype="float32"
),
)
# Remove id 2 AND replace id 1 with a new vector in one apply().
newv = np.array([[9.0, 9.0, 9.0]], dtype="float32")
stats = store.apply(add_ids=[1], add_vectors=newv, remove_ids=[2])
assert stats["removed"] >= 1
assert set(store.live_ids()) == {1, 3}
np.testing.assert_allclose(
np.asarray(store.reconstruct(1)), newv[0], atol=1e-5
)
reloaded = FaissVectorStore.load(
tmp_path / "coll.faiss",
dimension=3,
index_type="hnsw",
metric="l2",
normalize=False,
)
assert set(reloaded.live_ids()) == {1, 3}
def test_search_not_blocked_during_rebuild(self, tmp_path):
import threading
import time
store = _make_store(
tmp_path,
index_type="hnsw",
dimension=4,
metric="l2",
normalize=False,
)
store.apply(add_ids=[1, 2, 3, 4], add_vectors=_vecs(4, dim=4))
build_started = threading.Event()
reader_done = threading.Event()
real_build = store._build_base_index # staticmethod -> plain function
def slow_build(dimension, index_type, metric):
# Phase 2 runs OFF the lock; block here until the reader finishes.
build_started.set()
reader_done.wait(timeout=5)
return real_build(dimension, index_type, metric)
store._build_base_index = slow_build # instance-shadow the staticmethod
results = {}
def do_rebuild():
results["r"] = store.apply(
add_ids=[], add_vectors=None, remove_ids=[2]
)
t = threading.Thread(target=do_rebuild)
t.start()
assert build_started.wait(timeout=5), "rebuild never reached Phase 2"
# The off-lock build is blocked. A search must return WITHOUT waiting on
# it (under the old under-lock design this would deadlock on the lock,
# timing out the join below).
t0 = time.monotonic()
hits = store.search(_vecs(1, dim=4)[0], 4)
elapsed = time.monotonic() - t0
assert elapsed < 1.0, f"search blocked {elapsed:.2f}s during rebuild"
# search saw the pre-swap (old) index, which still had id 2.
assert 2 in {i for i, _ in hits}
reader_done.set()
t.join(timeout=5)
assert not t.is_alive()
assert results["r"]["removed"] == 1
assert set(store.live_ids()) == {1, 3, 4}
class TestHnswOffLockRace:
"""A concurrent write during the off-lock rebuild bumps the version; the
rebuild's swap must detect that and fall back to a fully-locked rebuild
rather than clobbering the concurrent write (no lost update)."""
def test_concurrent_add_during_rebuild_not_lost(self, tmp_path):
import threading
store = _make_store(
tmp_path,
index_type="hnsw",
dimension=4,
metric="l2",
normalize=False,
)
store.apply(add_ids=[1, 2, 3], add_vectors=_vecs(3, dim=4))
build_started = threading.Event()
concurrent_done = threading.Event()
real_build = store._build_base_index
def slow_build(dimension, index_type, metric):
build_started.set()
concurrent_done.wait(timeout=5)
return real_build(dimension, index_type, metric)
store._build_base_index = slow_build
def do_rebuild():
store.apply(add_ids=[], add_vectors=None, remove_ids=[2])
t = threading.Thread(target=do_rebuild)
t.start()
assert build_started.wait(timeout=5)
# Concurrent pure-add of id 9 while the rebuild is off-lock building.
# (Pure add takes no rebuild path, so it does not hit slow_build.)
store.apply(add_ids=[9], add_vectors=_vecs(1, dim=4, seed=9))
concurrent_done.set()
t.join(timeout=5)
assert not t.is_alive()
# The rebuild removed 2; the concurrent add of 9 survived -> {1,3,9}.
assert set(store.live_ids()) == {1, 3, 9}
reloaded = FaissVectorStore.load(
tmp_path / "coll.faiss",
dimension=4,
index_type="hnsw",
metric="l2",
normalize=False,
)
assert set(reloaded.live_ids()) == {1, 3, 9}
class TestHnswOffLockRetryExhaustion:
"""The off-lock rebuild retries a bounded number of times on a stale
snapshot, then gives up and falls back to a fully-locked rebuild. Guards
both the >1 retry-count guard (an off-by-one would either give up
prematurely or recurse forever) and the fallback call itself, which
re-acquires ``self._lock`` -- a non-reentrant ``threading.Lock`` -- from
OUTSIDE any held lock, so it must not deadlock."""
def test_hnsw_apply_gives_up_after_retries_and_falls_back_to_locked_rebuild(
self, tmp_path
):
store = _make_store(tmp_path, index_type="hnsw", dimension=4)
store.apply(add_ids=[1, 2, 3], add_vectors=_vecs(3, dim=4))
# Force the staleness check to ALWAYS see a mismatch: an ever-changing
# fingerprint differs between the Phase-1 snapshot and the Phase-3
# recheck on every attempt, no matter how many retries run.
counter = [0]
def always_changing_fingerprint():
counter[0] += 1
return (counter[0],)
store._file_fingerprint = always_changing_fingerprint
# Spy on the fallback so we can confirm it actually ran (and exactly
# once), not just that the final result happens to look right.
orig_apply_locked = store._apply_locked
fallback_calls = []
def spy_apply_locked(*args, **kwargs):
fallback_calls.append(1)
return orig_apply_locked(*args, **kwargs)
store._apply_locked = spy_apply_locked
stats = store.apply(add_ids=[], add_vectors=None, remove_ids=[2])
assert fallback_calls == [
1
] # fell back to the locked rebuild exactly once
assert stats["removed"] == 1
assert stats["added"] == 0
assert set(store.live_ids()) == {1, 3}
assert store.count() == 2
def test_rejected_survivor_buffers_are_released_before_retry(
self, tmp_path, monkeypatch
):
import itertools
import weakref
store = _make_store(tmp_path, index_type="hnsw", dimension=4)
store.apply(add_ids=[1, 2, 3], add_vectors=_vecs(3, dim=4))
generations = itertools.count()
monkeypatch.setattr(
store, "_file_fingerprint", lambda: (next(generations),)
)
original = np.empty
buffers = []
def allocate(shape, *args, **kwargs):
if shape == (2, 4):
assert all(reference() is None for reference in buffers)
result = original(shape, *args, **kwargs)
if shape == (2, 4):
buffers.append(weakref.ref(result))
return result
monkeypatch.setattr(np, "empty", allocate)
store.apply(add_ids=[], add_vectors=None, remove_ids=[2])
assert len(buffers) == 4 # Three rejected attempts and the fallback.
assert set(store.live_ids()) == {1, 3}
class TestApplyFailureRecovery:
"""apply()'s failure paths (_restore_from_disk / _reload_under_lock's
poison-discard branch) must never surface a partial in-memory mutation as
if it were durable, and must never dangle in a rebuilt-but-unpersisted
state. Failures here are driven through REAL code paths (a stubbed
``persist()`` that genuinely raises) rather than by poking ``_poisoned``
by hand, so the assertions exercise the actual recovery logic."""
def test_first_ever_apply_failure_poisons_then_next_apply_discards_stale_mutation(
self, tmp_path
):
store = _make_store(tmp_path, dimension=4)
# Simulate a genuine write failure (e.g. disk full) AFTER add() has
# already mutated the in-memory index but BEFORE anything durable is
# written -- no .faiss file exists yet at all.
def failing_persist(_path):
raise OSError("simulated disk failure on first-ever write")
store.persist = failing_persist
with pytest.raises(OSError, match="simulated disk failure"):
store.apply(add_ids=[1], add_vectors=_vecs(1, dim=4))
assert store._poisoned is True
assert not store._path.exists() # first write never landed
# Restore real persistence and drive a second, unrelated successful
# apply(). The never-persisted id 1 must NOT resurrect here.
store.persist = FaissVectorStore.persist.__get__(store)
stats = store.apply(add_ids=[2], add_vectors=_vecs(1, dim=4, seed=2))
assert stats == {"added": 1, "removed": 0}
assert store.live_ids() == [2]
assert store._poisoned is False
def test_restore_from_disk_poisons_store_when_disk_unreadable_after_failed_write(
self, tmp_path
):
store = _make_store(tmp_path, dimension=4)
store.apply(
add_ids=[1], add_vectors=_vecs(1, dim=4)
) # real durable file
# Simulate a torn/corrupting write: the persist call itself corrupts
# the on-disk bytes before failing, so _restore_from_disk()'s own
# re-read of disk (not a hand-set flag) is what discovers it can't
# recover and poisons the store.
path = store._path
def corrupt_then_fail(_path):
path.write_bytes(b"corrupted mid-write garbage")
raise OSError("simulated write failure mid-persist")
store.persist = corrupt_then_fail
with pytest.raises(
OSError, match="simulated write failure mid-persist"
):
store.apply(add_ids=[2], add_vectors=_vecs(1, dim=4, seed=2))
assert store._poisoned is True
def test_hnsw_rebuild_persist_failure_reverts_to_disk_not_dangling_graph(
self, tmp_path
):
store = _make_store(tmp_path, index_type="hnsw", dimension=4)
store.apply(add_ids=[1, 2, 3], add_vectors=_vecs(3, dim=4))
pre_rebuild_ids = set(store.live_ids())
# Fail persist() ITSELF, right after Phase 3 has already swapped
# self._index to the rebuilt (post-removal) graph but before the new
# graph reaches disk -- disk still holds the pre-rebuild file.
def failing_persist(_path):
raise OSError("simulated persist failure mid hnsw rebuild")
store.persist = failing_persist
with pytest.raises(
OSError, match="simulated persist failure mid hnsw rebuild"
):
store.apply(add_ids=[], add_vectors=None, remove_ids=[2])
# Reverted to the durable (pre-rebuild) state, not left dangling on
# the new, unpersisted, in-memory-only graph.
assert set(store.live_ids()) == pre_rebuild_ids
assert store._poisoned is False
def test_reload_under_lock_file_exists_branch_clears_poison_on_adopt(
self, tmp_path
):
"""REGRESSION for fix B (daa5e2ebb): the file-exists branch of
``_reload_under_lock`` must clear ``self._poisoned`` (~line 678)
after adopting the verified on-disk index -- otherwise a store that
was poisoned by some earlier, unrelated failure stays permanently
poisoned even after it has successfully reloaded a known-good file,
and every subsequent read keeps raising the "poisoned" RuntimeError
forever, with no way to recover short of constructing a fresh
instance.
CRITICAL (tautology trap): this calls ``_reload_under_lock()``
DIRECTLY rather than going through ``apply()`` -- ``_apply_locked``
and ``_apply_hnsw`` ALSO clear ``_poisoned`` at their own tail
(~lines 865/926/980), so asserting on the result of a full apply()
would pass even with fix B reverted and prove nothing.
"""
store = _make_store(tmp_path, dimension=4)
store.apply(add_ids=[1, 2], add_vectors=_vecs(2, dim=4))
assert store._path.exists() # a verified, matching-config file
# Poison the instance directly (the same acceptable-fallback
# approach the existing read-guard tests use) to isolate the
# file-exists branch of _reload_under_lock from however the
# poisoning originally happened.
store._poisoned = True
store._reload_under_lock()
assert store._poisoned is False
# Reads must actually succeed now (not raise the poisoned
# RuntimeError) and reflect the adopted on-disk state.
assert store.count() == 2
assert set(store.live_ids()) == {1, 2}
hits = store.search(_vecs(1, dim=4)[0], 2)
assert {i for i, _ in hits} == {1, 2}
class TestPersistLoadRoundtripSearch:
"""A real persist() -> load() cycle must be invisible to a caller: search
results (ids, order, scores) must be identical before and after, and for
HNSW, reconstruct() of every id must match what was actually indexed."""
@pytest.mark.parametrize("index_type", ["flat", "hnsw"])
def test_persist_load_roundtrip_preserves_search_ranking(
self, tmp_path, index_type
):
store = _make_store(
tmp_path,
dimension=4,
index_type=index_type,
metric="cosine",
normalize=True,
)
ids = [10, 20, 30, 40, 50]
vecs = _vecs(5, dim=4, seed=7)
store.add(ids, vecs)
query = _vecs(1, dim=4, seed=99)[0]
before = store.search(query, 3)
assert before # sanity: got hits before persisting at all
path = tmp_path / f"rt_{index_type}.faiss"
store.persist(path)
loaded = FaissVectorStore.load(
path,
dimension=4,
index_type=index_type,
metric="cosine",
normalize=True,
)
after = loaded.search(query, 3)
assert [i for i, _ in after] == [i for i, _ in before]
for (id_before, score_before), (id_after, score_after) in zip(
before, after
):
assert id_after == id_before
assert score_after == pytest.approx(score_before, abs=1e-5)
if index_type == "hnsw":
normalized = vecs.copy()
faiss.normalize_L2(normalized)
for vid, expected in zip(ids, normalized):
recon = loaded.reconstruct(vid)
assert recon is not None
np.testing.assert_allclose(
np.asarray(recon), expected, atol=1e-4
)
class TestMetricIndexTypeCanonicalization:
"""index_type/metric are canonicalized (strip + lower) so a caller who
passes mixed case or stray whitespace still gets the SAME physical index
every construction path builds -- otherwise 'COSINE' would build inner
product but a later 'cosine' comparison could mismatch it as L2."""
def test_uppercase_and_whitespace_metric_persist_reload_roundtrip(
self, tmp_path
):
store = _make_store(
tmp_path,
dimension=4,
index_type=" Flat ",
metric=" COSINE ",
normalize=True,
)
# Canonicalized immediately at construction time.
assert store.metric == "cosine"
assert store.index_type == "flat"
store.add([1, 2], _vecs(2, dim=4))
path = tmp_path / "canon.faiss"
store.persist(path)
# load() with the canonical spelling must NOT raise: create() and
# _physical_config_reason() must canonicalize identically.
loaded = FaissVectorStore.load(
path,
dimension=4,
index_type="flat",
metric="cosine",
normalize=True,
)
assert loaded.count() == 2
assert loaded.metric == "cosine"
assert loaded.index_type == "flat"
# The physical index really is inner-product -- non-canonical input
# did not silently fall through to the L2 default.
base = faiss.downcast_index(loaded._index.index)
assert base.metric_type == faiss.METRIC_INNER_PRODUCT
class TestPersistTempSweep:
def test_persist_sweeps_stale_temp_files(self, tmp_path):
import os
import time
store = _make_store(tmp_path)
store.add([1], _vecs(1))
path = tmp_path / "out.faiss"
# A stale temp left by a hard-killed prior write, backdated well past
# the sweep age threshold so it is recognized as a genuine orphan.
stale = tmp_path / "out.faiss.deadbeef.tmp"
stale.write_bytes(b"garbage")
old = time.time() - 4000
os.utime(stale, (old, old))
assert stale.exists()
store.persist(path)
assert path.exists()
assert not stale.exists() # swept
assert list(tmp_path.glob("*.tmp")) == []
def test_persist_leaves_recent_temp_untouched(self, tmp_path):
# A just-created temp may belong to a concurrent (multi-process) writer
# for the SAME path whose rename hasn't landed yet — the age gate must
# not sweep it, or that writer crashes with FileNotFoundError.
store = _make_store(tmp_path)
store.add([1], _vecs(1))
path = tmp_path / "out.faiss"
recent = tmp_path / "out.faiss.inflight.tmp"
recent.write_bytes(b"another writer's in-flight bytes")
store.persist(path)
assert path.exists()
assert recent.exists() # too recent to be treated as an orphan
def test_persist_leaves_sibling_index_temp_untouched(self, tmp_path):
store = _make_store(tmp_path)
store.add([1], _vecs(1))
path = tmp_path / "out.faiss"
sibling = tmp_path / "other.faiss.abc.tmp" # a different index's temp
sibling.write_bytes(b"x")
store.persist(path)
assert sibling.exists() # scoped to path.name, sibling untouched
class TestHnswOffLockRaceMultiInstance:
"""The off-lock rebuild's staleness check must be PER-FILE, not
per-instance: production builds a FRESH FaissVectorStore per operation and
shares only the write lock, so a concurrent writer is a different Python
object. A per-instance counter would never see it; the on-disk fingerprint
does."""
def test_concurrent_write_from_other_instance_not_lost(self, tmp_path):
import threading
lock = threading.Lock()
path = tmp_path / "coll.faiss"
# Instance A seeds the index.
a = FaissVectorStore.create(
dimension=4,
index_type="hnsw",
metric="l2",
normalize=False,
path=path,
lock=lock,
integrity_record=_noop_record,
integrity_verify=_ok_verify,
)
a.apply(add_ids=[1, 2, 3], add_vectors=_vecs(3, dim=4))
# Instance B — a SEPARATE object loaded from the same file, same lock.
b = FaissVectorStore.load(
path,
dimension=4,
index_type="hnsw",
metric="l2",
normalize=False,
lock=lock,
integrity_record=_noop_record,
integrity_verify=_ok_verify,
)
build_started = threading.Event()
concurrent_done = threading.Event()
real_build = b._build_base_index
def slow_build(dimension, index_type, metric):
build_started.set()
concurrent_done.wait(timeout=5)
return real_build(dimension, index_type, metric)
b._build_base_index = slow_build
def b_rebuild():
b.apply(add_ids=[], add_vectors=None, remove_ids=[2])
t = threading.Thread(target=b_rebuild)
t.start()
assert build_started.wait(timeout=5)
# Instance A (different object) persists a concurrent add of id 9 while
# B is mid off-lock rebuild. B must detect the file changed underneath
# (fingerprint), fall back, and preserve id 9.
a.apply(add_ids=[9], add_vectors=_vecs(1, dim=4, seed=9))
concurrent_done.set()
t.join(timeout=6)
assert not t.is_alive()
final = FaissVectorStore.load(
path,
dimension=4,
index_type="hnsw",
metric="l2",
normalize=False,
)
# id 9 (A's write) survived; id 2 (B's removal) is gone. A per-instance
# counter would have let B clobber A -> {1,3}, losing id 9.
assert set(final.live_ids()) == {1, 3, 9}
class TestLoadConfigValidation:
"""load() must verify the PHYSICAL index matches the caller-declared
index_type/metric, not trust the label — a stale/foreign file adopted under
a mismatched config either crashes .delete() or silently rebuilds the file.
"""
def _persist(self, tmp_path, *, index_type, metric, normalize):
store = _make_store(
tmp_path,
dimension=4,
index_type=index_type,
metric=metric,
normalize=normalize,
)
# apply() persists under the lock, leaving a real .faiss on disk.
store.apply(add_ids=[1, 2], add_vectors=_vecs(2, dim=4), remove_ids=[])
return tmp_path / "coll.faiss"
def _load(self, path, *, index_type, metric, normalize):
return FaissVectorStore.load(
path,
dimension=4,
index_type=index_type,
metric=metric,
normalize=normalize,
lock=threading.Lock(),
integrity_record=_noop_record,
integrity_verify=_ok_verify,
)
def test_hnsw_file_loaded_as_flat_raises(self, tmp_path):
path = self._persist(
tmp_path, index_type="hnsw", metric="l2", normalize=False
)
with pytest.raises(ValueError, match="index_type"):
self._load(path, index_type="flat", metric="l2", normalize=False)
def test_flat_file_loaded_as_hnsw_raises(self, tmp_path):
path = self._persist(
tmp_path, index_type="flat", metric="cosine", normalize=True
)
with pytest.raises(ValueError, match="index_type"):
self._load(path, index_type="hnsw", metric="cosine", normalize=True)
def test_metric_family_mismatch_raises(self, tmp_path):
# cosine builds METRIC_INNER_PRODUCT; loading as l2 (METRIC_L2) mismatches.
path = self._persist(
tmp_path, index_type="flat", metric="cosine", normalize=True
)
with pytest.raises(ValueError, match="metric"):
self._load(path, index_type="flat", metric="l2", normalize=False)
def test_matching_config_loads(self, tmp_path):
path = self._persist(
tmp_path, index_type="flat", metric="cosine", normalize=True
)
store = self._load(
path, index_type="flat", metric="cosine", normalize=True
)
assert store.count() == 2
def test_cosine_and_dot_product_are_interchangeable(self, tmp_path):
# Both map to METRIC_INNER_PRODUCT — same physical index, so loading a
# cosine file as dot_product must NOT raise (compare metric FAMILY).
path = self._persist(
tmp_path, index_type="flat", metric="cosine", normalize=True
)
store = self._load(
path, index_type="flat", metric="dot_product", normalize=True
)
assert store.count() == 2
def test_reload_rejects_wrong_type_file(self, tmp_path):
"""apply()'s reload-under-lock must enforce the SAME physical-type check
as load(): a wrong-typed file swapped in at the path (a concurrent
writer / stale leftover) must be refused, not silently adopted — which
would misroute the delete (HNSW-as-flat raises; flat-as-hnsw rebuilds)."""
import faiss
store = _make_store(
tmp_path,
dimension=4,
index_type="flat",
metric="l2",
normalize=False,
)
store.apply(add_ids=[1], add_vectors=_vecs(1, dim=4), remove_ids=[])
# Swap the on-disk file for an HNSW IndexIDMap2 of the same dimension.
hnsw = faiss.IndexIDMap2(faiss.IndexHNSWFlat(4, 32, faiss.METRIC_L2))
hnsw.add_with_ids(_vecs(2, dim=4), np.array([10, 11], dtype="int64"))
faiss.write_index(hnsw, str(tmp_path / "coll.faiss"))
# The next apply() reloads under the lock and must refuse the wrong type.
with pytest.raises(RuntimeError, match="physical index"):
store.apply(add_ids=[2], add_vectors=_vecs(1, dim=4), remove_ids=[])