~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Legacy conversion must never publish a generation later reads reject.
Regression pins for the documented pre-fix legacy population (see the #4200
comment in ``faiss_store.py``): pre-fix writers can leave duplicate ids — a
stale orphan vector under an id a later, unrelated chunk reuses — in a legacy
``.faiss`` id map, and such files must convert on their next write without
re-embedding. Conversion normalizes the id map with the same discipline as the
add path (drop duplicates deterministically, skip unrepresentable values,
fail loudly and non-destructively when no safe conversion exists), and settle
must never destroy the legacy pre-image before the replacement generation has
validated.
Only the legacy reader is faked (an id map of raw, unchecked ids exactly as a
pre-fix writer could persist them); every write, publication, validation and
settle step runs the real machinery.
"""
import json
import threading
import numpy as np
import pytest
from local_deep_research.vector_stores.base import VectorStoreIntegrityError
from local_deep_research.vector_stores.implementations import (
segmented_faiss as module,
)
from local_deep_research.vector_stores.segment_files import checkpoint_path
@pytest.fixture
def binding(tmp_path):
checksums = {}
def record(path):
checksums[str(path)] = path.read_bytes()
def verify(path):
return (
checksums.get(str(path)) == path.read_bytes(),
"checksum",
)
return dict(
path=tmp_path / "index.faiss",
dimension=8,
index_type="hnsw",
metric="l2",
normalize=False,
lock=threading.RLock(),
integrity_record=record,
integrity_verify=verify,
)
def vectors(n, seed=5):
return np.random.default_rng(seed).random((n, 8), dtype=np.float32)
def write_legacy(binding):
"""Persist legacy bytes and adopt them into the integrity record, as the
service's preflight does for an existing legacy file."""
binding["path"].write_bytes(b"LEGACY")
binding["integrity_record"](binding["path"])
class FakeLegacy:
"""Stand-in for the mmap'd legacy reader: an id map carrying raw ids with
no bounds or uniqueness checks, as a pre-fix writer could persist."""
def __init__(self, ids, dimension=8):
self.id_map = list(ids)
self.d = dimension
self._vectors = {
int(value): np.full(dimension, float(value), dtype="float32")
for value in ids
}
def reconstruct(self, i):
return self._vectors[int(i)]
@property
def ntotal(self):
return len(self.id_map)
@pytest.fixture
def legacy(monkeypatch):
def install(ids):
fake = FakeLegacy(ids)
monkeypatch.setattr(
module.faiss,
"vector_to_array",
lambda values: np.asarray(values, dtype="int64"),
)
monkeypatch.setattr(
module.SegmentedFaissStore, "_legacy", lambda self: fake
)
monkeypatch.setattr(module, "SEGMENT_VECTORS", 4)
return fake
return install
def test_conversion_deduplicates_reused_ids(binding, legacy):
"""A pre-#4200 stale orphan (duplicate ids, within and across segment
windows) must convert to one row per id instead of producing a generation
that _ids()/count() reject — or that serves the same chunk twice."""
fake = legacy([7, 5, 5, 1001, 1002, 1003, 7])
write_legacy(binding)
store = module.SegmentedFaissStore.load(**binding)
result = store.apply(add_ids=[], add_vectors=None, remove_ids=[999])
# Deduplication is a repair, not a removal: only explicitly removed ids
# count as removed.
assert result == {"added": 0, "removed": 0}
fresh = module.SegmentedFaissStore.load(**binding)
assert sorted(fresh.live_ids()) == [5, 7, 1001, 1002, 1003]
assert fresh.count() == 5
hits = fresh.search(np.ones(8, dtype="float32"), 10)
assert [hit[0] for hit in hits].count(7) == 1
np.testing.assert_allclose(fresh.reconstruct(7), fake.reconstruct(7))
def test_conversion_repairs_unrepresentable_ids(binding, legacy):
"""A negative legacy id (unrepresentable in a generated segment) is
skipped with the rest of the file surviving — such files must keep working
after their next write instead of being bricked."""
legacy([-3, 5, 8])
write_legacy(binding)
store = module.SegmentedFaissStore.load(**binding)
result = store.apply(add_ids=[1], add_vectors=vectors(1))
assert result == {"added": 1, "removed": 0}
fresh = module.SegmentedFaissStore.load(**binding)
assert sorted(fresh.live_ids()) == [1, 5, 8]
assert fresh.count() == 3
assert fresh.reconstruct(-3) is None
def test_conversion_fails_loudly_and_keeps_the_legacy_pre_image(
binding, legacy
):
"""A file whose rows cannot be represented at all is refused loudly and
non-destructively: no empty generation is published, the legacy bytes stay
readable, and the refusal surfaces as the integrity error whose established
recovery policy is refuse + quarantine + reindex (never a silent commit, a
destroyed pre-image, or a forced re-embed of healthy data)."""
fake = legacy([-3, -7])
write_legacy(binding)
original = binding["path"].read_bytes()
store = module.SegmentedFaissStore.load(**binding)
with pytest.raises(
VectorStoreIntegrityError, match="cannot be represented"
):
store.apply(add_ids=[1], add_vectors=vectors(1))
# Non-destructive: the legacy pre-image is byte-identical and still loads.
assert binding["path"].read_bytes() == original
fresh = module.SegmentedFaissStore.load(**binding)
assert fresh.count() == fake.ntotal # still the legacy file, intact
def test_publish_refuses_an_unloadable_generation(binding, monkeypatch):
"""_publish validates before writing and authenticating: an entry that
later reads reject can never become the published generation."""
store = module.SegmentedFaissStore.create(**binding)
store.apply(add_ids=[1], add_vectors=vectors(1))
before = binding["path"].read_bytes()
original = store._write
def corrupt_entry(index):
entry = original(index)
entry["min_id"] = -3 # a legacy artifact reaching publication
return entry
monkeypatch.setattr(store, "_write", corrupt_entry)
with pytest.raises(ValueError):
store.apply(add_ids=[2], add_vectors=vectors(1, 21))
# Nothing unloadable was published: the previous generation is intact.
assert binding["path"].read_bytes() == before
assert module.SegmentedFaissStore.load(**binding).count() == 1
def test_settle_restores_the_pre_image_when_the_generation_is_unloadable(
binding,
):
"""Settle validates the replacement generation BEFORE the checkpoint — the
only copy of the pre-image — is destroyed, and an unloadable generation is
surfaced and rolled back rather than reported as success."""
# Structural pin only: the checksum record is not under test here.
binding["integrity_verify"] = lambda path: (True, None)
store = module.SegmentedFaissStore.create(**binding)
store.apply(add_ids=[1], add_vectors=vectors(1))
generation_zero = binding["path"].read_bytes()
store.begin_transaction(first=True)
store.apply(add_ids=[2], add_vectors=vectors(1, 21))
# Corrupt the published generation between publish and settle — exactly
# the window the checkpoint ordering protects.
manifest = json.loads(binding["path"].read_bytes()[len(module.MAGIC) :])
manifest["segments"][0]["min_id"] = -3
binding["path"].write_bytes(
module.MAGIC + json.dumps(manifest, separators=(",", ":")).encode()
)
with pytest.raises(VectorStoreIntegrityError):
store.end_transaction(committed=True)
# The rejected publish left the pre-image intact and unreplaced at the
# path AND kept the rollback checkpoint holding the same bytes: neither
# copy is destroyed by a settle that refuses the generation.
assert binding["path"].read_bytes() == generation_zero
checkpoint = checkpoint_path(binding["path"])
assert checkpoint.exists() and not checkpoint.is_symlink()
assert checkpoint.read_bytes() == generation_zero
assert module.SegmentedFaissStore.load(**binding).count() == 1
def test_rejected_publish_leaves_legacy_pre_image_and_checkpoint_intact(
binding, legacy
):
"""A settle that rejects the published generation must leave BOTH copies
of the legacy pre-image intact and unreplaced — the rollback checkpoint
and the restored bytes at the path — exactly the B1 data-loss chain
(invalid manifest committed, legacy bytes gone, checkpoint gone) run in
reverse: nothing is destroyed, and the failure is surfaced."""
binding["integrity_verify"] = lambda path: (True, None)
legacy([4, 5])
write_legacy(binding)
original = binding["path"].read_bytes()
store = module.SegmentedFaissStore.load(**binding)
store.begin_transaction(first=True)
store.apply(add_ids=[1], add_vectors=vectors(1))
# Tamper the published generation between publish and settle — the window
# the checkpoint ordering protects.
manifest = json.loads(binding["path"].read_bytes()[len(module.MAGIC) :])
manifest["segments"][0]["min_id"] = -3
binding["path"].write_bytes(
module.MAGIC + json.dumps(manifest, separators=(",", ":")).encode()
)
with pytest.raises(VectorStoreIntegrityError):
store.end_transaction(committed=True)
checkpoint = checkpoint_path(binding["path"])
# The legacy bytes are intact and unreplaced at the path, and the rollback
# checkpoint is left in place holding the same pre-image.
assert binding["path"].read_bytes() == original
assert checkpoint.exists() and not checkpoint.is_symlink()
assert checkpoint.read_bytes() == original
# The store still serves reads from the preserved pre-image.
fresh = module.SegmentedFaissStore.load(**binding)
assert fresh.count() == 2
assert sorted(fresh.live_ids()) == [4, 5]