Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

9.9 KB
Raw Blame
"""Legacy conversion must never publish a generation later reads reject.

Regression pins for the documented pre-fix legacy population (see the #4200
comment in ``faiss_store.py``): pre-fix writers can leave duplicate ids — a
stale orphan vector under an id a later, unrelated chunk reuses — in a legacy
``.faiss`` id map, and such files must convert on their next write without
re-embedding. Conversion normalizes the id map with the same discipline as the
add path (drop duplicates deterministically, skip unrepresentable values,
fail loudly and non-destructively when no safe conversion exists), and settle
must never destroy the legacy pre-image before the replacement generation has
validated.

Only the legacy reader is faked (an id map of raw, unchecked ids exactly as a
pre-fix writer could persist them); every write, publication, validation and
settle step runs the real machinery.
"""

import json
import threading

import numpy as np
import pytest

from local_deep_research.vector_stores.base import VectorStoreIntegrityError
from local_deep_research.vector_stores.implementations import (
    segmented_faiss as module,
)
from local_deep_research.vector_stores.segment_files import checkpoint_path


@pytest.fixture
def binding(tmp_path):
    checksums = {}

    def record(path):
        checksums[str(path)] = path.read_bytes()

    def verify(path):
        return (
            checksums.get(str(path)) == path.read_bytes(),
            "checksum",
        )

    return dict(
        path=tmp_path / "index.faiss",
        dimension=8,
        index_type="hnsw",
        metric="l2",
        normalize=False,
        lock=threading.RLock(),
        integrity_record=record,
        integrity_verify=verify,
    )


def vectors(n, seed=5):
    return np.random.default_rng(seed).random((n, 8), dtype=np.float32)


def write_legacy(binding):
    """Persist legacy bytes and adopt them into the integrity record, as the
    service's preflight does for an existing legacy file."""
    binding["path"].write_bytes(b"LEGACY")
    binding["integrity_record"](binding["path"])


class FakeLegacy:
    """Stand-in for the mmap'd legacy reader: an id map carrying raw ids with
    no bounds or uniqueness checks, as a pre-fix writer could persist."""

    def __init__(self, ids, dimension=8):
        self.id_map = list(ids)
        self.d = dimension
        self._vectors = {
            int(value): np.full(dimension, float(value), dtype="float32")
            for value in ids
        }

    def reconstruct(self, i):
        return self._vectors[int(i)]

    @property
    def ntotal(self):
        return len(self.id_map)


@pytest.fixture
def legacy(monkeypatch):
    def install(ids):
        fake = FakeLegacy(ids)
        monkeypatch.setattr(
            module.faiss,
            "vector_to_array",
            lambda values: np.asarray(values, dtype="int64"),
        )
        monkeypatch.setattr(
            module.SegmentedFaissStore, "_legacy", lambda self: fake
        )
        monkeypatch.setattr(module, "SEGMENT_VECTORS", 4)
        return fake

    return install


def test_conversion_deduplicates_reused_ids(binding, legacy):
    """A pre-#4200 stale orphan (duplicate ids, within and across segment
    windows) must convert to one row per id instead of producing a generation
    that _ids()/count() reject — or that serves the same chunk twice."""
    fake = legacy([7, 5, 5, 1001, 1002, 1003, 7])
    write_legacy(binding)
    store = module.SegmentedFaissStore.load(**binding)

    result = store.apply(add_ids=[], add_vectors=None, remove_ids=[999])

    # Deduplication is a repair, not a removal: only explicitly removed ids
    # count as removed.
    assert result == {"added": 0, "removed": 0}
    fresh = module.SegmentedFaissStore.load(**binding)
    assert sorted(fresh.live_ids()) == [5, 7, 1001, 1002, 1003]
    assert fresh.count() == 5
    hits = fresh.search(np.ones(8, dtype="float32"), 10)
    assert [hit[0] for hit in hits].count(7) == 1
    np.testing.assert_allclose(fresh.reconstruct(7), fake.reconstruct(7))


def test_conversion_repairs_unrepresentable_ids(binding, legacy):
    """A negative legacy id (unrepresentable in a generated segment) is
    skipped with the rest of the file surviving — such files must keep working
    after their next write instead of being bricked."""
    legacy([-3, 5, 8])
    write_legacy(binding)
    store = module.SegmentedFaissStore.load(**binding)

    result = store.apply(add_ids=[1], add_vectors=vectors(1))

    assert result == {"added": 1, "removed": 0}
    fresh = module.SegmentedFaissStore.load(**binding)
    assert sorted(fresh.live_ids()) == [1, 5, 8]
    assert fresh.count() == 3
    assert fresh.reconstruct(-3) is None


def test_conversion_fails_loudly_and_keeps_the_legacy_pre_image(
    binding, legacy
):
    """A file whose rows cannot be represented at all is refused loudly and
    non-destructively: no empty generation is published, the legacy bytes stay
    readable, and the refusal surfaces as the integrity error whose established
    recovery policy is refuse + quarantine + reindex (never a silent commit, a
    destroyed pre-image, or a forced re-embed of healthy data)."""
    fake = legacy([-3, -7])
    write_legacy(binding)
    original = binding["path"].read_bytes()
    store = module.SegmentedFaissStore.load(**binding)

    with pytest.raises(
        VectorStoreIntegrityError, match="cannot be represented"
    ):
        store.apply(add_ids=[1], add_vectors=vectors(1))

    # Non-destructive: the legacy pre-image is byte-identical and still loads.
    assert binding["path"].read_bytes() == original
    fresh = module.SegmentedFaissStore.load(**binding)
    assert fresh.count() == fake.ntotal  # still the legacy file, intact


def test_publish_refuses_an_unloadable_generation(binding, monkeypatch):
    """_publish validates before writing and authenticating: an entry that
    later reads reject can never become the published generation."""
    store = module.SegmentedFaissStore.create(**binding)
    store.apply(add_ids=[1], add_vectors=vectors(1))
    before = binding["path"].read_bytes()
    original = store._write

    def corrupt_entry(index):
        entry = original(index)
        entry["min_id"] = -3  # a legacy artifact reaching publication
        return entry

    monkeypatch.setattr(store, "_write", corrupt_entry)
    with pytest.raises(ValueError):
        store.apply(add_ids=[2], add_vectors=vectors(1, 21))

    # Nothing unloadable was published: the previous generation is intact.
    assert binding["path"].read_bytes() == before
    assert module.SegmentedFaissStore.load(**binding).count() == 1


def test_settle_restores_the_pre_image_when_the_generation_is_unloadable(
    binding,
):
    """Settle validates the replacement generation BEFORE the checkpoint — the
    only copy of the pre-image — is destroyed, and an unloadable generation is
    surfaced and rolled back rather than reported as success."""
    # Structural pin only: the checksum record is not under test here.
    binding["integrity_verify"] = lambda path: (True, None)
    store = module.SegmentedFaissStore.create(**binding)
    store.apply(add_ids=[1], add_vectors=vectors(1))
    generation_zero = binding["path"].read_bytes()
    store.begin_transaction(first=True)
    store.apply(add_ids=[2], add_vectors=vectors(1, 21))
    # Corrupt the published generation between publish and settle — exactly
    # the window the checkpoint ordering protects.
    manifest = json.loads(binding["path"].read_bytes()[len(module.MAGIC) :])
    manifest["segments"][0]["min_id"] = -3
    binding["path"].write_bytes(
        module.MAGIC + json.dumps(manifest, separators=(",", ":")).encode()
    )

    with pytest.raises(VectorStoreIntegrityError):
        store.end_transaction(committed=True)

    # The rejected publish left the pre-image intact and unreplaced at the
    # path AND kept the rollback checkpoint holding the same bytes: neither
    # copy is destroyed by a settle that refuses the generation.
    assert binding["path"].read_bytes() == generation_zero
    checkpoint = checkpoint_path(binding["path"])
    assert checkpoint.exists() and not checkpoint.is_symlink()
    assert checkpoint.read_bytes() == generation_zero
    assert module.SegmentedFaissStore.load(**binding).count() == 1


def test_rejected_publish_leaves_legacy_pre_image_and_checkpoint_intact(
    binding, legacy
):
    """A settle that rejects the published generation must leave BOTH copies
    of the legacy pre-image intact and unreplaced — the rollback checkpoint
    and the restored bytes at the path — exactly the B1 data-loss chain
    (invalid manifest committed, legacy bytes gone, checkpoint gone) run in
    reverse: nothing is destroyed, and the failure is surfaced."""
    binding["integrity_verify"] = lambda path: (True, None)
    legacy([4, 5])
    write_legacy(binding)
    original = binding["path"].read_bytes()
    store = module.SegmentedFaissStore.load(**binding)
    store.begin_transaction(first=True)
    store.apply(add_ids=[1], add_vectors=vectors(1))
    # Tamper the published generation between publish and settle — the window
    # the checkpoint ordering protects.
    manifest = json.loads(binding["path"].read_bytes()[len(module.MAGIC) :])
    manifest["segments"][0]["min_id"] = -3
    binding["path"].write_bytes(
        module.MAGIC + json.dumps(manifest, separators=(",", ":")).encode()
    )

    with pytest.raises(VectorStoreIntegrityError):
        store.end_transaction(committed=True)

    checkpoint = checkpoint_path(binding["path"])
    # The legacy bytes are intact and unreplaced at the path, and the rollback
    # checkpoint is left in place holding the same pre-image.
    assert binding["path"].read_bytes() == original
    assert checkpoint.exists() and not checkpoint.is_symlink()
    assert checkpoint.read_bytes() == original
    # The store still serves reads from the preserved pre-image.
    fresh = module.SegmentedFaissStore.load(**binding)
    assert fresh.count() == 2
    assert sorted(fresh.live_ids()) == [4, 5]