Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

22.0 KB
Raw Blame
"""Real native FAISS persistence, bounded mutation, and recovery contracts."""

import hashlib
import json
import os
from pathlib import Path
import threading
import time

import numpy as np
import pytest

from local_deep_research.vector_stores.implementations import (
    segmented_faiss as module,
)
from local_deep_research.vector_stores.implementations.faiss_store import (
    FaissVectorStore,
)


@pytest.fixture
def binding(tmp_path):
    checksums = {}

    def record(path):
        checksums[str(path)] = hashlib.sha256(path.read_bytes()).digest()

    def verify(path):
        return (
            checksums.get(str(path))
            == hashlib.sha256(path.read_bytes()).digest(),
            "checksum",
        )

    return dict(
        path=tmp_path / "index.faiss",
        dimension=8,
        index_type="hnsw",
        metric="l2",
        normalize=False,
        lock=threading.RLock(),
        integrity_record=record,
        integrity_verify=verify,
    )


def vectors(n, seed=5):
    return np.random.default_rng(seed).random((n, 8), dtype=np.float32)


def state(path):
    return json.loads(path.read_bytes()[len(module.MAGIC) :])["segments"]


def test_append_reuses_full_segments_and_searches_all(binding, monkeypatch):
    monkeypatch.setattr(module, "SEGMENT_VECTORS", 4)
    store = module.SegmentedFaissStore.create(**binding)
    original = vectors(12)
    store.apply(add_ids=list(range(1, 13)), add_vectors=original)
    before = state(binding["path"])
    store.apply(add_ids=[13], add_vectors=vectors(1, 13))
    after = state(binding["path"])
    assert after[:3] == before
    assert all(entry["count"] <= 4 for entry in after)
    fresh = module.SegmentedFaissStore.load(**binding)
    assert fresh.count() == 13
    for position in (0, 4, 11):
        assert fresh.search(original[position], 1)[0][0] == position + 1
    assert fresh.search(vectors(1, 13)[0], 1)[0][0] == 13


@pytest.mark.parametrize("index_type", ["flat", "hnsw"])
def test_replace_delete_and_legacy_conversion(binding, monkeypatch, index_type):
    monkeypatch.setattr(module, "SEGMENT_VECTORS", 4)
    binding["index_type"] = index_type
    old = FaissVectorStore.create(**binding)
    original = vectors(11)
    old.apply(add_ids=list(range(1, 12)), add_vectors=original)
    from local_deep_research.vector_stores.legacy_rekey import _is_idmap2

    native_read = module.faiss.read_index

    def mapped_read(path, flags=0):
        assert flags & module.faiss.IO_FLAG_MMAP_IFC
        assert flags & module.faiss.IO_FLAG_READ_ONLY
        return native_read(path, flags)

    with monkeypatch.context() as check:
        check.setattr(module.faiss, "read_index", mapped_read)
        assert _is_idmap2(binding["path"]) is True
    store = module.SegmentedFaissStore.load(**binding)
    assert store.count() == 11
    assert store.search(original[6], 1)[0][0] == 7
    replacement = vectors(1, 41)
    result = store.apply(
        add_ids=[7], add_vectors=replacement, remove_ids=[1, 4]
    )
    assert result == {"added": 1, "removed": 3}
    fresh = module.SegmentedFaissStore.load(**binding)
    assert set(fresh.live_ids()) == set(range(1, 12)) - {1, 4}
    np.testing.assert_allclose(fresh.reconstruct(7), replacement[0])
    np.testing.assert_allclose(fresh.reconstruct(10), original[9])
    assert fresh.reconstruct(1) is None


def test_failure_before_publication_preserves_old_generation(
    binding, monkeypatch
):
    monkeypatch.setattr(module, "SEGMENT_VECTORS", 4)
    store = module.SegmentedFaissStore.create(**binding)
    store.apply(add_ids=list(range(1, 10)), add_vectors=vectors(9))
    before = binding["path"].read_bytes()
    original = store._write
    calls = 0

    def fail_after_one_segment(index):
        nonlocal calls
        calls += 1
        if calls == 2:
            raise OSError("simulated disk failure")
        return original(index)

    monkeypatch.setattr(store, "_write", fail_after_one_segment)
    with pytest.raises(OSError, match="disk failure"):
        store.apply(add_ids=[10, 11, 12, 13, 14], add_vectors=vectors(5))
    assert binding["path"].read_bytes() == before
    assert set(module.SegmentedFaissStore.load(**binding).live_ids()) == set(
        range(1, 10)
    )


@pytest.mark.parametrize("suffix", [".faiss", ".ids"])
def test_segment_tampering_is_refused(binding, suffix):
    store = module.SegmentedFaissStore.create(**binding)
    store.apply(add_ids=[1], add_vectors=vectors(1))
    entry = state(binding["path"])[0]
    path = (
        module.segment_directory(binding["path"]) / entry["file"]
    ).with_suffix(suffix)
    path.write_bytes(path.read_bytes() + b"tamper")
    with pytest.raises(ValueError, match="checksum"):
        store.search(vectors(1)[0], 1)


def test_fresh_instances_merge_writes_and_purge_once(binding, monkeypatch):
    monkeypatch.setattr(module, "SEGMENT_VECTORS", 4)
    a = module.SegmentedFaissStore.create(**binding)
    a.apply(add_ids=[1, 2, 3, 4], add_vectors=vectors(4))
    b = module.SegmentedFaissStore.load(**binding)
    a.apply(add_ids=[5, 6, 7, 8], add_vectors=vectors(4, 8))
    b.apply(add_ids=[9], add_vectors=vectors(1, 9), remove_ids=[1, 6])
    assert set(a.live_ids()) == {2, 3, 4, 5, 7, 8, 9}
    retained = {entry["file"] for entry in state(binding["path"])}
    assert {
        path.name
        for path in module.segment_directory(binding["path"]).glob("*.faiss")
    } == retained


@pytest.mark.parametrize("initial", [False, True])
def test_integrity_record_failure_restores_previous_root(binding, initial):
    # A plain Lock also proves checkpoint coordination never re-enters it.
    binding["lock"] = threading.Lock()
    store = module.SegmentedFaissStore.create(**binding)
    if initial:
        store.apply(add_ids=[1, 2], add_vectors=vectors(2))
    before = binding["path"].read_bytes() if initial else None

    def fail_record(_path):
        raise OSError("checksum database write failed")

    store._record = fail_record
    with pytest.raises(OSError, match="checksum database"):
        store.apply(add_ids=[3], add_vectors=vectors(1, 3))
    if initial:
        assert binding["path"].read_bytes() == before
        assert module.SegmentedFaissStore.load(**binding).live_ids() == [1, 2]
    else:
        assert not binding["path"].exists()
    assert not module.checkpoint_path(binding["path"]).exists()
    retained = (
        {entry["file"] for entry in state(binding["path"])}
        if initial
        else set()
    )
    assert {
        path.name
        for path in module.segment_directory(binding["path"]).glob("*.faiss")
    } == retained


@pytest.mark.parametrize(
    "manifest",
    [[], {}, {"segments": None}, {"segments": [{}]}, {"segments": ["bad"]}],
)
def test_malformed_manifest_raises_value_error(binding, manifest):
    if isinstance(manifest, dict):
        manifest.update(
            dimension=8, index_type="hnsw", metric="l2", normalize=False
        )
    binding["path"].write_bytes(module.MAGIC + json.dumps(manifest).encode())
    binding["integrity_record"](binding["path"])
    with pytest.raises(ValueError):
        module.SegmentedFaissStore.load(**binding)


def test_empty_store_validates_query_vector(binding):
    store = module.SegmentedFaissStore.create(**binding)
    with pytest.raises(ValueError, match="dimension"):
        store.search(np.zeros(9, dtype="float32"), 1)
    with pytest.raises(ValueError, match="finite"):
        store.search(np.full(8, np.nan, dtype="float32"), 1)


def test_next_write_retires_interrupted_segment_files_only(binding):
    store = module.SegmentedFaissStore.create(**binding)
    store.apply(add_ids=[1], add_vectors=vectors(1))
    directory = module.segment_directory(binding["path"])
    abandoned = [
        directory / ("0" * 32 + suffix)
        for suffix in (".faiss", ".ids", ".faiss.random.tmp", ".ids.random.tmp")
    ]
    for path in abandoned:
        path.write_bytes(b"interrupted vector write")
    unexpected = directory / "unrelated.txt"
    unexpected.write_bytes(b"keep")
    store.apply(add_ids=[2], add_vectors=vectors(1, 2))
    assert store.live_ids() == [1, 2]
    assert all(not path.exists() for path in abandoned)
    assert unexpected.read_bytes() == b"keep"


@pytest.mark.parametrize("operation", ["begin", "settle"])
def test_family_cleanup_retires_abandoned_checkpoint_copies(
    binding, monkeypatch, operation
):
    from local_deep_research.research_library.deletion.utils.cascade_helper import (
        CascadeHelper,
    )

    store = module.SegmentedFaissStore.create(**binding)
    store.apply(add_ids=[1], add_vectors=vectors(1))
    if operation == "settle":
        store.begin_transaction()
        store.apply(add_ids=[2], add_vectors=vectors(1, 2))

    copies = []
    real_mkstemp = module.tempfile.mkstemp
    real_replace = os.replace
    real_unlink = Path.unlink

    def mkstemp(*args, **kwargs):
        descriptor, temporary = real_mkstemp(*args, **kwargs)
        copies.append(Path(temporary))
        return descriptor, temporary

    def unavailable_link(*args, **kwargs):
        raise OSError("Hard links unavailable")

    def replace(source, destination):
        if Path(source) in copies:
            raise OSError("Checkpoint publication unavailable")
        return real_replace(source, destination)

    def unlink(path, *args, **kwargs):
        if path in copies:
            raise PermissionError("Temporary component unavailable")
        return real_unlink(path, *args, **kwargs)

    def reject_generation(*args, **kwargs):
        raise ValueError("Generation unavailable")

    with monkeypatch.context() as patch:
        patch.setattr(module.tempfile, "mkstemp", mkstemp)
        patch.setattr(os, "link", unavailable_link)
        patch.setattr(os, "replace", replace)
        patch.setattr(Path, "unlink", unlink)
        if operation == "settle":
            patch.setattr(store, "_state", reject_generation)
        with pytest.raises(OSError, match="Temporary component unavailable"):
            if operation == "begin":
                store.begin_transaction()
            else:
                store.end_transaction(committed=True)

    assert len(copies) == 1 and copies[0].is_file()
    old = time.time() - 7200
    os.utime(copies[0], (old, old))
    unrelated = binding["path"].parent / "unrelated.tmp"
    unrelated.write_bytes(b"keep")

    assert CascadeHelper.delete_faiss_index_files(str(binding["path"]))
    assert not binding["path"].exists()
    assert not module.segment_directory(binding["path"]).exists()
    assert not copies[0].exists()
    assert unrelated.read_bytes() == b"keep"


def test_index_storage_size_counts_segments_not_just_manifest(tmp_path):
    """Collection details must report vector bytes, not the manifest size."""
    from local_deep_research.vector_stores.segment_files import (
        index_storage_size,
        segment_directory,
    )

    legacy = tmp_path / "legacy.faiss"
    legacy.write_bytes(b"x" * 100)
    assert index_storage_size(legacy) == 100

    root = tmp_path / "index.faiss"
    root.write_bytes(b"m" * 10)
    directory = segment_directory(root)
    directory.mkdir()
    (directory / ("a" * 32 + ".faiss")).write_bytes(b"v" * 1000)
    (directory / ("a" * 32 + ".ids")).write_bytes(b"i" * 50)
    outside = tmp_path / "outside.bin"
    outside.write_bytes(b"o" * 7000)
    (directory / "link.faiss").symlink_to(outside)
    assert index_storage_size(root) == 1060

    # A corrupted recorded path reports an unknown size instead of raising.
    loop = tmp_path / "loop.faiss"
    loop.symlink_to(loop)
    for corrupted in (
        legacy / "under-a-file.faiss",  # ENOTDIR
        loop,  # ELOOP
        tmp_path / ("x" * 4096 + ".faiss"),  # ENAMETOOLONG
        Path(str(tmp_path / "nul") + "\0.faiss"),  # ValueError
    ):
        assert index_storage_size(corrupted) is None


def test_index_storage_size_tolerates_unreadable_segment_directory(
    tmp_path, monkeypatch
):
    from local_deep_research.vector_stores import segment_files

    root = tmp_path / "index.faiss"
    root.write_bytes(b"x" * 10)
    segment_files.segment_directory(root).mkdir()

    def denied(path):
        raise PermissionError("Permission denied")

    monkeypatch.setattr(segment_files.os, "scandir", denied)

    assert segment_files.index_storage_size(root) == 10


def test_index_storage_size_skips_files_retired_mid_scan(tmp_path, monkeypatch):
    """A writer retiring a segment while the details page sums sizes must
    not raise (readers hold no lock); vanished entries are skipped."""
    from local_deep_research.vector_stores import segment_files

    root = tmp_path / "index.faiss"
    root.write_bytes(b"m" * 10)
    directory = segment_files.segment_directory(root)
    directory.mkdir()
    (directory / ("a" * 32 + ".faiss")).write_bytes(b"v" * 1000)
    victim = directory / ("b" * 32 + ".faiss")
    victim.write_bytes(b"r" * 500)

    # Retire the victim after it is listed but before it is stat'ed, for
    # both a scandir-based and a Path.iterdir/is_file-based scan.
    real_scandir = os.scandir

    class _RetiringScan:
        def __init__(self, target):
            self._scan = real_scandir(target)

        def __enter__(self):
            return self

        def __exit__(self, *exc):
            self._scan.close()

        def __iter__(self):
            for entry in self._scan:
                yield _RetiringEntry(entry)

    class _RetiringEntry:
        def __init__(self, entry):
            self._entry = entry

        def __getattr__(self, name):
            return getattr(self._entry, name)

        def __fspath__(self):
            return self._entry.path

        def stat(self, **kwargs):
            if self._entry.name == victim.name:
                victim.unlink(missing_ok=True)
            return self._entry.stat(**kwargs)

    real_is_file = Path.is_file

    def retiring_is_file(self):
        result = real_is_file(self)
        if self == victim:
            victim.unlink(missing_ok=True)
        return result

    monkeypatch.setattr(segment_files.os, "scandir", _RetiringScan)
    monkeypatch.setattr(Path, "is_file", retiring_is_file)
    assert segment_files.index_storage_size(root) == 1010
    monkeypatch.undo()

    # The segment directory or the root itself vanishing is not an error.
    victim.write_bytes(b"r")
    segment_files.delete_segment_files(root)
    assert segment_files.index_storage_size(root) == 10
    root.unlink()
    assert segment_files.index_storage_size(root) is None


@pytest.fixture
def fresh_verification(monkeypatch):
    """Isolate the process-wide verified-file record from other tests."""
    monkeypatch.setattr(module, "_VERIFIED", module._VerifiedFiles(1024))


def _count_digests(monkeypatch):
    digested = []
    original = module._digest

    def counting(path):
        digested.append(Path(path).suffix)
        return original(path)

    monkeypatch.setattr(module, "_digest", counting)
    return digested


def _whole_second_times():
    """Emulate a filesystem storing whole-second timestamps (NFSv2/3,
    CIFS, VFAT, HFS+): every recorded time lands on a whole second."""

    original = module._file_signature

    def coarse(path):
        signature = original(path)
        if signature is None:
            return None
        return signature[:3] + tuple(
            value - value % 1_000_000_000 for value in signature[3:]
        )

    return coarse


def _one_coarse_tick():
    """Emulate one whole-second tick that spans both a verification and a
    rewrite inside it: the mount reports the same recorded fields for both."""

    original = module._file_signature
    remembered = {}

    def coarse(path):
        if "signature" not in remembered:
            signature = original(path)
            if signature is not None:
                remembered["signature"] = signature[:3] + tuple(
                    value - value % 1_000_000_000 for value in signature[3:]
                )
        return remembered.get("signature")

    return coarse


def test_repeated_search_hashes_each_segment_once(
    binding, monkeypatch, fresh_verification
):
    monkeypatch.setattr(module, "SEGMENT_VECTORS", 4)
    store = module.SegmentedFaissStore.create(**binding)
    store.apply(add_ids=list(range(1, 13)), add_vectors=vectors(12))
    reader = module.SegmentedFaissStore.load(**binding)
    module._VERIFIED.clear()
    digested = _count_digests(monkeypatch)

    first = reader.search(vectors(1, 3)[0], 5)
    assert sorted(digested) == [".faiss"] * 3 + [".ids"] * 3
    digested.clear()
    assert reader.search(vectors(1, 3)[0], 5) == first
    assert digested == []


@pytest.mark.parametrize("suffix", [".faiss", ".ids"])
@pytest.mark.parametrize("restore_mtime", [False, True])
def test_rewriting_a_verified_segment_is_detected(
    binding, fresh_verification, suffix, restore_mtime
):
    store = module.SegmentedFaissStore.create(**binding)
    store.apply(add_ids=[1, 2], add_vectors=vectors(2))
    store.search(vectors(1)[0], 1)
    entry = state(binding["path"])[0]
    path = (
        module.segment_directory(binding["path"]) / entry["file"]
    ).with_suffix(suffix)
    before = os.stat(path)
    data = bytearray(path.read_bytes())
    data[-1] ^= 0xFF  # Same size, same inode.
    time.sleep(0.01)
    path.write_bytes(bytes(data))
    if restore_mtime:
        # utime can restore the modification time but not the change time.
        os.utime(path, ns=(before.st_atime_ns, before.st_mtime_ns))
    with pytest.raises(ValueError, match="checksum"):
        store.search(vectors(1)[0], 1)


def test_coarse_stored_times_re_hash_segments_on_every_read(
    binding, monkeypatch, fresh_verification
):
    # Whole-second stored times cannot show a same-tick rewrite in the
    # recorded fields, so segments on such mounts forfeit the verification
    # cache and are hashed on every read (safe direction: pre-caching cost).
    monkeypatch.setattr(module, "SEGMENT_VECTORS", 4)
    store = module.SegmentedFaissStore.create(**binding)
    store.apply(add_ids=list(range(1, 13)), add_vectors=vectors(12))
    reader = module.SegmentedFaissStore.load(**binding)
    module._VERIFIED.clear()
    digested = _count_digests(monkeypatch)

    first = reader.search(vectors(1, 3)[0], 5)
    assert sorted(digested) == [".faiss"] * 3 + [".ids"] * 3
    digested.clear()
    assert reader.search(vectors(1, 3)[0], 5) == first
    assert digested == []  # Fine-grained times: the cache serves the read.

    monkeypatch.setattr(module, "_file_signature", _whole_second_times())
    for _ in range(2):
        digested.clear()
        assert reader.search(vectors(1, 3)[0], 5) == first
        assert sorted(digested) == [".faiss"] * 3 + [".ids"] * 3


def test_same_tick_rewrite_with_coarse_times_is_re_hashed(
    binding, fresh_verification, monkeypatch
):
    # The worst case: on a whole-second-timestamp filesystem, a same-size
    # rewrite and a utime restore inside one tick leave every recorded
    # field unchanged. The cache must not serve that read unhashed.
    store = module.SegmentedFaissStore.create(**binding)
    store.apply(add_ids=[1, 2], add_vectors=vectors(2))
    entry = state(binding["path"])[0]
    monkeypatch.setattr(module, "_file_signature", _one_coarse_tick())
    store.search(vectors(1)[0], 1)  # Verifies within the coarse tick.
    needle = np.asarray(store.reconstruct(1), dtype="<f4").tobytes()
    path = (
        module.segment_directory(binding["path"]) / entry["file"]
    ).with_suffix(".faiss")
    data = bytearray(path.read_bytes())
    # Corrupt a stored vector value in place: same size, same inode, ids
    # and bounds untouched, so the hash is the only thing that can notice.
    # (Corrupting the id map instead would be caught by the membership
    # cross-check even without the hash.)
    offset = bytes(data).find(needle)
    assert offset > 0
    data[offset] ^= 0xFF
    path.write_bytes(bytes(data))
    with pytest.raises(ValueError, match="checksum"):
        store.search(vectors(1)[0], 1)


def test_verified_files_are_bounded():
    record = module._VerifiedFiles(limit=2)
    for name in ("a", "b", "c"):
        record.remember(Path(name), "digest", ("signature",))
    assert [key[0] for key in record._entries] == ["b", "c"]


def test_orphan_scan_runs_periodically_or_for_reusable_ids(
    binding, monkeypatch
):
    monkeypatch.setattr(module, "ORPHAN_SCAN_INTERVAL", 3)
    store = module.SegmentedFaissStore.create(**binding)
    assert not store.orphan_scan_due(None)  # Nothing indexed yet.
    store.apply(add_ids=[1], add_vectors=vectors(1))
    assert store.orphan_scan_due(1)  # No scan has been recorded.
    store.apply(add_ids=[2], add_vectors=vectors(1, 2), orphan_scanned=True)
    assert not store.orphan_scan_due(2)
    # A vector above the highest chunk row could collide with a new row.
    assert store.orphan_scan_due(1)
    assert store.orphan_scan_due(None)
    for value in (3, 4):
        assert not store.orphan_scan_due(value - 1)
        store.apply(add_ids=[value], add_vectors=vectors(1, value))
    assert not store.orphan_scan_due(4)
    store.apply(add_ids=[5], add_vectors=vectors(1, 5))
    assert store.orphan_scan_due(5)  # The interval has elapsed.
    # Republishing an unchanged generation keeps its scan record.
    store.apply(add_ids=[6], add_vectors=vectors(1, 6), orphan_scanned=True)
    store.persist(binding["path"])
    assert not store.orphan_scan_due(6)


def test_small_rewritten_segments_merge_with_their_neighbour(
    binding, monkeypatch
):
    monkeypatch.setattr(module, "SEGMENT_VECTORS", 4)
    store = module.SegmentedFaissStore.create(**binding)
    original = vectors(12)
    store.apply(add_ids=list(range(1, 13)), add_vectors=original)
    assert [entry["count"] for entry in state(binding["path"])] == [4, 4, 4]

    store.apply(add_ids=[], add_vectors=None, remove_ids=[1, 2, 3, 5, 6])
    assert [entry["count"] for entry in state(binding["path"])] == [3, 4]
    assert sorted(store.live_ids()) == [4, 7, 8, 9, 10, 11, 12]
    for value in (4, 7, 9):
        np.testing.assert_allclose(
            store.reconstruct(value), original[value - 1], rtol=1e-6
        )
    # Untouched full segments are never rewritten to merge.
    untouched = state(binding["path"])[-1]["file"]
    store.apply(add_ids=[], add_vectors=None, remove_ids=[4])
    assert state(binding["path"])[-1]["file"] == untouched