Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

4.1 KB
Raw Blame
import hashlib
import json
import pickle
from types import SimpleNamespace

from leann.api import Fts5BM25Index, LeannBuilder
from leann.cli import LeannCLI


def _content_id(text: str) -> str:
    return hashlib.sha256(text.encode("utf-8")).hexdigest()[:16]


def test_builder_content_hash_passage_ids_are_content_stable():
    builder = LeannBuilder(backend_name="hnsw", passage_id_scheme="content-hash")

    builder.add_text("same text", metadata={"source": "a.txt"})
    builder.add_text("same text", metadata={"source": "b.txt"})
    builder.add_text("different text", metadata={"source": "c.txt"})

    same_id = _content_id("same text")
    assert builder.chunks[0]["id"] == same_id
    assert builder.chunks[1]["id"] == same_id
    assert builder.chunks[2]["id"] == _content_id("different text")


def test_existing_index_id_scheme_treats_legacy_meta_as_sequential(tmp_path):
    cli = LeannCLI()
    index_path = tmp_path / "legacy.leann"

    assert cli._existing_index_id_scheme(str(index_path)) is None

    meta_path = tmp_path / "legacy.leann.meta.json"
    meta_path.write_text(json.dumps({"version": "1.0", "backend_name": "hnsw"}), encoding="utf-8")

    assert cli._existing_index_id_scheme(str(index_path)) == "sequential"

    meta_path.write_text(
        json.dumps({"version": "1.1", "backend_name": "hnsw", "passage_id_scheme": "content-hash"}),
        encoding="utf-8",
    )

    assert cli._existing_index_id_scheme(str(index_path)) == "content-hash"


def test_migrate_ids_rewrites_passages_offsets_idmap_meta_and_bm25(tmp_path, monkeypatch):
    monkeypatch.chdir(tmp_path)
    index_dir = tmp_path / ".leann" / "indexes" / "sample"
    index_dir.mkdir(parents=True)

    passages = [
        {"id": "0", "text": "alpha beta", "metadata": {"source": "a.txt"}},
        {"id": "1", "text": "gamma delta", "metadata": {"source": "b.txt"}},
    ]
    passages_file = index_dir / "documents.leann.passages.jsonl"
    offsets = {}
    with open(passages_file, "w", encoding="utf-8") as f:
        for passage in passages:
            offsets[passage["id"]] = f.tell()
            json.dump(passage, f)
            f.write("\n")

    offset_file = index_dir / "documents.leann.passages.idx"
    with open(offset_file, "wb") as f:
        pickle.dump(offsets, f)

    idmap_file = index_dir / "documents.ids.txt"
    idmap_file.write_text("0\n1\n", encoding="utf-8")

    bm25_db = index_dir / "documents.leann.bm25.sqlite"
    bm25 = Fts5BM25Index(str(bm25_db))
    bm25.fit(passages)
    bm25.close()

    meta_path = index_dir / "documents.leann.meta.json"
    meta_path.write_text(
        json.dumps(
            {
                "version": "1.0",
                "backend_name": "hnsw",
                "embedding_model": "dummy",
                "dimensions": 3,
                "backend_kwargs": {},
                "embedding_mode": "sentence-transformers",
                "bm25_backend": "fts5",
                "bm25_db": "documents.leann.bm25.sqlite",
                "passage_id_scheme": "sequential",
            }
        ),
        encoding="utf-8",
    )

    cli = LeannCLI()
    cli.migrate_ids(SimpleNamespace(index_name="sample", dry_run=False, yes=True))

    expected_ids = [_content_id("alpha beta"), _content_id("gamma delta")]
    with open(passages_file, encoding="utf-8") as f:
        migrated_passages = [json.loads(line) for line in f if line.strip()]
    assert [p["id"] for p in migrated_passages] == expected_ids

    with open(offset_file, "rb") as f:
        migrated_offsets = pickle.load(f)
    assert set(migrated_offsets) == set(expected_ids)
    assert idmap_file.read_text(encoding="utf-8").splitlines() == expected_ids

    meta = json.loads(meta_path.read_text(encoding="utf-8"))
    assert meta["version"] == "1.1"
    assert meta["passage_id_scheme"] == "content-hash"
    assert meta["bm25_backend"] == "fts5"
    assert meta["bm25_db"] == "documents.leann.bm25.sqlite"

    migrated_bm25 = Fts5BM25Index(str(bm25_db))
    try:
        bm25_results = migrated_bm25.search("alpha", top_k=1)
    finally:
        migrated_bm25.close()
    assert [result.id for result in bm25_results] == [expected_ids[0]]