[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
fix: align precomputed embedding IDs with passages (#422)
dafyy321-pixel7c268aad18c9
diff --git a/packages/leann-core/src/leann/api.py b/packages/leann-core/src/leann/api.pyindex 0ff88e6..aa15734 100644--- a/packages/leann-core/src/leann/api.py+++ b/packages/leann-core/src/leann/api.py@@ -706,16 +706,21 @@ class LeannBuilder:Args:index_path: Path where the index will be saved- ids: List of document IDs (will be converted to strings)+ ids: Document IDs in embedding order. Converted to strings and used as+ passage IDs even when add_text() assigned different IDs; chunk metadata+ is preserved.embeddings: numpy array of shape (n_documents, embedding_dim)Raises:- ValueError: If ids and embeddings counts don't match, or dimension mismatch+ ValueError: If IDs are duplicated, counts don't match, or dimensions differ"""if len(ids) != embeddings.shape[0]:raise ValueError(f"Mismatch between number of IDs ({len(ids)}) and embeddings ({embeddings.shape[0]})")+ string_ids = [str(id_val) for id_val in ids]+ if len(string_ids) != len(set(string_ids)):+ raise ValueError("Document IDs must be unique after conversion to strings")# Validate/set dimensionsembedding_dim = embeddings.shape[1]@@ -733,16 +738,19 @@ class LeannBuilder:# If no text chunks provided, create placeholder text entriesif not self.chunks:logger.info("No text chunks provided, creating placeholder entries...")- for id_val in ids:+ for id_val in string_ids:self.add_text(f"Document {id_val}",- metadata={"id": str(id_val), "from_embeddings": True},+ metadata={"id": id_val, "from_embeddings": True},)else:raise ValueError(f"Number of text chunks ({len(self.chunks)}) doesn't match number of embeddings ({len(ids)})")+ for chunk, string_id in zip(self.chunks, string_ids):+ chunk["id"] = string_id+# Build file structurepath = Path(index_path)index_dir = path.parent@@ -772,7 +780,6 @@ class LeannBuilder:pickle.dump(offset_map, f)# Build the vector index using precomputed embeddings- string_ids = [str(id_val) for id_val in ids]# Persist ID map (order == embeddings order)try:idmap_file = (diff --git a/tests/test_build_from_arrays.py b/tests/test_build_from_arrays.pyindex d917c8f..706032e 100644--- a/tests/test_build_from_arrays.py+++ b/tests/test_build_from_arrays.py@@ -3,15 +3,66 @@ Tests for LeannBuilder.build_index_from_arrays and its integration withbuild_index_from_embeddings (pickle-based path)."""+import jsonimport osimport pickleimport tempfilefrom pathlib import Path+from unittest.mock import Mockimport numpy as npimport pytest+def test_build_from_arrays_custom_ids_are_searchable(tmp_path, monkeypatch):+ """Search resolves supplied IDs to the text added before the array build."""+ from leann.api import BACKEND_REGISTRY, LeannBuilder, LeannSearcher++ backend = Mock()+ monkeypatch.setitem(BACKEND_REGISTRY, "array-test", backend)+ index_path = str(tmp_path / "custom-ids.leann")+ ids = ["doc-a", "doc-b"]+ embeddings = np.array([[1, 0], [0, 1]], dtype=np.float32)++ builder = LeannBuilder(backend_name="array-test", dimensions=2)+ builder.add_text("Alpha document", metadata={"id": "original-a", "source": "alpha"})+ builder.add_text("Beta document", metadata={"source": "beta"})+ builder.build_index_from_arrays(index_path, ids, embeddings)++ assert backend.builder.return_value.build.call_args.args[1] == ids+ with open(f"{index_path}.passages.jsonl", encoding="utf-8") as f:+ assert [json.loads(line)["id"] for line in f] == ids+ with open(f"{index_path}.passages.idx", "rb") as f:+ assert set(pickle.load(f)) == set(ids)++ backend.searcher.return_value.compute_query_embedding.return_value = embeddings[0]+ backend.searcher.return_value.search.return_value = {+ "labels": [ids],+ "distances": [[0.9, 0.8]],+ }+ with LeannSearcher(index_path, enable_warmup=False, recompute_embeddings=False) as searcher:+ results = searcher.search("document", top_k=2)++ assert [(result.id, result.text, result.metadata["source"]) for result in results] == [+ ("doc-a", "Alpha document", "alpha"),+ ("doc-b", "Beta document", "beta"),+ ]+ assert results[0].metadata["id"] == "original-a"+++def test_build_from_arrays_rejects_colliding_ids(tmp_path, monkeypatch):+ """Numeric and text IDs that become identical strings cannot share a passage."""+ from leann.api import BACKEND_REGISTRY, LeannBuilder++ monkeypatch.setitem(BACKEND_REGISTRY, "array-test", Mock())+ builder = LeannBuilder(backend_name="array-test", dimensions=2)+ with pytest.raises(ValueError, match="IDs must be unique"):+ builder.build_index_from_arrays(+ str(tmp_path / "colliding-ids.leann"), [1, "1"], np.zeros((2, 2), dtype=np.float32)+ )+ assert not (tmp_path / "colliding-ids.leann.passages.jsonl").exists()++@pytest.mark.skipif(os.environ.get("CI") == "true", reason="Skip model tests in CI to avoid MPS memory issues")