Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

fix: align precomputed embedding IDs with passages (#422)

dafyy321-pixel7c268aad18c9
diff --git a/packages/leann-core/src/leann/api.py b/packages/leann-core/src/leann/api.py
index 0ff88e6..aa15734 100644
--- a/packages/leann-core/src/leann/api.py
+++ b/packages/leann-core/src/leann/api.py
@@ -706,16 +706,21 @@ class LeannBuilder:
Args:
index_path: Path where the index will be saved
- ids: List of document IDs (will be converted to strings)
+ ids: Document IDs in embedding order. Converted to strings and used as
+ passage IDs even when add_text() assigned different IDs; chunk metadata
+ is preserved.
embeddings: numpy array of shape (n_documents, embedding_dim)
Raises:
- ValueError: If ids and embeddings counts don't match, or dimension mismatch
+ ValueError: If IDs are duplicated, counts don't match, or dimensions differ
"""
if len(ids) != embeddings.shape[0]:
raise ValueError(
f"Mismatch between number of IDs ({len(ids)}) and embeddings ({embeddings.shape[0]})"
)
+ string_ids = [str(id_val) for id_val in ids]
+ if len(string_ids) != len(set(string_ids)):
+ raise ValueError("Document IDs must be unique after conversion to strings")
# Validate/set dimensions
embedding_dim = embeddings.shape[1]
@@ -733,16 +738,19 @@ class LeannBuilder:
# If no text chunks provided, create placeholder text entries
if not self.chunks:
logger.info("No text chunks provided, creating placeholder entries...")
- for id_val in ids:
+ for id_val in string_ids:
self.add_text(
f"Document {id_val}",
- metadata={"id": str(id_val), "from_embeddings": True},
+ metadata={"id": id_val, "from_embeddings": True},
)
else:
raise ValueError(
f"Number of text chunks ({len(self.chunks)}) doesn't match number of embeddings ({len(ids)})"
)
+ for chunk, string_id in zip(self.chunks, string_ids):
+ chunk["id"] = string_id
+
# Build file structure
path = Path(index_path)
index_dir = path.parent
@@ -772,7 +780,6 @@ class LeannBuilder:
pickle.dump(offset_map, f)
# Build the vector index using precomputed embeddings
- string_ids = [str(id_val) for id_val in ids]
# Persist ID map (order == embeddings order)
try:
idmap_file = (
diff --git a/tests/test_build_from_arrays.py b/tests/test_build_from_arrays.py
index d917c8f..706032e 100644
--- a/tests/test_build_from_arrays.py
+++ b/tests/test_build_from_arrays.py
@@ -3,15 +3,66 @@ Tests for LeannBuilder.build_index_from_arrays and its integration with
build_index_from_embeddings (pickle-based path).
"""
+import json
import os
import pickle
import tempfile
from pathlib import Path
+from unittest.mock import Mock
import numpy as np
import pytest
+def test_build_from_arrays_custom_ids_are_searchable(tmp_path, monkeypatch):
+ """Search resolves supplied IDs to the text added before the array build."""
+ from leann.api import BACKEND_REGISTRY, LeannBuilder, LeannSearcher
+
+ backend = Mock()
+ monkeypatch.setitem(BACKEND_REGISTRY, "array-test", backend)
+ index_path = str(tmp_path / "custom-ids.leann")
+ ids = ["doc-a", "doc-b"]
+ embeddings = np.array([[1, 0], [0, 1]], dtype=np.float32)
+
+ builder = LeannBuilder(backend_name="array-test", dimensions=2)
+ builder.add_text("Alpha document", metadata={"id": "original-a", "source": "alpha"})
+ builder.add_text("Beta document", metadata={"source": "beta"})
+ builder.build_index_from_arrays(index_path, ids, embeddings)
+
+ assert backend.builder.return_value.build.call_args.args[1] == ids
+ with open(f"{index_path}.passages.jsonl", encoding="utf-8") as f:
+ assert [json.loads(line)["id"] for line in f] == ids
+ with open(f"{index_path}.passages.idx", "rb") as f:
+ assert set(pickle.load(f)) == set(ids)
+
+ backend.searcher.return_value.compute_query_embedding.return_value = embeddings[0]
+ backend.searcher.return_value.search.return_value = {
+ "labels": [ids],
+ "distances": [[0.9, 0.8]],
+ }
+ with LeannSearcher(index_path, enable_warmup=False, recompute_embeddings=False) as searcher:
+ results = searcher.search("document", top_k=2)
+
+ assert [(result.id, result.text, result.metadata["source"]) for result in results] == [
+ ("doc-a", "Alpha document", "alpha"),
+ ("doc-b", "Beta document", "beta"),
+ ]
+ assert results[0].metadata["id"] == "original-a"
+
+
+def test_build_from_arrays_rejects_colliding_ids(tmp_path, monkeypatch):
+ """Numeric and text IDs that become identical strings cannot share a passage."""
+ from leann.api import BACKEND_REGISTRY, LeannBuilder
+
+ monkeypatch.setitem(BACKEND_REGISTRY, "array-test", Mock())
+ builder = LeannBuilder(backend_name="array-test", dimensions=2)
+ with pytest.raises(ValueError, match="IDs must be unique"):
+ builder.build_index_from_arrays(
+ str(tmp_path / "colliding-ids.leann"), [1, "1"], np.zeros((2, 2), dtype=np.float32)
+ )
+ assert not (tmp_path / "colliding-ids.leann.passages.jsonl").exists()
+
+
@pytest.mark.skipif(
os.environ.get("CI") == "true", reason="Skip model tests in CI to avoid MPS memory issues"
)