Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

feat(cli): add `leann rebuild` for re-running a build with stored config (#123) (#326)

Abieee075f04e79
diff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.py
index 0d50cf3..27ba4b6 100644
--- a/packages/leann-core/src/leann/cli.py
+++ b/packages/leann-core/src/leann/cli.py
@@ -38,6 +38,45 @@ def _normalize_path(path: str) -> str:
return str(Path(path).resolve())
+def _cleanup_path(path: Path) -> None:
+ if path.is_dir():
+ import shutil
+
+ shutil.rmtree(path)
+ elif path.exists():
+ path.unlink()
+
+
+def _existing_index_artifacts(index_dir: Path) -> bool:
+ return (
+ (index_dir / "documents.leann.meta.json").exists()
+ and (index_dir / "documents.leann.passages.jsonl").exists()
+ and (index_dir / "documents.leann.passages.idx").exists()
+ )
+
+
+def _publish_rebuilt_index(staging_dir: Path, index_dir: Path) -> None:
+ """Publish a staged full rebuild, restoring the previous directory if publish fails."""
+ backup_dir = index_dir.with_name(f".{index_dir.name}.backup-{uuid.uuid4().hex}")
+ live_moved = False
+ published = False
+ try:
+ if index_dir.exists():
+ os.replace(index_dir, backup_dir)
+ live_moved = True
+ os.replace(staging_dir, index_dir)
+ published = True
+ except Exception:
+ if published:
+ _cleanup_path(index_dir)
+ if live_moved and backup_dir.exists():
+ os.replace(backup_dir, index_dir)
+ raise
+ else:
+ if backup_dir.exists():
+ _cleanup_path(backup_dir)
+
+
@contextlib.contextmanager
def suppress_cpp_output(suppress: bool = True):
"""Context manager to suppress C++ stdout/stderr output from FAISS/HNSW
@@ -371,6 +410,18 @@ Examples:
help="Report changes without rebuilding (original watch behavior)",
)
+ rebuild_parser = subparsers.add_parser(
+ "rebuild",
+ help="Rebuild an existing index using its stored config (delta by default; --force for full rebuild)",
+ )
+ rebuild_parser.add_argument("index_name", help="Index name")
+ rebuild_parser.add_argument(
+ "-f",
+ "--force",
+ action="store_true",
+ help="Full rebuild from scratch instead of incremental delta",
+ )
+
# Search command
search_parser = subparsers.add_parser("search", help="Search documents")
search_parser.add_argument("index_name", help="Index name")
@@ -1779,6 +1830,54 @@ Examples:
opts["prompt_template"] = args.embedding_prompt_template
return opts
+ def _build_config_from_args(
+ self,
+ args,
+ docs_paths: list[str],
+ *,
+ doc_chunk_size: Optional[int] = None,
+ doc_chunk_overlap: Optional[int] = None,
+ code_chunk_size: Optional[int] = None,
+ code_chunk_overlap: Optional[int] = None,
+ ) -> dict[str, Any]:
+ """Capture the CLI build settings needed to replay `leann build`."""
+ embedding_options = self._build_embedding_options(args)
+ config: dict[str, Any] = {
+ "docs": [str(Path(path).resolve()) for path in docs_paths],
+ "file_types": args.file_types,
+ "include_hidden": bool(args.include_hidden),
+ "doc_chunk_size": doc_chunk_size
+ if doc_chunk_size is not None
+ else int(args.doc_chunk_size),
+ "doc_chunk_overlap": doc_chunk_overlap
+ if doc_chunk_overlap is not None
+ else int(args.doc_chunk_overlap),
+ "code_chunk_size": code_chunk_size
+ if code_chunk_size is not None
+ else int(args.code_chunk_size),
+ "code_chunk_overlap": code_chunk_overlap
+ if code_chunk_overlap is not None
+ else int(args.code_chunk_overlap),
+ "use_ast_chunking": bool(args.use_ast_chunking),
+ "ast_chunk_size": int(args.ast_chunk_size),
+ "ast_chunk_overlap": int(args.ast_chunk_overlap),
+ "ast_fallback_traditional": bool(args.ast_fallback_traditional),
+ "graph_degree": int(args.graph_degree),
+ "complexity": int(args.complexity),
+ "num_threads": int(args.num_threads),
+ "compact": bool(args.compact),
+ "recompute": bool(args.recompute),
+ }
+ if "host" in embedding_options:
+ config["embedding_host"] = embedding_options["host"]
+ if "base_url" in embedding_options:
+ config["embedding_api_base"] = embedding_options["base_url"]
+ if args.embedding_prompt_template:
+ config["embedding_prompt_template"] = args.embedding_prompt_template
+ if args.query_prompt_template:
+ config["query_prompt_template"] = args.query_prompt_template
+ return config
+
def _resolve_sync_roots(self, docs_paths: list[str]) -> list[str]:
roots: set[str] = set()
for path in docs_paths:
@@ -2076,6 +2175,7 @@ Examples:
roots: list[str],
include_extensions: Optional[list[str]],
ignore_patterns: Optional[list[str]],
+ build_config: Optional[dict[str, Any]] = None,
) -> None:
sync_config_path = index_dir / "sync_roots.json"
config = {
@@ -2083,6 +2183,8 @@ Examples:
"include_extensions": include_extensions,
"ignore_patterns": ignore_patterns,
}
+ if build_config is not None:
+ config["build_config"] = build_config
with open(sync_config_path, "w", encoding="utf-8") as f:
json.dump(config, f, indent=2)
@@ -2224,6 +2326,14 @@ Examples:
separator="\n",
paragraph_separator="\n\n",
)
+ build_config = self._build_config_from_args(
+ args,
+ docs_paths,
+ doc_chunk_size=doc_chunk_size,
+ doc_chunk_overlap=doc_chunk_overlap,
+ code_chunk_size=code_chunk_size,
+ code_chunk_overlap=code_chunk_overlap,
+ )
# Detect changes first so we can skip load_documents for remove-only
index_dir.mkdir(parents=True, exist_ok=True)
@@ -2286,6 +2396,7 @@ Examples:
self._resolve_sync_roots(docs_paths),
self._parse_file_types(args.file_types),
self._sync_ignore_patterns(args.include_hidden),
+ build_config,
)
self.register_project_dir()
return
@@ -2336,6 +2447,7 @@ Examples:
self._resolve_sync_roots(docs_paths),
self._parse_file_types(args.file_types),
self._sync_ignore_patterns(args.include_hidden),
+ build_config,
)
self.register_project_dir()
return
@@ -2354,6 +2466,7 @@ Examples:
self._resolve_sync_roots(docs_paths),
self._parse_file_types(args.file_types),
self._sync_ignore_patterns(args.include_hidden),
+ build_config,
)
self.register_project_dir()
return
@@ -2373,6 +2486,12 @@ Examples:
return
print(f"Building index '{index_name}' with {args.backend_name} backend...")
+ publish_from_staging = _existing_index_artifacts(index_dir)
+ target_index_dir = index_dir
+ target_index_path = index_path
+ if publish_from_staging:
+ target_index_dir = index_dir.with_name(f".{index_dir.name}.rebuild-{uuid.uuid4().hex}")
+ target_index_path = str(target_index_dir / "documents.leann")
builder = LeannBuilder(
backend_name=args.backend_name,
@@ -2389,15 +2508,35 @@ Examples:
for chunk in all_texts:
builder.add_text(chunk["text"], metadata=chunk["metadata"])
- builder.build_index(index_path)
- for fs in synchronizers:
- fs.create_snapshot()
- self._write_sync_config(
- index_dir,
- self._resolve_sync_roots(docs_paths),
- self._parse_file_types(args.file_types),
- self._sync_ignore_patterns(args.include_hidden),
- )
+ try:
+ builder.build_index(target_index_path)
+ target_synchronizers = (
+ self._build_synchronizers(
+ docs_paths,
+ target_index_dir,
+ file_types=args.file_types,
+ include_hidden=args.include_hidden,
+ )
+ if publish_from_staging
+ else synchronizers
+ )
+ for fs in target_synchronizers:
+ fs.create_snapshot()
+ self._write_sync_config(
+ target_index_dir,
+ self._resolve_sync_roots(docs_paths),
+ self._parse_file_types(args.file_types),
+ self._sync_ignore_patterns(args.include_hidden),
+ build_config,
+ )
+ if publish_from_staging:
+ _publish_rebuilt_index(target_index_dir, index_dir)
+ except Exception:
+ if publish_from_staging and target_index_dir.exists():
+ import shutil
+
+ shutil.rmtree(target_index_dir)
+ raise
print(f"Index built at {index_path}")
self.register_project_dir()
@@ -2460,34 +2599,55 @@ Examples:
print(f" - {file_path}")
print(f" chunks: {chunk_display}")
- async def _watch_trigger_build(self, index_name: str) -> None:
- """Trigger an idempotent build for the given index, reusing its stored config."""
+ def _reconstruct_build_args(
+ self, index_name: str, *, force: bool = False, verbose: bool = False
+ ) -> Optional[list[str]]:
+ """Reconstruct the `leann build` CLI args for an existing index from its
+ stored config (.meta.json + sync_roots.json).
+
+ Returns None when the index can't be rebuilt this way (no sync config,
+ missing metadata, etc.). With verbose=True the reason gets printed (for
+ the user-driven `leann rebuild` path); with verbose=False the failures
+ are silent (for the watch loop, which polls and shouldn't spam logs).
+ """
resolved = self._resolve_index_for_watch(index_name)
if not resolved:
- return
+ return None
index_dir = resolved["index_dir"]
sync_config_path = index_dir / "sync_roots.json"
if not sync_config_path.exists():
- return
+ if verbose:
+ print(
+ f"Cannot rebuild '{index_name}': no sync config at {sync_config_path}. "
+ f"This usually means the index was built via the Python API rather than "
+ f"`leann build --docs <dir>`, so the original docs roots aren't recorded."
+ )
+ return None
with open(sync_config_path, encoding="utf-8") as f:
config = json.load(f)
roots = config.get("roots") or []
if not roots:
- return
+ if verbose:
+ print(f"Cannot rebuild '{index_name}': sync config has no document roots.")
+ return None
meta_path = index_dir / "documents.leann.meta.json"
if not meta_path.exists():
- print(f"Index metadata missing for '{index_name}', cannot rebuild.")
- return
+ if verbose:
+ print(f"Cannot rebuild '{index_name}': index metadata missing at {meta_path}.")
+ else:
+ print(f"Index metadata missing for '{index_name}', cannot rebuild.")
+ return None
with open(meta_path, encoding="utf-8") as f:
meta = json.load(f)
- parser = self.create_parser()
+ build_config = config.get("build_config") or meta.get("build_config") or {}
+ docs = build_config.get("docs") or roots
build_args_list = [
"build",
index_name,
"--docs",
- *roots,
+ *[str(doc) for doc in docs],
"--backend-name",
meta.get("backend_name", "hnsw"),
"--embedding-model",
@@ -2496,11 +2656,105 @@ Examples:
meta.get("embedding_mode", "sentence-transformers"),
]
bkw = meta.get("backend_kwargs", {})
- if not bkw.get("is_compact", False):
- build_args_list.append("--no-compact")
- if bkw.get("is_recompute", True):
- build_args_list.append("--recompute")
+ def add_option(flag: str, value: Any) -> None:
+ if value is not None:
+ build_args_list.extend([flag, str(value)])
+
+ def add_bool(true_flag: str, false_flag: str, value: Any) -> None:
+ if value is None:
+ return
+ build_args_list.append(true_flag if bool(value) else false_flag)
+
+ def config_or_backend(config_key: str, backend_key: str) -> Any:
+ if config_key in build_config:
+ return build_config[config_key]
+ return bkw.get(backend_key)
+
+ add_option("--graph-degree", config_or_backend("graph_degree", "graph_degree"))
+ add_option("--complexity", config_or_backend("complexity", "complexity"))
+ add_option("--num-threads", config_or_backend("num_threads", "num_threads"))
+
+ compact = (
+ build_config["compact"]
+ if "compact" in build_config
+ else meta.get("is_compact", bkw.get("is_compact"))
+ )
+ recompute = (
+ build_config["recompute"]
+ if "recompute" in build_config
+ else meta.get("is_recompute", bkw.get("is_recompute"))
+ )
+ add_bool("--compact", "--no-compact", compact)
+ add_bool("--recompute", "--no-recompute", recompute)
+
+ file_types = build_config.get("file_types")
+ if file_types is None:
+ include_extensions = config.get("include_extensions")
+ if include_extensions:
+ file_types = ",".join(include_extensions)
+ add_option("--file-types", file_types)
+
+ include_hidden = build_config.get("include_hidden")
+ if include_hidden is None:
+ include_hidden = config.get("ignore_patterns") is None
+ if include_hidden:
+ build_args_list.append("--include-hidden")
+
+ add_option("--doc-chunk-size", build_config.get("doc_chunk_size"))
+ add_option("--doc-chunk-overlap", build_config.get("doc_chunk_overlap"))
+ add_option("--code-chunk-size", build_config.get("code_chunk_size"))
+ add_option("--code-chunk-overlap", build_config.get("code_chunk_overlap"))
+ if build_config.get("use_ast_chunking"):
+ build_args_list.append("--use-ast-chunking")
+ add_option("--ast-chunk-size", build_config.get("ast_chunk_size"))
+ add_option("--ast-chunk-overlap", build_config.get("ast_chunk_overlap"))
+ if build_config.get("ast_fallback_traditional"):
+ build_args_list.append("--ast-fallback-traditional")
+
+ embedding_options = meta.get("embedding_options") or {}
+ embedding_host = build_config.get("embedding_host", embedding_options.get("host"))
+ embedding_api_base = build_config.get(
+ "embedding_api_base", embedding_options.get("base_url")
+ )
+ embedding_prompt_template = build_config.get("embedding_prompt_template")
+ if embedding_prompt_template is None:
+ embedding_prompt_template = embedding_options.get(
+ "build_prompt_template", embedding_options.get("prompt_template")
+ )
+ query_prompt_template = build_config.get(
+ "query_prompt_template", embedding_options.get("query_prompt_template")
+ )
+ add_option("--embedding-host", embedding_host)
+ add_option("--embedding-api-base", embedding_api_base)
+ add_option("--embedding-prompt-template", embedding_prompt_template)
+ add_option("--query-prompt-template", query_prompt_template)
+ if force:
+ build_args_list.append("--force")
+ return build_args_list
+
+ async def rebuild_index(self, args) -> None:
+ """Rebuild an existing index using its stored config.
+
+ By default this is an incremental delta (leann build is idempotent and
+ only re-indexes added/changed/removed files). Pass --force to do a full
+ rebuild from scratch.
+ """
+ build_args_list = self._reconstruct_build_args(
+ args.index_name, force=getattr(args, "force", False), verbose=True
+ )
+ if build_args_list is None:
+ return
+ parser = self.create_parser()
+ build_args = parser.parse_args(build_args_list)
+ await self.build_index(build_args)
+
+ async def _watch_trigger_build(self, index_name: str) -> None:
+ """Trigger an idempotent build for the given index, reusing its stored config."""
+ build_args_list = self._reconstruct_build_args(index_name, verbose=False)
+ if build_args_list is None:
+ return
+ parser = self.create_parser()
build_args = parser.parse_args(build_args_list)
await self.build_index(build_args)
@@ -3228,6 +3482,9 @@ Examples:
await self.build_index(args)
elif args.command == "watch":
await self.watch_index(args)
+ elif args.command == "rebuild":
+ with suppress_cpp_output(suppress):
+ await self.rebuild_index(args)
elif args.command == "search":
with suppress_cpp_output(suppress):
await self.search_documents(args)
diff --git a/tests/test_readme_examples.py b/tests/test_readme_examples.py
index fb9b1f3..8fe0704 100644
--- a/tests/test_readme_examples.py
+++ b/tests/test_readme_examples.py
@@ -7,12 +7,87 @@ import platform
import tempfile
from pathlib import Path
+import numpy as np
import pytest
+TEST_EMBEDDING_MODEL = "test-deterministic-embeddings"
+TEST_EMBEDDING_DIMENSIONS = 8
+
+
+def _deterministic_embeddings(
+ chunks,
+ model_name,
+ mode="sentence-transformers",
+ use_server=True,
+ port=None,
+ is_build=False,
+ provider_options=None,
+):
+ del model_name, mode, use_server, port, is_build, provider_options
+
+ embeddings = []
+ for chunk in chunks:
+ text = str(chunk).lower()
+ vector = np.zeros(TEST_EMBEDDING_DIMENSIONS, dtype=np.float32)
+ if any(term in text for term in ("fantastical", "banana", "crocodile")):
+ vector[0] = 1.0
+ elif any(term in text for term in ("storage", "leann", "saves")):
+ vector[1] = 1.0
+ else:
+ vector[2] = 1.0
+ embeddings.append(vector)
+ return np.vstack(embeddings)
+
+
+def _deterministic_direct_embeddings(
+ chunks,
+ model_name,
+ mode="sentence-transformers",
+ is_build=False,
+ provider_options=None,
+):
+ return _deterministic_embeddings(
+ chunks,
+ model_name,
+ mode=mode,
+ use_server=False,
+ is_build=is_build,
+ provider_options=provider_options,
+ )
+
+
+def deterministic_embeddings(monkeypatch):
+ """Keep README example tests offline and deterministic in CI."""
+ monkeypatch.setattr("leann.api.compute_embeddings", _deterministic_embeddings)
+ monkeypatch.setattr(
+ "leann.embedding_compute.compute_embeddings",
+ _deterministic_direct_embeddings,
+ )
+
+
+def _test_builder_kwargs(backend_name):
+ kwargs = {
+ "backend_name": backend_name,
+ "embedding_model": TEST_EMBEDDING_MODEL,
+ "dimensions": TEST_EMBEDDING_DIMENSIONS,
+ }
+ if backend_name == "hnsw":
+ kwargs.update({"is_recompute": False, "is_compact": False})
+ return kwargs
+
+
+def _skip_if_backend_unavailable(backend_name):
+ from leann.api import get_registered_backends
+
+ if backend_name not in get_registered_backends():
+ pytest.skip(f"Backend {backend_name!r} is not installed")
+
@pytest.mark.parametrize("backend_name", ["hnsw", "diskann"])
-def test_readme_basic_example(backend_name):
+def test_readme_basic_example(backend_name, deterministic_embeddings):
"""Test the basic example from README.md with both backends."""
+ _skip_if_backend_unavailable(backend_name)
# Skip on macOS CI due to MPS environment issues with all-MiniLM-L6-v2
if os.environ.get("CI") == "true" and platform.system() == "Darwin":
pytest.skip("Skipping on macOS CI due to MPS environment issues with all-MiniLM-L6-v2")
@@ -20,21 +95,14 @@ def test_readme_basic_example(backend_name):
if os.environ.get("CI") == "true" and backend_name == "diskann":
pytest.skip("Skip DiskANN tests in CI due to resource constraints and instability")
- # This is the exact code from README (with smaller model for CI)
+ # Exercise the README flow without depending on live model downloads in CI.
from leann import LeannBuilder, LeannChat, LeannSearcher
from leann.api import SearchResult
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:
INDEX_PATH = str(Path(temp_dir) / f"demo_{backend_name}.leann")
- if os.environ.get("CI") == "true":
- builder = LeannBuilder(
- backend_name=backend_name,
- embedding_model="sentence-transformers/all-MiniLM-L6-v2",
- dimensions=384,
- )
- else:
- builder = LeannBuilder(backend_name=backend_name)
+ builder = LeannBuilder(**_test_builder_kwargs(backend_name))
builder.add_text("LEANN saves 97% storage compared to traditional vector databases.")
builder.add_text("Tung Tung Tung Sahur called—they need their banana-crocodile hybrid back")
builder.build_index(INDEX_PATH)
@@ -44,7 +112,7 @@ def test_readme_basic_example(backend_name):
index_files = list(index_dir.glob(f"{Path(INDEX_PATH).stem}.*"))
assert len(index_files) > 0
- with LeannSearcher(INDEX_PATH) as searcher:
+ with LeannSearcher(INDEX_PATH, recompute_embeddings=False, enable_warmup=False) as searcher:
results = searcher.search("fantastical AI-generated creatures", top_k=1)
assert len(results) > 0
@@ -54,8 +122,16 @@ def test_readme_basic_example(backend_name):
)
assert "banana" in results[0].text or "crocodile" in results[0].text
- chat = LeannChat(INDEX_PATH, llm_config={"type": "simulated"})
- response = chat.ask("How much storage does LEANN save?", top_k=1)
+ chat = LeannChat(
+ INDEX_PATH,
+ llm_config={"type": "simulated"},
+ recompute_embeddings=False,
+ )
+ response = chat.ask(
+ "How much storage does LEANN save?",
+ top_k=1,
+ recompute_embeddings=False,
+ )
# Verify chat works
assert isinstance(response, str)
@@ -75,7 +151,7 @@ def test_readme_imports():
assert callable(LeannChat)
-def test_backend_options():
+def test_backend_options(deterministic_embeddings):
"""Test different backend options mentioned in documentation."""
# Skip on macOS CI due to MPS environment issues with all-MiniLM-L6-v2
if os.environ.get("CI") == "true" and platform.system() == "Darwin":
@@ -85,15 +161,9 @@ def test_backend_options():
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:
is_ci = os.environ.get("CI") == "true"
- embedding_model = (
- "sentence-transformers/all-MiniLM-L6-v2" if is_ci else "facebook/contriever"
- )
- dimensions = 384 if is_ci else None
hnsw_path = str(Path(temp_dir) / "test_hnsw.leann")
- builder_hnsw = LeannBuilder(
- backend_name="hnsw", embedding_model=embedding_model, dimensions=dimensions
- )
+ builder_hnsw = LeannBuilder(**_test_builder_kwargs("hnsw"))
builder_hnsw.add_text("Test document for HNSW backend")
builder_hnsw.build_index(hnsw_path)
assert Path(hnsw_path).parent.exists()
@@ -104,11 +174,10 @@ def test_backend_options():
"Skip DiskANN portion in CI - small datasets trigger MKL parameter "
"errors and pytest-timeout thread kills cause segfaults on Windows"
)
+ _skip_if_backend_unavailable("diskann")
diskann_path = str(Path(temp_dir) / "test_diskann.leann")
- builder_diskann = LeannBuilder(
- backend_name="diskann", embedding_model=embedding_model, dimensions=dimensions
- )
+ builder_diskann = LeannBuilder(**_test_builder_kwargs("diskann"))
builder_diskann.add_text("Test document for DiskANN backend")
builder_diskann.build_index(diskann_path)
assert Path(diskann_path).parent.exists()
@@ -116,8 +185,9 @@ def test_backend_options():
@pytest.mark.parametrize("backend_name", ["hnsw", "diskann"])
-def test_llm_config_simulated(backend_name):
+def test_llm_config_simulated(backend_name, deterministic_embeddings):
"""Test simulated LLM configuration option with both backends."""
+ _skip_if_backend_unavailable(backend_name)
# Skip on macOS CI due to MPS environment issues with all-MiniLM-L6-v2
if os.environ.get("CI") == "true" and platform.system() == "Darwin":
pytest.skip("Skipping on macOS CI due to MPS environment issues with all-MiniLM-L6-v2")
@@ -130,20 +200,13 @@ def test_llm_config_simulated(backend_name):
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:
index_path = str(Path(temp_dir) / f"test_{backend_name}.leann")
- if os.environ.get("CI") == "true":
- builder = LeannBuilder(
- backend_name=backend_name,
- embedding_model="sentence-transformers/all-MiniLM-L6-v2",
- dimensions=384,
- )
- else:
- builder = LeannBuilder(backend_name=backend_name)
+ builder = LeannBuilder(**_test_builder_kwargs(backend_name))
builder.add_text("Test document for LLM testing")
builder.build_index(index_path)
llm_config = {"type": "simulated"}
chat = LeannChat(index_path, llm_config=llm_config)
- response = chat.ask("What is this document about?", top_k=1)
+ response = chat.ask("What is this document about?", top_k=1, recompute_embeddings=False)
assert isinstance(response, str)
assert len(response) > 0
diff --git a/tests/test_rebuild_cli.py b/tests/test_rebuild_cli.py
new file mode 100644
index 0000000..eca86a8
--- /dev/null
+++ b/tests/test_rebuild_cli.py
@@ -0,0 +1,348 @@
+import asyncio
+import json
+import pickle
+from pathlib import Path
+
+import pytest
+from leann.cli import LeannCLI
+
+
+def test_reconstruct_build_args_replays_stored_build_config(tmp_path, monkeypatch):
+ monkeypatch.chdir(tmp_path)
+ docs_root = tmp_path / "docs"
+ docs_root.mkdir()
+ docs_file = docs_root / "only.md"
+ docs_file.write_text("hello", encoding="utf-8")
+
+ cli = LeannCLI()
+ index_dir = tmp_path / ".leann" / "indexes" / "sample"
+ index_dir.mkdir(parents=True)
+ meta_path = index_dir / "documents.leann.meta.json"
+ meta_path.write_text(
+ json.dumps(
+ {
+ "backend_name": "hnsw",
+ "embedding_model": "text-embedding-3-small",
+ "embedding_mode": "openai",
+ "backend_kwargs": {
+ "graph_degree": 32,
+ "complexity": 64,
+ "num_threads": 1,
+ "is_compact": False,
+ "is_recompute": True,
+ },
+ "embedding_options": {
+ "base_url": "https://metadata.example/v1",
+ "api_key": "secret-should-not-be-replayed",
+ "build_prompt_template": "metadata passage: ",
+ "query_prompt_template": "metadata query: ",
+ },
+ }
+ ),
+ encoding="utf-8",
+ )
+ sync_config = {
+ "roots": [str(docs_root)],
+ "include_extensions": [".md"],
+ "ignore_patterns": ["**/.*"],
+ "build_config": {
+ "docs": [str(docs_file.resolve())],
+ "file_types": ".md,.txt",
+ "include_hidden": True,
+ "doc_chunk_size": 384,
+ "doc_chunk_overlap": 96,
+ "code_chunk_size": 640,
+ "code_chunk_overlap": 80,
+ "use_ast_chunking": True,
+ "ast_chunk_size": 420,
+ "ast_chunk_overlap": 72,
+ "ast_fallback_traditional": True,
+ "graph_degree": 48,
+ "complexity": 96,
+ "num_threads": 4,
+ "compact": True,
+ "recompute": False,
+ "embedding_api_base": "https://stored.example/v1",
+ "embedding_prompt_template": "stored passage: ",
+ "query_prompt_template": "stored query: ",
+ },
+ }
+ (index_dir / "sync_roots.json").write_text(json.dumps(sync_config), encoding="utf-8")
+
+ reconstructed = cli._reconstruct_build_args("sample", force=True)
+
+ assert reconstructed is not None
+ assert "--embedding-api-key" not in reconstructed
+ parsed = cli.create_parser().parse_args(reconstructed)
+ assert parsed.docs == [str(docs_file.resolve())]
+ assert parsed.file_types == ".md,.txt"
+ assert parsed.include_hidden is True
+ assert parsed.doc_chunk_size == 384
+ assert parsed.doc_chunk_overlap == 96
+ assert parsed.code_chunk_size == 640
+ assert parsed.code_chunk_overlap == 80
+ assert parsed.use_ast_chunking is True
+ assert parsed.ast_chunk_size == 420
+ assert parsed.ast_chunk_overlap == 72
+ assert parsed.graph_degree == 48
+ assert parsed.complexity == 96
+ assert parsed.num_threads == 4
+ assert parsed.compact is True
+ assert parsed.recompute is False
+ assert parsed.embedding_api_base == "https://stored.example/v1"
+ assert parsed.embedding_prompt_template == "stored passage: "
+ assert parsed.query_prompt_template == "stored query: "
+ assert parsed.force is True
+
+
+def test_reconstruct_build_args_falls_back_to_metadata_and_sync_config(tmp_path, monkeypatch):
+ monkeypatch.chdir(tmp_path)
+ docs_root = tmp_path / "docs"
+ docs_root.mkdir()
+
+ cli = LeannCLI()
+ index_dir = tmp_path / ".leann" / "indexes" / "legacy"
+ index_dir.mkdir(parents=True)
+ (index_dir / "documents.leann.meta.json").write_text(
+ json.dumps(
+ {
+ "backend_name": "hnsw",
+ "embedding_model": "facebook/contriever",
+ "embedding_mode": "ollama",
+ "backend_kwargs": {
+ "graph_degree": 40,
+ "complexity": 88,
+ "num_threads": 3,
+ "is_compact": True,
+ "is_recompute": False,
+ },
+ "embedding_options": {
+ "host": "http://ollama.example:11434",
+ "prompt_template": "passage: ",
+ },
+ }
+ ),
+ encoding="utf-8",
+ )
+ (index_dir / "sync_roots.json").write_text(
+ json.dumps(
+ {
+ "roots": [str(docs_root)],
+ "include_extensions": [".md", ".py"],
+ "ignore_patterns": None,
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ reconstructed = cli._reconstruct_build_args("legacy")
+
+ assert reconstructed is not None
+ parsed = cli.create_parser().parse_args(reconstructed)
+ assert parsed.docs == [str(docs_root)]
+ assert parsed.file_types == ".md,.py"
+ assert parsed.include_hidden is True
+ assert parsed.graph_degree == 40
+ assert parsed.complexity == 88
+ assert parsed.num_threads == 3
+ assert parsed.compact is True
+ assert parsed.recompute is False
+ assert parsed.embedding_host == "http://ollama.example:11434"
+ assert parsed.embedding_prompt_template == "passage: "
+
+
+def test_successful_full_build_persists_rebuild_config(tmp_path, monkeypatch):
+ monkeypatch.chdir(tmp_path)
+ docs_root = tmp_path / "docs"
+ docs_root.mkdir()
+ docs_file = docs_root / "only.py"
+ docs_file.write_text("def hello():\n return 'world'\n", encoding="utf-8")
+
+ cli = LeannCLI()
+ args = cli.create_parser().parse_args(
+ [
+ "build",
+ "sample",
+ "--docs",
+ str(docs_file),
+ "--backend-name",
+ "hnsw",
+ "--file-types",
+ ".py",
+ "--include-hidden",
+ "--doc-chunk-size",
+ "10",
+ "--doc-chunk-overlap",
+ "12",
+ "--code-chunk-size",
+ "20",
+ "--code-chunk-overlap",
+ "25",
+ "--use-ast-chunking",
+ "--ast-chunk-size",
+ "123",
+ "--ast-chunk-overlap",
+ "17",
+ "--graph-degree",
+ "44",
+ "--complexity",
+ "77",
+ "--num-threads",
+ "2",
+ "--compact",
+ "--no-recompute",
+ ]
+ )
+
+ class FakeSynchronizer:
+ def detect_changes(self):
+ return {str(docs_file.resolve())}, set(), set()
+
+ def create_snapshot(self):
+ pass
+
+ class SuccessfulBuilder:
+ def __init__(self, **kwargs):
+ self.kwargs = kwargs
+
+ def add_text(self, _text, metadata=None):
+ pass
+
+ def build_index(self, index_path):
+ target_dir = Path(index_path).parent
+ target_dir.mkdir(parents=True, exist_ok=True)
+ (target_dir / "documents.leann.meta.json").write_text(
+ json.dumps(
+ {
+ "backend_name": self.kwargs["backend_name"],
+ "embedding_model": self.kwargs["embedding_model"],
+ "embedding_mode": self.kwargs["embedding_mode"],
+ "backend_kwargs": {
+ "graph_degree": self.kwargs["graph_degree"],
+ "complexity": self.kwargs["complexity"],
+ "num_threads": self.kwargs["num_threads"],
+ "is_compact": self.kwargs["is_compact"],
+ "is_recompute": self.kwargs["is_recompute"],
+ },
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ monkeypatch.setattr(cli, "_build_synchronizers", lambda *_args, **_kwargs: [FakeSynchronizer()])
+ monkeypatch.setattr(
+ cli,
+ "load_documents",
+ lambda *_args, **_kwargs: [{"text": "rebuilt", "metadata": {"file_path": str(docs_file)}}],
+ )
+ monkeypatch.setattr(cli, "register_project_dir", lambda: None)
+ monkeypatch.setattr("leann.cli.LeannBuilder", SuccessfulBuilder)
+
+ asyncio.run(cli.build_index(args))
+
+ sync_config = json.loads(
+ (tmp_path / ".leann" / "indexes" / "sample" / "sync_roots.json").read_text(encoding="utf-8")
+ )
+ build_config = sync_config["build_config"]
+ assert build_config["docs"] == [str(docs_file.resolve())]
+ assert build_config["file_types"] == ".py"
+ assert build_config["include_hidden"] is True
+ assert build_config["doc_chunk_size"] == 10
+ assert build_config["doc_chunk_overlap"] == 9
+ assert build_config["code_chunk_size"] == 20
+ assert build_config["code_chunk_overlap"] == 19
+ assert build_config["use_ast_chunking"] is True
+ assert build_config["ast_chunk_size"] == 123
+ assert build_config["ast_chunk_overlap"] == 17
+ assert build_config["graph_degree"] == 44
+ assert build_config["complexity"] == 77
+ assert build_config["num_threads"] == 2
+ assert build_config["compact"] is True
+ assert build_config["recompute"] is False
+
+
+def test_full_rebuild_failure_preserves_existing_index(tmp_path, monkeypatch):
+ monkeypatch.chdir(tmp_path)
+ docs_root = tmp_path / "docs"
+ docs_root.mkdir()
+ docs_file = docs_root / "only.md"
+ docs_file.write_text("hello", encoding="utf-8")
+
+ cli = LeannCLI()
+ index_dir = tmp_path / ".leann" / "indexes" / "sample"
+ index_dir.mkdir(parents=True)
+ meta_path = index_dir / "documents.leann.meta.json"
+ meta_path.write_text(
+ json.dumps(
+ {
+ "backend_name": "hnsw",
+ "embedding_model": "facebook/contriever",
+ "embedding_mode": "sentence-transformers",
+ "backend_kwargs": {"is_compact": False, "is_recompute": True},
+ }
+ ),
+ encoding="utf-8",
+ )
+ passages_file = index_dir / "documents.leann.passages.jsonl"
+ passages_file.write_text(
+ json.dumps({"id": "old", "text": "old text", "metadata": {}}) + "\n",
+ encoding="utf-8",
+ )
+ offset_file = index_dir / "documents.leann.passages.idx"
+ with open(offset_file, "wb") as f:
+ pickle.dump({"old": 0}, f)
+ original_artifacts = {
+ path: path.read_bytes() for path in (meta_path, passages_file, offset_file)
+ }
+ args = cli.create_parser().parse_args(
+ [
+ "build",
+ "sample",
+ "--docs",
+ str(docs_file),
+ "--backend-name",
+ "hnsw",
+ "--force",
+ ]
+ )
+ built_paths: list[str] = []
+
+ class FakeSynchronizer:
+ def create_snapshot(self):
+ pass
+
+ class FailingBuilder:
+ def __init__(self, **_kwargs):
+ pass
+
+ def add_text(self, _text, metadata=None):
+ pass
+
+ def build_index(self, index_path):
+ built_paths.append(index_path)
+ target_dir = Path(index_path).parent
+ target_dir.mkdir(parents=True, exist_ok=True)
+ (target_dir / "documents.leann.passages.jsonl").write_text(
+ json.dumps({"id": "new", "text": "new text", "metadata": {}}) + "\n",
+ encoding="utf-8",
+ )
+ raise RuntimeError("simulated build failure")
+
+ monkeypatch.setattr(cli, "_build_synchronizers", lambda *_args, **_kwargs: [FakeSynchronizer()])
+ monkeypatch.setattr(
+ cli,
+ "load_documents",
+ lambda *_args, **_kwargs: [{"text": "rebuilt", "metadata": {"file_path": str(docs_file)}}],
+ )
+ monkeypatch.setattr(cli, "register_project_dir", lambda: None)
+ monkeypatch.setattr("leann.cli.LeannBuilder", FailingBuilder)
+
+ with pytest.raises(RuntimeError, match="simulated build failure"):
+ asyncio.run(cli.build_index(args))
+
+ assert built_paths
+ assert built_paths[0] != str(index_dir / "documents.leann")
+ assert {
+ path: path.read_bytes() for path in (meta_path, passages_file, offset_file)
+ } == original_artifacts
+ assert not list(index_dir.parent.glob(".sample.rebuild-*"))