[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
fix(cli): bound index discovery scans (#389)
Barry2llena60417b3bef5
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.mdindex 2e4d5f2..471927a 100644--- a/docs/CHANGELOG.md+++ b/docs/CHANGELOG.md@@ -41,18 +41,8 @@ fixes). Newest entries at the bottom.comparable recall (GPU latency stays ~flat while CPU grows linearly with nprobe).- Docs: `docs/flashlib_backend_guide.md` gains a `flashlib_ivf` section.-## 2026-07-24: `leann-core` 0.3.8 — HNSW full-rebuild fallback corpus wipe (#386)--- Fixed a bug where, on the HNSW backend, `leann build` / `leann watch` falling- back to a full rebuild (any modified or removed file, since HNSW only- supports add-only incremental updates) rebuilt the index from only the- changed files instead of the full corpus, silently dropping every untouched- file. Root cause: the fallback reused `all_texts` from the incremental path- (which intentionally loads only the delta) instead of reloading the full- corpus via `load_documents(docs_paths, ...)`.-- This was fixed on `main` in commit `956beb5` (merged via #279) but never- reached PyPI — `leann-core` 0.3.7 was still affected. Bumped `leann-core` to- 0.3.8 to ship the fix.-- Added `tests/test_hnsw_rebuild_fallback.py`: regression test that builds an- HNSW index, modifies one file, and asserts untouched files are still present- in the rebuilt corpus.+## 2026-07-27: Bounded `leann list` index discovery++- Bound App-format metadata discovery to a configurable directory depth; `leann list` defaults to depth 5 and accepts `--max-depth` for deeper projects.+- Prune common dependency, virtual-environment, cache, and system directories during discovery.+- Preserve discovery of registered CLI-format and App-format projects without recursively scanning an entire home directory.diff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.pyindex 967342c..7b6adf3 100644--- a/packages/leann-core/src/leann/cli.py+++ b/packages/leann-core/src/leann/cli.py@@ -19,7 +19,7 @@ from tqdm import tqdmfrom .api import Fts5BM25Index, LeannBuilder, LeannChat, LeannSearcherfrom .embedding_server_manager import EmbeddingServerManagerfrom .interactive_utils import create_cli_session-from .registry import register_project_directory+from .registry import DEFAULT_INDEX_SCAN_DEPTH, iter_index_meta_files, register_project_directoryfrom .settings import (resolve_anthropic_base_url,resolve_atlascloud_api_key,@@ -33,6 +33,13 @@ from .settings import (from .sync import DEFAULT_INDEX_EXTENSIONS, FileSynchronizer, parse_include_extensions+def _non_negative_int(value: str) -> int:+ parsed_value = int(value)+ if parsed_value < 0:+ raise argparse.ArgumentTypeError("must be non-negative")+ return parsed_value++def _default_embedding_model() -> str:"""Pick a sensible default embedding model based on platform.@@ -856,7 +863,16 @@ Examples:)# List command- subparsers.add_parser("list", help="List all indexes")+ list_parser = subparsers.add_parser("list", help="List all indexes")+ list_parser.add_argument(+ "--max-depth",+ type=_non_negative_int,+ default=DEFAULT_INDEX_SCAN_DEPTH,+ help=(+ "Maximum directory depth for app index discovery "+ f"(default: {DEFAULT_INDEX_SCAN_DEPTH})"+ ),+ )# Remove commandremove_parser = subparsers.add_parser("remove", help="Remove an index")@@ -947,7 +963,24 @@ Examples:# If anything goes wrong, assume it's not a submodulereturn False- def list_indexes(self):+ @staticmethod+ def _project_has_discoverable_indexes(project_path: Path, max_depth: int) -> bool:+ """Return whether a project contains an index within the scan boundary."""+ cli_indexes_dir = project_path / ".leann" / "indexes"+ return cli_indexes_dir.exists() or any(+ iter_index_meta_files(project_path, max_depth=max_depth)+ )++ @staticmethod+ def _nested_project_dirs(project_path: Path, projects: list[Path]) -> list[Path]:+ """Return registered project roots nested below a project."""+ return [+ candidate+ for candidate in projects+ if candidate != project_path and candidate.is_relative_to(project_path)+ ]++ def list_indexes(self, max_depth: int = DEFAULT_INDEX_SCAN_DEPTH):# Get all project directories with .leannglobal_registry = Path.home() / ".leann" / "projects.json"all_projects = []@@ -961,16 +994,21 @@ Examples:except Exception:pass- # Filter to only existing directories with .leann+ # Filter to existing projects with a discoverable CLI- or App-format index.valid_projects = []for project_dir in all_projects:project_path = Path(project_dir)- if project_path.exists() and (project_path / ".leann" / "indexes").exists():+ if project_path.exists() and self._project_has_discoverable_indexes(+ project_path, max_depth+ ):valid_projects.append(project_path)- # Add current project if it has .leann but not in registry+ # Add current project if it has a discoverable index but is not in registry.current_path = Path.cwd()- if (current_path / ".leann" / "indexes").exists() and current_path not in valid_projects:+ if (+ self._project_has_discoverable_indexes(current_path, max_depth)+ and current_path not in valid_projects+ ):valid_projects.append(current_path)# Separate current and other projects@@ -980,6 +1018,10 @@ Examples:if project_path != current_path:other_projects.append(project_path)+ # Exclude only registered projects nested below the current project.+ # Registered ancestors must not hide indexes owned by the current directory.+ current_exclude_dirs = self._nested_project_dirs(current_path, valid_projects)+print("📚 LEANN Indexes")print("=" * 50)@@ -992,7 +1034,7 @@ Examples:print(" " + "─" * 45)current_indexes = self._discover_indexes_in_project(- current_path, exclude_dirs=other_projects+ current_path, exclude_dirs=current_exclude_dirs, max_depth=max_depth)if current_indexes:for idx in current_indexes:@@ -1011,7 +1053,10 @@ Examples:print(" " + "─" * 45)for project_path in other_projects:- project_indexes = self._discover_indexes_in_project(project_path)+ nested_projects = self._nested_project_dirs(project_path, valid_projects)+ project_indexes = self._discover_indexes_in_project(+ project_path, exclude_dirs=nested_projects, max_depth=max_depth+ )if not project_indexes:continue@@ -1035,9 +1080,14 @@ Examples:projects_count = 0for p in valid_projects:if p == current_path:- discovered = self._discover_indexes_in_project(p, exclude_dirs=other_projects)+ discovered = self._discover_indexes_in_project(+ p, exclude_dirs=current_exclude_dirs, max_depth=max_depth+ )else:- discovered = self._discover_indexes_in_project(p)+ nested_projects = self._nested_project_dirs(p, valid_projects)+ discovered = self._discover_indexes_in_project(+ p, exclude_dirs=nested_projects, max_depth=max_depth+ )if len(discovered) > 0:projects_count += 1print(f"📊 Total: {total_indexes} indexes across {projects_count} projects")@@ -1057,13 +1107,18 @@ Examples:print(" leann build my-docs --docs ./documents")def _discover_indexes_in_project(- self, project_path: Path, exclude_dirs: Optional[list[Path]] = None+ self,+ project_path: Path,+ exclude_dirs: Optional[list[Path]] = None,+ max_depth: Optional[int] = None,):"""Discover all indexes in a project directory (both CLI and apps formats)exclude_dirs: when provided, skip any APP-format index files that arelocated under these directories. This prevents duplicates when thecurrent project is a parent directory of other registered projects.+ max_depth: maximum number of directories to descend for APP-format indexes.+ None preserves full-depth discovery for non-CLI callers."""indexes = []exclude_dirs = exclude_dirs or []@@ -1100,9 +1155,9 @@ Examples:})- # 2. Apps format: *.leann.meta.json files anywhere in the project+ # 2. Apps format: *.leann.meta.json files within the configured scan depthcli_indexes_dir = project_path / ".leann" / "indexes"- for meta_file in project_path.rglob("*.leann.meta.json"):+ for meta_file in iter_index_meta_files(project_path, max_depth=max_depth):if meta_file.is_file():# Skip CLI-built indexes (which store meta under .leann/indexes/<name>/)try:@@ -3732,7 +3787,7 @@ Examples:suppress = not getattr(args, "verbose", False)if args.command == "list":- self.list_indexes()+ self.list_indexes(args.max_depth)elif args.command == "remove":self.remove_index(args.index_name, args.force)elif args.command == "build":diff --git a/packages/leann-core/src/leann/registry.py b/packages/leann-core/src/leann/registry.pyindex 2b74a20..b950ee6 100644--- a/packages/leann-core/src/leann/registry.py+++ b/packages/leann-core/src/leann/registry.py@@ -4,6 +4,8 @@ import importlibimport importlib.metadataimport jsonimport logging+import os+from collections.abc import Iteratorfrom pathlib import Pathfrom typing import TYPE_CHECKING, Optional, Union@@ -14,6 +16,35 @@ if TYPE_CHECKING:logger = logging.getLogger(__name__)BACKEND_REGISTRY: dict[str, "LeannBackendFactoryInterface"] = {}+DEFAULT_INDEX_SCAN_DEPTH = 5+INDEX_SCAN_SKIP_DIRS = frozenset(+ {".git", ".venv", "venv", "node_modules", "__pycache__", "Library"}+)+++def iter_index_meta_files(+ root: Union[str, Path], max_depth: Optional[int] = DEFAULT_INDEX_SCAN_DEPTH+) -> Iterator[Path]:+ """Yield LEANN metadata files within an optionally bounded directory tree.++ The root directory is depth zero. A max_depth of None scans all depths.+ Known dependency, cache, and system directories are always pruned before traversal.+ """+ if max_depth is not None and max_depth < 0:+ raise ValueError("max_depth must be non-negative")++ root_path = Path(root)+ for current_dir, dirnames, filenames in os.walk(root_path):+ current_path = Path(current_dir)+ depth = len(current_path.relative_to(root_path).parts)+ if max_depth is not None and depth >= max_depth:+ dirnames.clear()+ else:+ dirnames[:] = [name for name in dirnames if name not in INDEX_SCAN_SKIP_DIRS]++ for filename in filenames:+ if filename.endswith(".leann.meta.json"):+ yield current_path / filenamedef register_backend(name: str):@@ -49,7 +80,10 @@ def autodiscover_backends():# print("INFO: Backend auto-discovery finished.")-def register_project_directory(project_dir: Optional[Union[str, Path]] = None):+def register_project_directory(+ project_dir: Optional[Union[str, Path]] = None,+ max_depth: Optional[int] = None,+):"""Register a project directory in the global LEANN registry.@@ -57,6 +91,8 @@ def register_project_directory(project_dir: Optional[Union[str, Path]] = None):Args:project_dir: Directory to register. If None, uses current working directory.+ max_depth: Maximum directory depth used when looking for App-format indexes.+ None preserves full-depth discovery for existing API callers."""if project_dir is None:project_dir = Path.cwd()@@ -64,9 +100,9 @@ def register_project_directory(project_dir: Optional[Union[str, Path]] = None):project_dir = Path(project_dir)# Only register directories that have some kind of LEANN content.- # Check CLI-format first to avoid an expensive rglob on large directories.+ # Check CLI-format first to avoid even a bounded scan when it is unnecessary.has_cli_indexes = (project_dir / ".leann" / "indexes").exists()- if not has_cli_indexes and not any(project_dir.rglob("*.leann.meta.json")):+ if not has_cli_indexes and not any(iter_index_meta_files(project_dir, max_depth=max_depth)):# Don't register if there are no LEANN indexesreturndiff --git a/tests/test_cli_list_performance.py b/tests/test_cli_list_performance.pynew file mode 100644index 0000000..de93fab--- /dev/null+++ b/tests/test_cli_list_performance.py@@ -0,0 +1,220 @@+import asyncio+import json+import sys++import pytest+from leann import registry+from leann.cli import LeannCLI+from leann.registry import iter_index_meta_files+++def test_index_discovery_finds_metadata_within_default_depth(tmp_path):+ meta_file = tmp_path / "project" / "data" / "sample.leann.meta.json"+ meta_file.parent.mkdir(parents=True)+ meta_file.touch()++ assert list(iter_index_meta_files(tmp_path)) == [meta_file]+++def test_index_discovery_does_not_descend_past_max_depth(tmp_path):+ shallow_meta = tmp_path / "one" / "shallow.leann.meta.json"+ shallow_meta.parent.mkdir()+ shallow_meta.touch()+ deep_meta = tmp_path / "one" / "two" / "deep.leann.meta.json"+ deep_meta.parent.mkdir()+ deep_meta.touch()++ assert list(iter_index_meta_files(tmp_path, max_depth=1)) == [shallow_meta]++[email protected]("excluded_dir", [".git", ".venv", "node_modules", "Library"])+def test_index_discovery_skips_large_irrelevant_directories(tmp_path, excluded_dir):+ hidden_meta = tmp_path / excluded_dir / "hidden.leann.meta.json"+ hidden_meta.parent.mkdir()+ hidden_meta.touch()++ assert list(iter_index_meta_files(tmp_path)) == []+++def test_index_discovery_rejects_negative_max_depth(tmp_path):+ with pytest.raises(ValueError, match="max_depth must be non-negative"):+ list(iter_index_meta_files(tmp_path, max_depth=-1))+++def test_list_parser_accepts_custom_max_depth():+ args = LeannCLI().create_parser().parse_args(["list", "--max-depth", "5"])++ assert args.max_depth == 5+++def test_list_parser_rejects_negative_max_depth(capsys):+ with pytest.raises(SystemExit):+ LeannCLI().create_parser().parse_args(["list", "--max-depth", "-1"])++ assert "must be non-negative" in capsys.readouterr().err+++def test_cli_project_discovery_respects_max_depth(tmp_path):+ shallow_meta = tmp_path / "shallow" / "index.leann.meta.json"+ shallow_meta.parent.mkdir()+ shallow_meta.touch()+ deep_meta = tmp_path / "one" / "two" / "index.leann.meta.json"+ deep_meta.parent.mkdir(parents=True)+ deep_meta.touch()++ indexes = LeannCLI()._discover_indexes_in_project(tmp_path, max_depth=1)++ assert [index["name"] for index in indexes] == ["shallow"]+++def test_cli_project_discovery_without_max_depth_finds_deep_app_index(tmp_path):+ deep_meta = (+ tmp_path / "one" / "two" / "three" / "four" / "five" / "six" / "index.leann.meta.json"+ )+ deep_meta.parent.mkdir(parents=True)+ deep_meta.touch()++ indexes = LeannCLI()._discover_indexes_in_project(tmp_path)++ assert [index["name"] for index in indexes] == ["six"]+++def test_list_command_passes_max_depth_to_discovery(monkeypatch):+ cli = LeannCLI()+ received = {}++ def capture_max_depth(max_depth):+ received["max_depth"] = max_depth++ monkeypatch.setattr(cli, "list_indexes", capture_max_depth)+ monkeypatch.setattr(sys, "argv", ["leann", "list", "--max-depth", "7"])++ asyncio.run(cli.run())++ assert received == {"max_depth": 7}+++def test_project_registration_respects_explicit_max_depth(tmp_path, monkeypatch):+ home = tmp_path / "home"+ home.mkdir()+ project = tmp_path / "project"+ deep_meta = (+ project / "one" / "two" / "three" / "four" / "five" / "six" / "index.leann.meta.json"+ )+ deep_meta.parent.mkdir(parents=True)+ deep_meta.touch()+ monkeypatch.setattr(registry.Path, "home", classmethod(lambda cls: home))++ registry.register_project_directory(project, max_depth=registry.DEFAULT_INDEX_SCAN_DEPTH)++ assert not (home / ".leann" / "projects.json").exists()+++def test_project_registration_without_max_depth_finds_deep_app_index(tmp_path, monkeypatch):+ home = tmp_path / "home"+ home.mkdir()+ project = tmp_path / "project"+ deep_meta = (+ project / "one" / "two" / "three" / "four" / "five" / "six" / "index.leann.meta.json"+ )+ deep_meta.parent.mkdir(parents=True)+ deep_meta.touch()+ monkeypatch.setattr(registry.Path, "home", classmethod(lambda cls: home))++ registry.register_project_directory(project)++ registry_file = home / ".leann" / "projects.json"+ assert json.loads(registry_file.read_text()) == [str(project.resolve())]+++def test_project_registration_accepts_custom_max_depth(tmp_path, monkeypatch):+ home = tmp_path / "home"+ home.mkdir()+ project = tmp_path / "project"+ deep_meta = (+ project / "one" / "two" / "three" / "four" / "five" / "six" / "index.leann.meta.json"+ )+ deep_meta.parent.mkdir(parents=True)+ deep_meta.touch()+ monkeypatch.setattr(registry.Path, "home", classmethod(lambda cls: home))++ registry.register_project_directory(project, max_depth=6)++ registry_file = home / ".leann" / "projects.json"+ assert json.loads(registry_file.read_text()) == [str(project.resolve())]+++def test_registered_app_project_is_listed(tmp_path, monkeypatch, capsys):+ home = tmp_path / "home"+ home.mkdir()+ project = tmp_path / "registered-project"+ meta_file = project / "app-index" / "documents.leann.meta.json"+ meta_file.parent.mkdir(parents=True)+ meta_file.touch()+ current = tmp_path / "current-project"+ current.mkdir()+ monkeypatch.setattr(registry.Path, "home", classmethod(lambda cls: home))+ registry.register_project_directory(project)+ monkeypatch.chdir(current)++ LeannCLI().list_indexes()++ output = capsys.readouterr().out+ assert "registered-project" in output+ assert "app-index" in output+++def test_registered_app_project_respects_requested_max_depth(tmp_path, monkeypatch, capsys):+ home = tmp_path / "home"+ home.mkdir()+ project = tmp_path / "registered-project"+ meta_file = (+ project / "one" / "two" / "three" / "four" / "five" / "six" / "deep-app.leann.meta.json"+ )+ meta_file.parent.mkdir(parents=True)+ meta_file.touch()+ registry_file = home / ".leann" / "projects.json"+ registry_file.parent.mkdir()+ registry_file.write_text(json.dumps([str(project.resolve())]))+ current = tmp_path / "current-project"+ current.mkdir()+ monkeypatch.setattr(registry.Path, "home", classmethod(lambda cls: home))+ monkeypatch.chdir(current)++ LeannCLI().list_indexes(max_depth=6)++ output = capsys.readouterr().out+ assert "registered-project" in output+ assert "six" in output+++def test_registered_ancestor_does_not_hide_current_app_index(tmp_path, monkeypatch, capsys):+ home = tmp_path / "home"+ home.mkdir()+ registered_ancestor = tmp_path / "registered-ancestor"+ current = registered_ancestor / "current-project"+ meta_file = current / "current-app" / "documents.leann.meta.json"+ meta_file.parent.mkdir(parents=True)+ meta_file.touch()+ registry_file = home / ".leann" / "projects.json"+ registry_file.parent.mkdir()+ registry_file.write_text(json.dumps([str(registered_ancestor.resolve())]))+ monkeypatch.setattr(registry.Path, "home", classmethod(lambda cls: home))+ monkeypatch.chdir(current)++ LeannCLI().list_indexes()++ output = capsys.readouterr().out+ current_section = output.split("Other Projects", maxsplit=1)[0]+ assert "current-app" in current_section+ assert output.count("current-app") == 1+++def test_cli_format_index_discovery_is_not_limited_by_app_scan_depth(tmp_path):+ index_dir = tmp_path / ".leann" / "indexes" / "sample"+ index_dir.mkdir(parents=True)+ (index_dir / "documents.leann.meta.json").touch()++ indexes = LeannCLI()._discover_indexes_in_project(tmp_path, max_depth=0)++ assert [(index["name"], index["type"]) for index in indexes] == [("sample", "cli")]