Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

fix(cli): bound index discovery scans (#389)

Barry2llena60417b3bef5
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 2e4d5f2..471927a 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -41,18 +41,8 @@ fixes). Newest entries at the bottom.
comparable recall (GPU latency stays ~flat while CPU grows linearly with nprobe).
- Docs: `docs/flashlib_backend_guide.md` gains a `flashlib_ivf` section.
-## 2026-07-24: `leann-core` 0.3.8 — HNSW full-rebuild fallback corpus wipe (#386)
-
-- Fixed a bug where, on the HNSW backend, `leann build` / `leann watch` falling
- back to a full rebuild (any modified or removed file, since HNSW only
- supports add-only incremental updates) rebuilt the index from only the
- changed files instead of the full corpus, silently dropping every untouched
- file. Root cause: the fallback reused `all_texts` from the incremental path
- (which intentionally loads only the delta) instead of reloading the full
- corpus via `load_documents(docs_paths, ...)`.
-- This was fixed on `main` in commit `956beb5` (merged via #279) but never
- reached PyPI — `leann-core` 0.3.7 was still affected. Bumped `leann-core` to
- 0.3.8 to ship the fix.
-- Added `tests/test_hnsw_rebuild_fallback.py`: regression test that builds an
- HNSW index, modifies one file, and asserts untouched files are still present
- in the rebuilt corpus.
+## 2026-07-27: Bounded `leann list` index discovery
+
+- Bound App-format metadata discovery to a configurable directory depth; `leann list` defaults to depth 5 and accepts `--max-depth` for deeper projects.
+- Prune common dependency, virtual-environment, cache, and system directories during discovery.
+- Preserve discovery of registered CLI-format and App-format projects without recursively scanning an entire home directory.
diff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.py
index 967342c..7b6adf3 100644
--- a/packages/leann-core/src/leann/cli.py
+++ b/packages/leann-core/src/leann/cli.py
@@ -19,7 +19,7 @@ from tqdm import tqdm
from .api import Fts5BM25Index, LeannBuilder, LeannChat, LeannSearcher
from .embedding_server_manager import EmbeddingServerManager
from .interactive_utils import create_cli_session
-from .registry import register_project_directory
+from .registry import DEFAULT_INDEX_SCAN_DEPTH, iter_index_meta_files, register_project_directory
from .settings import (
resolve_anthropic_base_url,
resolve_atlascloud_api_key,
@@ -33,6 +33,13 @@ from .settings import (
from .sync import DEFAULT_INDEX_EXTENSIONS, FileSynchronizer, parse_include_extensions
+def _non_negative_int(value: str) -> int:
+ parsed_value = int(value)
+ if parsed_value < 0:
+ raise argparse.ArgumentTypeError("must be non-negative")
+ return parsed_value
+
+
def _default_embedding_model() -> str:
"""Pick a sensible default embedding model based on platform.
@@ -856,7 +863,16 @@ Examples:
)
# List command
- subparsers.add_parser("list", help="List all indexes")
+ list_parser = subparsers.add_parser("list", help="List all indexes")
+ list_parser.add_argument(
+ "--max-depth",
+ type=_non_negative_int,
+ default=DEFAULT_INDEX_SCAN_DEPTH,
+ help=(
+ "Maximum directory depth for app index discovery "
+ f"(default: {DEFAULT_INDEX_SCAN_DEPTH})"
+ ),
+ )
# Remove command
remove_parser = subparsers.add_parser("remove", help="Remove an index")
@@ -947,7 +963,24 @@ Examples:
# If anything goes wrong, assume it's not a submodule
return False
- def list_indexes(self):
+ @staticmethod
+ def _project_has_discoverable_indexes(project_path: Path, max_depth: int) -> bool:
+ """Return whether a project contains an index within the scan boundary."""
+ cli_indexes_dir = project_path / ".leann" / "indexes"
+ return cli_indexes_dir.exists() or any(
+ iter_index_meta_files(project_path, max_depth=max_depth)
+ )
+
+ @staticmethod
+ def _nested_project_dirs(project_path: Path, projects: list[Path]) -> list[Path]:
+ """Return registered project roots nested below a project."""
+ return [
+ candidate
+ for candidate in projects
+ if candidate != project_path and candidate.is_relative_to(project_path)
+ ]
+
+ def list_indexes(self, max_depth: int = DEFAULT_INDEX_SCAN_DEPTH):
# Get all project directories with .leann
global_registry = Path.home() / ".leann" / "projects.json"
all_projects = []
@@ -961,16 +994,21 @@ Examples:
except Exception:
pass
- # Filter to only existing directories with .leann
+ # Filter to existing projects with a discoverable CLI- or App-format index.
valid_projects = []
for project_dir in all_projects:
project_path = Path(project_dir)
- if project_path.exists() and (project_path / ".leann" / "indexes").exists():
+ if project_path.exists() and self._project_has_discoverable_indexes(
+ project_path, max_depth
+ ):
valid_projects.append(project_path)
- # Add current project if it has .leann but not in registry
+ # Add current project if it has a discoverable index but is not in registry.
current_path = Path.cwd()
- if (current_path / ".leann" / "indexes").exists() and current_path not in valid_projects:
+ if (
+ self._project_has_discoverable_indexes(current_path, max_depth)
+ and current_path not in valid_projects
+ ):
valid_projects.append(current_path)
# Separate current and other projects
@@ -980,6 +1018,10 @@ Examples:
if project_path != current_path:
other_projects.append(project_path)
+ # Exclude only registered projects nested below the current project.
+ # Registered ancestors must not hide indexes owned by the current directory.
+ current_exclude_dirs = self._nested_project_dirs(current_path, valid_projects)
+
print("📚 LEANN Indexes")
print("=" * 50)
@@ -992,7 +1034,7 @@ Examples:
print(" " + "─" * 45)
current_indexes = self._discover_indexes_in_project(
- current_path, exclude_dirs=other_projects
+ current_path, exclude_dirs=current_exclude_dirs, max_depth=max_depth
)
if current_indexes:
for idx in current_indexes:
@@ -1011,7 +1053,10 @@ Examples:
print(" " + "─" * 45)
for project_path in other_projects:
- project_indexes = self._discover_indexes_in_project(project_path)
+ nested_projects = self._nested_project_dirs(project_path, valid_projects)
+ project_indexes = self._discover_indexes_in_project(
+ project_path, exclude_dirs=nested_projects, max_depth=max_depth
+ )
if not project_indexes:
continue
@@ -1035,9 +1080,14 @@ Examples:
projects_count = 0
for p in valid_projects:
if p == current_path:
- discovered = self._discover_indexes_in_project(p, exclude_dirs=other_projects)
+ discovered = self._discover_indexes_in_project(
+ p, exclude_dirs=current_exclude_dirs, max_depth=max_depth
+ )
else:
- discovered = self._discover_indexes_in_project(p)
+ nested_projects = self._nested_project_dirs(p, valid_projects)
+ discovered = self._discover_indexes_in_project(
+ p, exclude_dirs=nested_projects, max_depth=max_depth
+ )
if len(discovered) > 0:
projects_count += 1
print(f"📊 Total: {total_indexes} indexes across {projects_count} projects")
@@ -1057,13 +1107,18 @@ Examples:
print(" leann build my-docs --docs ./documents")
def _discover_indexes_in_project(
- self, project_path: Path, exclude_dirs: Optional[list[Path]] = None
+ self,
+ project_path: Path,
+ exclude_dirs: Optional[list[Path]] = None,
+ max_depth: Optional[int] = None,
):
"""Discover all indexes in a project directory (both CLI and apps formats)
exclude_dirs: when provided, skip any APP-format index files that are
located under these directories. This prevents duplicates when the
current project is a parent directory of other registered projects.
+ max_depth: maximum number of directories to descend for APP-format indexes.
+ None preserves full-depth discovery for non-CLI callers.
"""
indexes = []
exclude_dirs = exclude_dirs or []
@@ -1100,9 +1155,9 @@ Examples:
}
)
- # 2. Apps format: *.leann.meta.json files anywhere in the project
+ # 2. Apps format: *.leann.meta.json files within the configured scan depth
cli_indexes_dir = project_path / ".leann" / "indexes"
- for meta_file in project_path.rglob("*.leann.meta.json"):
+ for meta_file in iter_index_meta_files(project_path, max_depth=max_depth):
if meta_file.is_file():
# Skip CLI-built indexes (which store meta under .leann/indexes/<name>/)
try:
@@ -3732,7 +3787,7 @@ Examples:
suppress = not getattr(args, "verbose", False)
if args.command == "list":
- self.list_indexes()
+ self.list_indexes(args.max_depth)
elif args.command == "remove":
self.remove_index(args.index_name, args.force)
elif args.command == "build":
diff --git a/packages/leann-core/src/leann/registry.py b/packages/leann-core/src/leann/registry.py
index 2b74a20..b950ee6 100644
--- a/packages/leann-core/src/leann/registry.py
+++ b/packages/leann-core/src/leann/registry.py
@@ -4,6 +4,8 @@ import importlib
import importlib.metadata
import json
import logging
+import os
+from collections.abc import Iterator
from pathlib import Path
from typing import TYPE_CHECKING, Optional, Union
@@ -14,6 +16,35 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
BACKEND_REGISTRY: dict[str, "LeannBackendFactoryInterface"] = {}
+DEFAULT_INDEX_SCAN_DEPTH = 5
+INDEX_SCAN_SKIP_DIRS = frozenset(
+ {".git", ".venv", "venv", "node_modules", "__pycache__", "Library"}
+)
+
+
+def iter_index_meta_files(
+ root: Union[str, Path], max_depth: Optional[int] = DEFAULT_INDEX_SCAN_DEPTH
+) -> Iterator[Path]:
+ """Yield LEANN metadata files within an optionally bounded directory tree.
+
+ The root directory is depth zero. A max_depth of None scans all depths.
+ Known dependency, cache, and system directories are always pruned before traversal.
+ """
+ if max_depth is not None and max_depth < 0:
+ raise ValueError("max_depth must be non-negative")
+
+ root_path = Path(root)
+ for current_dir, dirnames, filenames in os.walk(root_path):
+ current_path = Path(current_dir)
+ depth = len(current_path.relative_to(root_path).parts)
+ if max_depth is not None and depth >= max_depth:
+ dirnames.clear()
+ else:
+ dirnames[:] = [name for name in dirnames if name not in INDEX_SCAN_SKIP_DIRS]
+
+ for filename in filenames:
+ if filename.endswith(".leann.meta.json"):
+ yield current_path / filename
def register_backend(name: str):
@@ -49,7 +80,10 @@ def autodiscover_backends():
# print("INFO: Backend auto-discovery finished.")
-def register_project_directory(project_dir: Optional[Union[str, Path]] = None):
+def register_project_directory(
+ project_dir: Optional[Union[str, Path]] = None,
+ max_depth: Optional[int] = None,
+):
"""
Register a project directory in the global LEANN registry.
@@ -57,6 +91,8 @@ def register_project_directory(project_dir: Optional[Union[str, Path]] = None):
Args:
project_dir: Directory to register. If None, uses current working directory.
+ max_depth: Maximum directory depth used when looking for App-format indexes.
+ None preserves full-depth discovery for existing API callers.
"""
if project_dir is None:
project_dir = Path.cwd()
@@ -64,9 +100,9 @@ def register_project_directory(project_dir: Optional[Union[str, Path]] = None):
project_dir = Path(project_dir)
# Only register directories that have some kind of LEANN content.
- # Check CLI-format first to avoid an expensive rglob on large directories.
+ # Check CLI-format first to avoid even a bounded scan when it is unnecessary.
has_cli_indexes = (project_dir / ".leann" / "indexes").exists()
- if not has_cli_indexes and not any(project_dir.rglob("*.leann.meta.json")):
+ if not has_cli_indexes and not any(iter_index_meta_files(project_dir, max_depth=max_depth)):
# Don't register if there are no LEANN indexes
return
diff --git a/tests/test_cli_list_performance.py b/tests/test_cli_list_performance.py
new file mode 100644
index 0000000..de93fab
--- /dev/null
+++ b/tests/test_cli_list_performance.py
@@ -0,0 +1,220 @@
+import asyncio
+import json
+import sys
+
+import pytest
+from leann import registry
+from leann.cli import LeannCLI
+from leann.registry import iter_index_meta_files
+
+
+def test_index_discovery_finds_metadata_within_default_depth(tmp_path):
+ meta_file = tmp_path / "project" / "data" / "sample.leann.meta.json"
+ meta_file.parent.mkdir(parents=True)
+ meta_file.touch()
+
+ assert list(iter_index_meta_files(tmp_path)) == [meta_file]
+
+
+def test_index_discovery_does_not_descend_past_max_depth(tmp_path):
+ shallow_meta = tmp_path / "one" / "shallow.leann.meta.json"
+ shallow_meta.parent.mkdir()
+ shallow_meta.touch()
+ deep_meta = tmp_path / "one" / "two" / "deep.leann.meta.json"
+ deep_meta.parent.mkdir()
+ deep_meta.touch()
+
+ assert list(iter_index_meta_files(tmp_path, max_depth=1)) == [shallow_meta]
+
+
[email protected]("excluded_dir", [".git", ".venv", "node_modules", "Library"])
+def test_index_discovery_skips_large_irrelevant_directories(tmp_path, excluded_dir):
+ hidden_meta = tmp_path / excluded_dir / "hidden.leann.meta.json"
+ hidden_meta.parent.mkdir()
+ hidden_meta.touch()
+
+ assert list(iter_index_meta_files(tmp_path)) == []
+
+
+def test_index_discovery_rejects_negative_max_depth(tmp_path):
+ with pytest.raises(ValueError, match="max_depth must be non-negative"):
+ list(iter_index_meta_files(tmp_path, max_depth=-1))
+
+
+def test_list_parser_accepts_custom_max_depth():
+ args = LeannCLI().create_parser().parse_args(["list", "--max-depth", "5"])
+
+ assert args.max_depth == 5
+
+
+def test_list_parser_rejects_negative_max_depth(capsys):
+ with pytest.raises(SystemExit):
+ LeannCLI().create_parser().parse_args(["list", "--max-depth", "-1"])
+
+ assert "must be non-negative" in capsys.readouterr().err
+
+
+def test_cli_project_discovery_respects_max_depth(tmp_path):
+ shallow_meta = tmp_path / "shallow" / "index.leann.meta.json"
+ shallow_meta.parent.mkdir()
+ shallow_meta.touch()
+ deep_meta = tmp_path / "one" / "two" / "index.leann.meta.json"
+ deep_meta.parent.mkdir(parents=True)
+ deep_meta.touch()
+
+ indexes = LeannCLI()._discover_indexes_in_project(tmp_path, max_depth=1)
+
+ assert [index["name"] for index in indexes] == ["shallow"]
+
+
+def test_cli_project_discovery_without_max_depth_finds_deep_app_index(tmp_path):
+ deep_meta = (
+ tmp_path / "one" / "two" / "three" / "four" / "five" / "six" / "index.leann.meta.json"
+ )
+ deep_meta.parent.mkdir(parents=True)
+ deep_meta.touch()
+
+ indexes = LeannCLI()._discover_indexes_in_project(tmp_path)
+
+ assert [index["name"] for index in indexes] == ["six"]
+
+
+def test_list_command_passes_max_depth_to_discovery(monkeypatch):
+ cli = LeannCLI()
+ received = {}
+
+ def capture_max_depth(max_depth):
+ received["max_depth"] = max_depth
+
+ monkeypatch.setattr(cli, "list_indexes", capture_max_depth)
+ monkeypatch.setattr(sys, "argv", ["leann", "list", "--max-depth", "7"])
+
+ asyncio.run(cli.run())
+
+ assert received == {"max_depth": 7}
+
+
+def test_project_registration_respects_explicit_max_depth(tmp_path, monkeypatch):
+ home = tmp_path / "home"
+ home.mkdir()
+ project = tmp_path / "project"
+ deep_meta = (
+ project / "one" / "two" / "three" / "four" / "five" / "six" / "index.leann.meta.json"
+ )
+ deep_meta.parent.mkdir(parents=True)
+ deep_meta.touch()
+ monkeypatch.setattr(registry.Path, "home", classmethod(lambda cls: home))
+
+ registry.register_project_directory(project, max_depth=registry.DEFAULT_INDEX_SCAN_DEPTH)
+
+ assert not (home / ".leann" / "projects.json").exists()
+
+
+def test_project_registration_without_max_depth_finds_deep_app_index(tmp_path, monkeypatch):
+ home = tmp_path / "home"
+ home.mkdir()
+ project = tmp_path / "project"
+ deep_meta = (
+ project / "one" / "two" / "three" / "four" / "five" / "six" / "index.leann.meta.json"
+ )
+ deep_meta.parent.mkdir(parents=True)
+ deep_meta.touch()
+ monkeypatch.setattr(registry.Path, "home", classmethod(lambda cls: home))
+
+ registry.register_project_directory(project)
+
+ registry_file = home / ".leann" / "projects.json"
+ assert json.loads(registry_file.read_text()) == [str(project.resolve())]
+
+
+def test_project_registration_accepts_custom_max_depth(tmp_path, monkeypatch):
+ home = tmp_path / "home"
+ home.mkdir()
+ project = tmp_path / "project"
+ deep_meta = (
+ project / "one" / "two" / "three" / "four" / "five" / "six" / "index.leann.meta.json"
+ )
+ deep_meta.parent.mkdir(parents=True)
+ deep_meta.touch()
+ monkeypatch.setattr(registry.Path, "home", classmethod(lambda cls: home))
+
+ registry.register_project_directory(project, max_depth=6)
+
+ registry_file = home / ".leann" / "projects.json"
+ assert json.loads(registry_file.read_text()) == [str(project.resolve())]
+
+
+def test_registered_app_project_is_listed(tmp_path, monkeypatch, capsys):
+ home = tmp_path / "home"
+ home.mkdir()
+ project = tmp_path / "registered-project"
+ meta_file = project / "app-index" / "documents.leann.meta.json"
+ meta_file.parent.mkdir(parents=True)
+ meta_file.touch()
+ current = tmp_path / "current-project"
+ current.mkdir()
+ monkeypatch.setattr(registry.Path, "home", classmethod(lambda cls: home))
+ registry.register_project_directory(project)
+ monkeypatch.chdir(current)
+
+ LeannCLI().list_indexes()
+
+ output = capsys.readouterr().out
+ assert "registered-project" in output
+ assert "app-index" in output
+
+
+def test_registered_app_project_respects_requested_max_depth(tmp_path, monkeypatch, capsys):
+ home = tmp_path / "home"
+ home.mkdir()
+ project = tmp_path / "registered-project"
+ meta_file = (
+ project / "one" / "two" / "three" / "four" / "five" / "six" / "deep-app.leann.meta.json"
+ )
+ meta_file.parent.mkdir(parents=True)
+ meta_file.touch()
+ registry_file = home / ".leann" / "projects.json"
+ registry_file.parent.mkdir()
+ registry_file.write_text(json.dumps([str(project.resolve())]))
+ current = tmp_path / "current-project"
+ current.mkdir()
+ monkeypatch.setattr(registry.Path, "home", classmethod(lambda cls: home))
+ monkeypatch.chdir(current)
+
+ LeannCLI().list_indexes(max_depth=6)
+
+ output = capsys.readouterr().out
+ assert "registered-project" in output
+ assert "six" in output
+
+
+def test_registered_ancestor_does_not_hide_current_app_index(tmp_path, monkeypatch, capsys):
+ home = tmp_path / "home"
+ home.mkdir()
+ registered_ancestor = tmp_path / "registered-ancestor"
+ current = registered_ancestor / "current-project"
+ meta_file = current / "current-app" / "documents.leann.meta.json"
+ meta_file.parent.mkdir(parents=True)
+ meta_file.touch()
+ registry_file = home / ".leann" / "projects.json"
+ registry_file.parent.mkdir()
+ registry_file.write_text(json.dumps([str(registered_ancestor.resolve())]))
+ monkeypatch.setattr(registry.Path, "home", classmethod(lambda cls: home))
+ monkeypatch.chdir(current)
+
+ LeannCLI().list_indexes()
+
+ output = capsys.readouterr().out
+ current_section = output.split("Other Projects", maxsplit=1)[0]
+ assert "current-app" in current_section
+ assert output.count("current-app") == 1
+
+
+def test_cli_format_index_discovery_is_not_limited_by_app_scan_depth(tmp_path):
+ index_dir = tmp_path / ".leann" / "indexes" / "sample"
+ index_dir.mkdir(parents=True)
+ (index_dir / "documents.leann.meta.json").touch()
+
+ indexes = LeannCLI()._discover_indexes_in_project(tmp_path, max_depth=0)
+
+ assert [(index["name"], index["type"]) for index in indexes] == [("sample", "cli")]