[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
fix: remove conflicting index command registrations (#416)
Yifan Chenabac77dba71e
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.mdindex 6e3ba60..75df727 100644--- a/docs/CHANGELOG.md+++ b/docs/CHANGELOG.md@@ -66,3 +66,8 @@ fixes). Newest entries at the bottom.already-working Windows `cp314` installs. Completing the wheel matrix is thecorrect fix; the next 0.3.8 patch release will carry full `cp314` coverageonce CI confirms the new platforms build cleanly.++## 2026-09-06: Restore CLI parsing after duplicate index command registration++- Remove the seven duplicate index command parsers and handlers introduced by #269 after #285 had already supplied them. This fixes `argparse.ArgumentError: conflicting subparser: index-browser`, which prevented every CLI command, including help, from running.+- Retain the #285 command interface (`--max-count`, `--index-name`, `--no-recompute`), application readers, and shared builder; native indexing and standalone wheel reader packaging are unchanged.diff --git a/docs/user-scripts-tr.md b/docs/user-scripts-tr.mdindex 070ae85..6002fac 100644--- a/docs/user-scripts-tr.md+++ b/docs/user-scripts-tr.md@@ -179,7 +179,7 @@ ollama listC: Daha küçük bir dataset ile test edin:```bash---max-items 1000+--max-count 1000```## İleri Düzey Kullanımdiff --git a/docs/user-scripts.md b/docs/user-scripts.mdindex 447ac94..22f2d36 100644--- a/docs/user-scripts.md+++ b/docs/user-scripts.md@@ -177,7 +177,7 @@ ollama listTry with a smaller dataset:```bash---max-items 1000+--max-count 1000```## Advanced Usagediff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.pyindex 61b2a36..34b9f92 100644--- a/packages/leann-core/src/leann/cli.py+++ b/packages/leann-core/src/leann/cli.py@@ -486,79 +486,6 @@ Examples:),)- # Browser Index Command- browser_parser = subparsers.add_parser("index-browser", help="Index browser history")- browser_parser.add_argument(- "browser_type", choices=["chrome", "brave"], help="Type of browser"- )- browser_parser.add_argument(- "--profile", type=str, default="Default", help="Profile name (default: Default)"- )- browser_parser.add_argument(- "--index-name", type=str, default=None, help="Custom index name"- )- browser_parser.add_argument(- "--max-items", type=int, default=1000, help="Max history items to index"- )- add_embedding_args(browser_parser)-- # Email indexing command- email_parser = subparsers.add_parser("index-email", help="Index Apple Mail emails")- email_parser.add_argument(- "index_name", nargs="?", default="apple-mail", help="Index name (default: apple-mail)"- )- email_parser.add_argument(- "--max-items", type=int, default=2000, help="Max emails to index (default: 2000)"- )- add_embedding_args(email_parser)-- # Calendar indexing command- calendar_parser = subparsers.add_parser(- "index-calendar", help="Index Apple Calendar events"- )- calendar_parser.add_argument(- "index_name",- nargs="?",- default="apple-calendar",- help="Index name (default: apple-calendar)",- )- calendar_parser.add_argument(- "--max-items", type=int, default=1000, help="Max events to index (default: 1000)"- )- add_embedding_args(calendar_parser)-- # WeChat indexing command- wechat_parser = subparsers.add_parser("index-wechat", help="Index WeChat chat history")- wechat_parser.add_argument(- "index_name", nargs="?", default="wechat", help="Index name (default: wechat)"- )- wechat_parser.add_argument(- "--export-dir",- type=str,- default="./wechat_export",- help="Directory containing exported WeChat data (default: ./wechat_export)",- )- wechat_parser.add_argument(- "--max-items", type=int, default=1000, help="Max messages to index (default: 1000)"- )- add_embedding_args(wechat_parser)-- # iMessage indexing command- imessage_parser = subparsers.add_parser("index-imessage", help="Index iMessage history")- imessage_parser.add_argument(- "index_name", nargs="?", default="imessage", help="Index name (default: imessage)"- )- imessage_parser.add_argument(- "--db-path",- type=str,- default=None,- help="Path to chat.db (default: ~/Library/Messages/chat.db)",- )- imessage_parser.add_argument(- "--max-items", type=int, default=1000, help="Max messages to index (default: 1000)"- )- add_embedding_args(imessage_parser)-# Slack indexing commandslack_parser = subparsers.add_parser("index-slack", help="Index Slack workspace via MCP")slack_parser.add_argument(@@ -580,38 +507,6 @@ Examples:)add_embedding_args(slack_parser)- # ChatGPT indexing command- chatgpt_parser = subparsers.add_parser("index-chatgpt", help="Index ChatGPT export")- chatgpt_parser.add_argument(- "index_name", nargs="?", default="chatgpt", help="Index name (default: chatgpt)"- )- chatgpt_parser.add_argument(- "--export-path",- type=str,- required=True,- help="Path to ChatGPT export file (.html/.zip) or directory",- )- chatgpt_parser.add_argument(- "--max-items", type=int, default=1000, help="Max items to index (default: 1000)"- )- add_embedding_args(chatgpt_parser)-- # Claude indexing command- claude_parser = subparsers.add_parser("index-claude", help="Index Claude export")- claude_parser.add_argument(- "index_name", nargs="?", default="claude", help="Index name (default: claude)"- )- claude_parser.add_argument(- "--export-path",- type=str,- required=True,- help="Path to Claude export file (.json/.zip) or directory",- )- claude_parser.add_argument(- "--max-items", type=int, default=1000, help="Max items to index (default: 1000)"- )- add_embedding_args(claude_parser)-# Watch commandwatch_parser = subparsers.add_parser("watch",@@ -2615,220 +2510,6 @@ Examples:chunk_ids_by_file.setdefault(file_path, []).append(str(chunk_id))return chunk_ids_by_file- async def index_browser(self, args):- """Build an index from browser history."""- from .readers import ChromeHistoryReader-- browser_type = args.browser_type- profile = args.profile- index_name = args.index_name or f"{browser_type}-history"-- index_dir = self.indexes_dir / index_name- index_path = str(index_dir / "documents.leann")-- print(f"🌐 Indexing {browser_type.capitalize()} history (profile: {profile})...")-- paths = ChromeHistoryReader.find_browser_paths()- if browser_type not in paths:- print(f"❌ Could not find {browser_type} profile directory automatically.")- return-- profile_path = paths[browser_type] / profile-- reader = ChromeHistoryReader()- documents = reader.load_data(- chrome_profile_path=str(profile_path), max_count=args.max_items- )-- if not documents:- print("❌ No history entries found to index.")- return-- print(f"📚 Loaded {len(documents)} entries. Building index...")-- index_dir.mkdir(parents=True, exist_ok=True)-- embedding_options = {}- if args.embedding_mode == "ollama":- embedding_options["host"] = resolve_ollama_host(None)-- builder = LeannBuilder(- backend_name="hnsw",- embedding_model=args.embedding_model,- embedding_mode=args.embedding_mode,- embedding_options=embedding_options or None,- is_recompute=False,- is_compact=False,- )-- for doc in documents:- builder.add_text(doc.text, metadata=doc.metadata)-- builder.build_index(index_path)- print(f"✅ Browser history index built at: {index_path}")- print(f' Usage: leann search {index_name} "query"')-- async def index_email(self, args):- """Build an index from Apple Mail emails."""- from .readers import AppleMailReader-- index_name = args.index_name- index_dir = self.indexes_dir / index_name- index_path = str(index_dir / "documents.leann")-- print("📧 Indexing Apple Mail emails...")-- reader = AppleMailReader()- documents = reader.load_data(max_count=args.max_items)-- if not documents:- print("❌ No emails found to index. Make sure Full Disk Access is granted.")- return-- print(f"📚 Loaded {len(documents)} emails. Building index...")- index_dir.mkdir(parents=True, exist_ok=True)-- embedding_options = {}- if args.embedding_mode == "ollama":- embedding_options["host"] = resolve_ollama_host(None)-- builder = LeannBuilder(- backend_name="hnsw",- embedding_model=args.embedding_model,- embedding_mode=args.embedding_mode,- embedding_options=embedding_options or None,- is_recompute=False,- is_compact=False,- )-- for doc in documents:- builder.add_text(doc.text, metadata=doc.metadata)-- builder.build_index(index_path)- print(f"✅ Email index built at: {index_path}")- print(f' Usage: leann search {index_name} "query"')-- async def index_calendar(self, args):- """Build an index from Apple Calendar events."""- from .readers import AppleCalendarReader-- index_name = args.index_name- index_dir = self.indexes_dir / index_name- index_path = str(index_dir / "documents.leann")-- print("📅 Indexing Apple Calendar events...")-- reader = AppleCalendarReader()- documents = reader.load_data(max_count=args.max_items)-- if not documents:- print("❌ No calendar events found to index. Make sure Full Disk Access is granted.")- return-- print(f"📚 Loaded {len(documents)} events. Building index...")- index_dir.mkdir(parents=True, exist_ok=True)-- embedding_options = {}- if args.embedding_mode == "ollama":- embedding_options["host"] = resolve_ollama_host(None)-- builder = LeannBuilder(- backend_name="hnsw",- embedding_model=args.embedding_model,- embedding_mode=args.embedding_mode,- embedding_options=embedding_options or None,- is_recompute=False,- is_compact=False,- )-- for doc in documents:- builder.add_text(doc.text, metadata=doc.metadata)-- builder.build_index(index_path)- print(f"✅ Calendar index built at: {index_path}")- print(f' Usage: leann search {index_name} "query"')-- async def index_wechat(self, args):- """Build an index from WeChat chat history."""- from .readers import WeChatHistoryReader-- index_name = args.index_name- index_dir = self.indexes_dir / index_name- index_path = str(index_dir / "documents.leann")-- print("💬 Indexing WeChat chat history...")-- reader = WeChatHistoryReader()- export_dir = getattr(args, "export_dir", "./wechat_export")- documents = reader.load_data(wechat_export_dir=export_dir, max_count=args.max_items)-- if not documents:- print("❌ No WeChat data found. Make sure WeChat is exported first.")- return-- print(f"📚 Loaded {len(documents)} messages. Building index...")- index_dir.mkdir(parents=True, exist_ok=True)-- embedding_options = {}- if args.embedding_mode == "ollama":- embedding_options["host"] = resolve_ollama_host(None)-- builder = LeannBuilder(- backend_name="hnsw",- embedding_model=args.embedding_model,- embedding_mode=args.embedding_mode,- embedding_options=embedding_options or None,- is_recompute=False,- is_compact=False,- )-- for doc in documents:- builder.add_text(doc.text, metadata=doc.metadata)-- builder.build_index(index_path)- print(f"✅ WeChat index built at: {index_path}")- print(f' Usage: leann search {index_name} "query"')-- async def index_imessage(self, args):- """Build an index from iMessage history."""- from .readers import IMessageReader-- index_name = args.index_name- index_dir = self.indexes_dir / index_name- index_path = str(index_dir / "documents.leann")-- print("💬 Indexing iMessage history...")-- reader = IMessageReader()- documents = reader.load_data(max_count=args.max_items)-- if not documents:- print("❌ No iMessage data found. Make sure Full Disk Access is granted.")- return-- print(f"📚 Loaded {len(documents)} messages. Building index...")- index_dir.mkdir(parents=True, exist_ok=True)-- embedding_options = {}- if args.embedding_mode == "ollama":- embedding_options["host"] = resolve_ollama_host(None)-- builder = LeannBuilder(- backend_name="hnsw",- embedding_model=args.embedding_model,- embedding_mode=args.embedding_mode,- embedding_options=embedding_options or None,- is_recompute=False,- is_compact=False,- )-- for doc in documents:- builder.add_text(doc.text, metadata=doc.metadata)-- builder.build_index(index_path)- print(f"✅ iMessage index built at: {index_path}")- print(f' Usage: leann search {index_name} "query"')-async def index_slack(self, args):"""Build an index from Slack workspace via MCP."""from .readers import SlackMCPReader@@ -2872,86 +2553,6 @@ Examples:print(f"✅ Slack index built at: {index_path}")print(f' Usage: leann search {index_name} "query"')- async def index_chatgpt(self, args):- """Build an index from ChatGPT export."""- from .readers import ChatGPTReader-- index_name = args.index_name- index_dir = self.indexes_dir / index_name- index_path = str(index_dir / "documents.leann")-- print("🤖 Indexing ChatGPT export...")-- reader = ChatGPTReader()- documents = reader.load_data(export_path=args.export_path)-- if not documents:- print("❌ No ChatGPT data found. Make sure export file is valid.")- return-- print(f"📚 Loaded {len(documents)} items. Building index...")- index_dir.mkdir(parents=True, exist_ok=True)-- embedding_options = {}- if args.embedding_mode == "ollama":- embedding_options["host"] = resolve_ollama_host(None)-- builder = LeannBuilder(- backend_name="hnsw",- embedding_model=args.embedding_model,- embedding_mode=args.embedding_mode,- embedding_options=embedding_options or None,- is_recompute=False,- is_compact=False,- )-- for doc in documents:- builder.add_text(doc.text, metadata=doc.metadata)-- builder.build_index(index_path)- print(f"✅ ChatGPT index built at: {index_path}")- print(f' Usage: leann search {index_name} "query"')-- async def index_claude(self, args):- """Build an index from Claude export."""- from .readers import ClaudeReader-- index_name = args.index_name- index_dir = self.indexes_dir / index_name- index_path = str(index_dir / "documents.leann")-- print("🤖 Indexing Claude export...")-- reader = ClaudeReader()- documents = reader.load_data(export_path=args.export_path)-- if not documents:- print("❌ No Claude data found. Make sure export file is valid.")- return-- print(f"📚 Loaded {len(documents)} items. Building index...")- index_dir.mkdir(parents=True, exist_ok=True)-- embedding_options = {}- if args.embedding_mode == "ollama":- embedding_options["host"] = resolve_ollama_host(None)-- builder = LeannBuilder(- backend_name="hnsw",- embedding_model=args.embedding_model,- embedding_mode=args.embedding_mode,- embedding_options=embedding_options or None,- is_recompute=False,- is_compact=False,- )-- for doc in documents:- builder.add_text(doc.text, metadata=doc.metadata)-- builder.build_index(index_path)- print(f"✅ Claude index built at: {index_path}")- print(f' Usage: leann search {index_name} "query"')-async def build_index(self, args):docs_paths = args.docs# Use current directory name if index_name not provideddiff --git a/tests/test_cli_index_commands.py b/tests/test_cli_index_commands.pynew file mode 100644index 0000000..d92d24c--- /dev/null+++ b/tests/test_cli_index_commands.py@@ -0,0 +1,199 @@+"""Exercise index command parsing and dispatch without accessing personal data."""++import asyncio+import os+import shutil+import sqlite3+import sys+from pathlib import Path+from types import ModuleType+from unittest.mock import Mock, call++import pytest+from leann import cli as cli_module+from leann.cli import LeannCLI+from llama_index.core import Document++SOURCES = {+ "browser": ("apps.history_data.history", "ChromeHistoryReader", "browser_history"),+ "email": ("apps.email_data.LEANN_email_reader", "EmlxReader", "email"),+ "calendar": (None, None, "calendar"),+ "imessage": ("apps.imessage_data.imessage_reader", "IMessageReader", "imessage"),+ "wechat": ("apps.history_data.wechat_history", "WeChatHistoryReader", "wechat"),+ "chatgpt": ("apps.chatgpt_data.chatgpt_reader", "ChatGPTReader", "chatgpt"),+ "claude": ("apps.claude_data.claude_reader", "ClaudeReader", "claude"),+}+++def cli(tmp_path, monkeypatch):+ monkeypatch.chdir(tmp_path)+ monkeypatch.setattr(Path, "home", classmethod(lambda cls: tmp_path))+ monkeypatch.setattr(+ os.path,+ "expanduser",+ lambda path: str(tmp_path / path[2:]) if path.startswith("~/") else path,+ )+ return LeannCLI()+++def command_args(source, tmp_path):+ argv = [f"index-{source}"]+ if source == "browser":+ argv.append("brave")+ elif source == "wechat":+ argv.extend(["--export-dir", str(tmp_path / "export")])+ elif source in {"chatgpt", "claude"}:+ argv.extend(["--export-path", str(tmp_path / "export.json")])+ return argv++[email protected]("source", SOURCES)+def test_index_command_defaults(cli, source, tmp_path):+ args = cli.create_parser().parse_args(command_args(source, tmp_path))+ assert args.index_name == SOURCES[source][2]+ assert args.max_count == 1000+ assert not args.no_recompute++[email protected]("source", SOURCES)[email protected]("empty", [False, True], ids=["documents", "empty"])[email protected]("no_recompute", [False, True], ids=["recompute", "stored"])+def test_index_command_dispatch(cli, source, empty, no_recompute, tmp_path, monkeypatch):+ # Install fake modules before dispatch: no application reader is imported or run.+ documents = [] if empty else [Document(text="Synthetic text", metadata={"source": source})]+ module_name, reader_name, _ = SOURCES[source]+ reader_class = Mock()+ reader_class.return_value.load_data.return_value = documents+ if module_name:+ assert reader_name is not None+ parts = module_name.split(".")+ for end in range(1, len(parts) + 1):+ name = ".".join(parts[:end])+ module = ModuleType(name)+ module.__path__ = []+ monkeypatch.setitem(sys.modules, name, module)+ module = sys.modules[module_name]+ setattr(module, reader_name, reader_class)+ if source == "email":+ monkeypatch.setattr(+ module,+ "find_all_messages_directories",+ Mock(return_value=[tmp_path / "mail"]),+ raising=False,+ )+ else:+ # The legacy calendar handler embeds its reader. Exercise its actual SQL+ # against a synthetic database, redirecting its fixed scratch path.+ calendar_cache = tmp_path / "Library" / "Calendars" / "Calendar Cache"+ calendar_cache.parent.mkdir(parents=True)+ connection = sqlite3.connect(calendar_cache)+ connection.execute(+ "CREATE TABLE CI_EVENT (summary, description, location, start_date, end_date)"+ )+ if not empty:+ connection.executemany(+ "INSERT INTO CI_EVENT VALUES (?, ?, ?, ?, ?)",+ [(f"Synthetic event {i}", "Details", "Room", i, i + 1) for i in range(3)],+ )+ connection.commit()+ monkeypatch.setattr(shutil, "copy2", Mock())+ monkeypatch.setattr(sqlite3, "connect", Mock(return_value=connection))+ original_exists = os.path.exists+ monkeypatch.setattr(+ os.path,+ "exists",+ lambda path: False+ if path == "/tmp/leann_calendar_index_copy"+ else original_exists(path),+ )++ builder_class = Mock()+ register = Mock()+ monkeypatch.setattr(cli_module, "LeannBuilder", builder_class)+ monkeypatch.setattr(cli, "register_project_dir", register)+ argv = [+ *command_args(source, tmp_path),+ "--index-name",+ "synthetic-index",+ "--max-count",+ "2",+ "--embedding-model",+ "synthetic-model",+ "--embedding-mode",+ "ollama",+ "--embedding-host",+ "http://127.0.0.1:9999",+ ]+ if no_recompute:+ argv.append("--no-recompute")+ args = cli.create_parser().parse_args(argv)+ asyncio.run(cli.run(args))++ if source == "browser":+ reader_class.return_value.load_data.assert_called_once_with(+ chrome_profile_path=str(+ tmp_path / "Library/Application Support/BraveSoftware/Brave-Browser/Default"+ ),+ max_count=2,+ )+ elif source == "email":+ reader_class.return_value.load_data.assert_called_once_with(+ str(tmp_path / "mail"), max_count=2+ )+ elif source == "imessage":+ reader_class.assert_called_once_with(concatenate_conversations=True)+ reader_class.return_value.load_data.assert_called_once_with()+ elif source == "wechat":+ reader_class.return_value.load_data.assert_called_once_with(+ input_dir=str(tmp_path / "export"),+ max_count=2,+ concatenate_messages=True,+ )+ elif source in {"chatgpt", "claude"}:+ reader_class.assert_called_once_with(concatenate_conversations=True)+ reader_class.return_value.load_data.assert_called_once_with(+ input_dir=str(tmp_path / "export.json"),+ max_count=2,+ )++ if empty:+ builder_class.assert_not_called()+ register.assert_not_called()+ return++ builder_class.assert_called_once_with(+ backend_name="hnsw",+ embedding_model="synthetic-model",+ embedding_mode="ollama",+ embedding_options={"host": "http://127.0.0.1:9999"},+ is_recompute=not no_recompute,+ )+ builder = builder_class.return_value+ if source == "calendar":+ assert builder.add_text.call_count == 2+ assert "Synthetic event 2" in builder.add_text.call_args_list[0].args[0]+ assert "Synthetic event 1" in builder.add_text.call_args_list[1].args[0]+ else:+ assert builder.add_text.call_args_list == [+ call(doc.text, metadata=doc.metadata) for doc in documents+ ]+ builder.build_index.assert_called_once_with(cli.get_index_path("synthetic-index"))+ register.assert_called_once_with()+++def test_top_level_help(cli, capsys):+ with pytest.raises(SystemExit) as exc:+ cli.create_parser().parse_args(["--help"])+ assert exc.value.code == 0+ output = capsys.readouterr().out+ for source in SOURCES:+ assert f"index-{source}" in output+++def test_unrelated_list_dispatch(cli, monkeypatch):+ list_indexes = Mock()+ monkeypatch.setattr(cli, "list_indexes", list_indexes)+ args = cli.create_parser().parse_args(["list", "--max-depth", "2"])+ asyncio.run(cli.run(args))+ list_indexes.assert_called_once_with(2)