[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
import argparse
import asyncio
import contextlib
import hashlib
import io
import json
import os
import pickle
import re
import sys
import time
import uuid
from pathlib import Path
from typing import Any, Optional, Union
from llama_index.core import SimpleDirectoryReader
from llama_index.core.node_parser import SentenceSplitter
from tqdm import tqdm
from .api import Fts5BM25Index, LeannBuilder, LeannChat, LeannSearcher
from .embedding_server_manager import EmbeddingServerManager
from .interactive_utils import create_cli_session
from .registry import DEFAULT_INDEX_SCAN_DEPTH, iter_index_meta_files, register_project_directory
from .settings import (
resolve_anthropic_base_url,
resolve_atlascloud_api_key,
resolve_atlascloud_base_url,
resolve_litellm_api_key,
resolve_litellm_base_url,
resolve_minimax_api_key,
resolve_minimax_base_url,
resolve_ollama_host,
resolve_openai_api_key,
resolve_openai_base_url,
)
from .sync import DEFAULT_INDEX_EXTENSIONS, FileSynchronizer, parse_include_extensions
def _non_negative_int(value: str) -> int:
parsed_value = int(value)
if parsed_value < 0:
raise argparse.ArgumentTypeError("must be non-negative")
return parsed_value
def _default_embedding_model() -> str:
"""Pick a sensible default embedding model based on platform.
| Platform | Default model |
|------------|------------------------------------------------|
| NVIDIA GPU | BAAI/bge-base-en-v1.5 |
| macOS | sentence-transformers/all-MiniLM-L6-v2 |
| Other/CPU | sentence-transformers/all-MiniLM-L6-v2 |
"""
try:
import torch
if torch.cuda.is_available():
return "BAAI/bge-base-en-v1.5"
except ImportError:
pass
# macOS (MPS or CPU) and all other platforms: lightweight model
return "sentence-transformers/all-MiniLM-L6-v2"
def _normalize_path(path: str) -> str:
"""Return absolute path string for consistent keys."""
if not path:
return path
return str(Path(path).resolve())
def _cleanup_path(path: Path) -> None:
if path.is_dir():
import shutil
shutil.rmtree(path)
elif path.exists():
path.unlink()
def _existing_index_artifacts(index_dir: Path) -> bool:
return (
(index_dir / "documents.leann.meta.json").exists()
and (index_dir / "documents.leann.passages.jsonl").exists()
and (index_dir / "documents.leann.passages.idx").exists()
)
def _publish_rebuilt_index(staging_dir: Path, index_dir: Path) -> None:
"""Publish a staged full rebuild, restoring the previous directory if publish fails."""
backup_dir = index_dir.with_name(f".{index_dir.name}.backup-{uuid.uuid4().hex}")
live_moved = False
published = False
try:
if index_dir.exists():
os.replace(index_dir, backup_dir)
live_moved = True
os.replace(staging_dir, index_dir)
published = True
except Exception:
if published:
_cleanup_path(index_dir)
if live_moved and backup_dir.exists():
os.replace(backup_dir, index_dir)
raise
else:
if backup_dir.exists():
_cleanup_path(backup_dir)
@contextlib.contextmanager
def suppress_cpp_output(suppress: bool = True):
"""Context manager to suppress C++ stdout/stderr output from FAISS/HNSW
while preserving Python print() output.
C++ native code writes directly to OS file descriptors (fd 1 / fd 2).
Python print() goes through sys.stdout / sys.stderr, which are Python
file objects. We redirect the OS fds to /dev/null (silencing C++) but
point sys.stdout / sys.stderr at copies of the *original* fds so that
Python output still reaches the terminal.
"""
if not suppress:
yield
return
# 1. Duplicate the original OS file descriptors.
# May fail in no-console environments (e.g. pythonw, Windows GUI apps)
# where fd 1/2 are not valid — in that case, skip suppression.
try:
saved_stdout_fd = os.dup(1)
saved_stderr_fd = os.dup(2)
except OSError:
yield
return
# 2. Build Python file objects that write to the saved (real) fds.
# closefd=False so closing these wrappers won't close the duped fds.
py_stdout = io.TextIOWrapper(
io.FileIO(saved_stdout_fd, mode="w", closefd=False), encoding=sys.stdout.encoding or "utf-8"
)
py_stderr = io.TextIOWrapper(
io.FileIO(saved_stderr_fd, mode="w", closefd=False), encoding=sys.stderr.encoding or "utf-8"
)
old_sys_stdout = sys.stdout
old_sys_stderr = sys.stderr
try:
# 3. Redirect OS-level fds to /dev/null → silences C++ output
devnull = os.open(os.devnull, os.O_WRONLY)
os.dup2(devnull, 1)
os.dup2(devnull, 2)
os.close(devnull)
# 4. Point Python's sys.stdout/stderr at the real terminal
sys.stdout = py_stdout
sys.stderr = py_stderr
yield
finally:
# 5. Restore everything
# Flush wrappers first (they still need the saved fds to be open)
py_stdout.flush()
py_stderr.flush()
sys.stdout = old_sys_stdout
sys.stderr = old_sys_stderr
os.dup2(saved_stdout_fd, 1)
os.dup2(saved_stderr_fd, 2)
os.close(saved_stdout_fd)
os.close(saved_stderr_fd)
def extract_pdf_text_with_pymupdf(file_path: str) -> str | None:
"""Extract text from PDF using PyMuPDF for better quality."""
try:
if os.path.getsize(file_path) == 0:
# Empty file: nothing to extract, skip instead of letting
# fitz.EmptyFileError abort the whole build.
return ""
import fitz # PyMuPDF
doc = fitz.open(file_path)
text = ""
for page in doc:
text += page.get_text()
doc.close()
return text
except ImportError:
# Fallback to default reader
return None
except Exception:
# Skip corrupted PDFs instead of aborting the whole build
return ""
def extract_pdf_text_with_pdfplumber(file_path: str) -> str | None:
"""Extract text from PDF using pdfplumber for better quality."""
try:
if os.path.getsize(file_path) == 0:
# Empty file: nothing to extract, skip instead of aborting the build.
return ""
import pdfplumber
text = ""
with pdfplumber.open(file_path) as pdf:
for page in pdf.pages:
text += page.extract_text() or ""
return text
except ImportError:
# Fallback to default reader
return None
except Exception:
# Skip corrupted PDFs instead of aborting the whole build
return ""
class LeannCLI:
def __init__(self):
# Always use project-local .leann directory (like .git)
self.indexes_dir = Path.cwd() / ".leann" / "indexes"
self.indexes_dir.mkdir(parents=True, exist_ok=True)
# Default parser for documents
self.node_parser = SentenceSplitter(
chunk_size=256, chunk_overlap=128, separator=" ", paragraph_separator="\n\n"
)
# Code-optimized parser
self.code_parser = SentenceSplitter(
chunk_size=512, # Larger chunks for code context
chunk_overlap=50, # Less overlap to preserve function boundaries
separator="\n", # Split by lines for code
paragraph_separator="\n\n", # Preserve logical code blocks
)
def get_index_path(self, index_name: str) -> str:
index_dir = self.indexes_dir / index_name
return str(index_dir / "documents.leann")
def index_exists(self, index_name: str) -> bool:
index_dir = self.indexes_dir / index_name
meta_file = index_dir / "documents.leann.meta.json"
return meta_file.exists()
def create_parser(self) -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
prog="leann",
description="The smallest vector index in the world. RAG Everything with LEANN!",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
leann build my-docs --docs ./documents # Build index from directory
leann build my-code --docs ./src ./tests ./config # Build index from multiple directories
leann build my-files --docs ./file1.py ./file2.txt ./docs/ # Build index from files and directories
leann build my-mixed --docs ./readme.md ./src/ ./config.json # Build index from mixed files/dirs
leann build my-ppts --docs ./ --file-types .pptx,.pdf # Index only PowerPoint and PDF files
leann search my-docs "query" # Search in my-docs index
leann ask my-docs "question" # Ask my-docs index
leann react my-docs "complex question" # Use ReAct agent for multiturn retrieval
leann index-browser chrome # Index Chrome browser history
leann index-email # Index Apple Mail
leann index-imessage # Index iMessage conversations
leann index-chatgpt --export-path ~/chatgpt-export.zip # Index ChatGPT export
leann list # List all stored indexes
leann remove my-docs # Remove an index (local first, then global)
""",
)
# Global verbosity options
verbosity_group = parser.add_mutually_exclusive_group()
verbosity_group.add_argument(
"-v",
"--verbose",
action="store_true",
help="Show detailed output including C++ backend logs from FAISS/HNSW",
)
verbosity_group.add_argument(
"-q",
"--quiet",
action="store_true",
help="Suppress all non-essential output (default behavior)",
)
subparsers = parser.add_subparsers(dest="command", help="Available commands")
def add_embedding_args(target_parser: argparse.ArgumentParser) -> None:
"""Add common embedding arguments to a parser."""
target_parser.add_argument(
"--embedding-model",
type=str,
default="facebook/contriever",
help="Embedding model (default: facebook/contriever)",
)
target_parser.add_argument(
"--embedding-mode",
type=str,
default="sentence-transformers",
choices=["sentence-transformers", "openai", "mlx", "ollama"],
help="Embedding backend mode (default: sentence-transformers)",
)
target_parser.add_argument(
"--embedding-host",
type=str,
default=None,
help="Override Ollama-compatible embedding host",
)
target_parser.add_argument(
"--embedding-api-base",
type=str,
default=None,
help="Base URL for OpenAI-compatible embedding services",
)
target_parser.add_argument(
"--embedding-api-key",
type=str,
default=None,
help="API key for embedding service (defaults to OPENAI_API_KEY)",
)
# Build command
build_parser = subparsers.add_parser("build", help="Build document index")
build_parser.add_argument(
"index_name", nargs="?", help="Index name (default: current directory name)"
)
build_parser.add_argument(
"--docs",
type=str,
nargs="+",
default=["."],
help="Documents directories and/or files (default: current directory)",
)
build_parser.add_argument(
"--backend-name",
type=str,
default="hnsw",
choices=["hnsw", "diskann", "ivf"],
help="Backend to use (default: hnsw)",
)
_default_model = _default_embedding_model()
build_parser.add_argument(
"--embedding-model",
type=str,
default=_default_model,
help=f"Embedding model (default: {_default_model})",
)
build_parser.add_argument(
"--embedding-mode",
type=str,
default="sentence-transformers",
choices=["sentence-transformers", "openai", "mlx", "ollama"],
help="Embedding backend mode (default: sentence-transformers)",
)
build_parser.add_argument(
"--embedding-host",
type=str,
default=None,
help="Override Ollama-compatible embedding host",
)
build_parser.add_argument(
"--embedding-api-base",
type=str,
default=None,
help="Base URL for OpenAI-compatible embedding services",
)
build_parser.add_argument(
"--embedding-api-key",
type=str,
default=None,
help="API key for embedding service (defaults to OPENAI_API_KEY)",
)
build_parser.add_argument(
"--embedding-prompt-template",
type=str,
default=None,
help="Prompt template to prepend to all texts for embedding (e.g., 'query: ' for search)",
)
build_parser.add_argument(
"--embedding-batch-size",
type=int,
default=None,
help="Embedding batch size for sentence-transformers (overrides device defaults; also set LEANN_CUDA_BATCH_SIZE env)",
)
build_parser.add_argument(
"--query-prompt-template",
type=str,
default=None,
help="Prompt template for queries (different from build template for task-specific models)",
)
build_parser.add_argument(
"--force",
"-f",
action="store_true",
help="Force full rebuild of existing index (without this, build does incremental update: add new files only)",
)
build_parser.add_argument(
"--graph-degree", type=int, default=32, help="Graph degree (default: 32)"
)
build_parser.add_argument(
"--complexity", type=int, default=64, help="Build complexity (default: 64)"
)
build_parser.add_argument("--num-threads", type=int, default=1)
build_parser.add_argument(
"--compact",
action=argparse.BooleanOptionalAction,
default=False,
help="Use compact (CSR) graph storage. Compact indices are read-only and cannot be updated incrementally. Default: false (allows incremental updates while still pruning embeddings for 97%% compression).",
)
build_parser.add_argument(
"--recompute",
action=argparse.BooleanOptionalAction,
default=True,
help="Enable recomputation (default: true)",
)
build_parser.add_argument(
"--file-types",
type=str,
help="Comma-separated list of file extensions to include (e.g., '.txt,.pdf,.pptx'). If not specified, uses default supported types.",
)
build_parser.add_argument(
"--include-hidden",
action=argparse.BooleanOptionalAction,
default=False,
help="Include hidden files and directories (paths starting with '.') during indexing (default: false)",
)
build_parser.add_argument(
"--doc-chunk-size",
type=int,
default=256,
help="Document chunk size in TOKENS (default: 256), including overlap. Capped to 70%% of the embedding model's sequence limit so chunks are fully embedded (e.g. 179 for all-MiniLM-L6-v2)",
)
build_parser.add_argument(
"--doc-chunk-overlap",
type=int,
default=128,
help="Document chunk overlap in TOKENS (default: 128). Added to chunk size, not included in it",
)
build_parser.add_argument(
"--code-chunk-size",
type=int,
default=512,
help="Code chunk size in TOKENS (default: 512), including overlap. Capped to 70%% of the embedding model's sequence limit so chunks are fully embedded (e.g. 179 for all-MiniLM-L6-v2)",
)
build_parser.add_argument(
"--code-chunk-overlap",
type=int,
default=50,
help="Code chunk overlap in TOKENS (default: 50). Added to chunk size, not included in it",
)
build_parser.add_argument(
"--use-ast-chunking",
action="store_true",
help="Enable AST-aware chunking for code files (requires astchunk)",
)
build_parser.add_argument(
"--ast-chunk-size",
type=int,
default=300,
help="AST chunk size in CHARACTERS (non-whitespace) (default: 300). Final chunks may be larger due to overlap and expansion. For 512 token models: recommended 300 chars (300 + 64 overlap ~= 480 tokens)",
)
build_parser.add_argument(
"--ast-chunk-overlap",
type=int,
default=64,
help="AST chunk overlap in CHARACTERS (default: 64). Added to chunk size, not included in it. ~1.2 tokens per character for code",
)
build_parser.add_argument(
"--ast-fallback-traditional",
action="store_true",
default=True,
help="Fall back to traditional chunking if AST chunking fails (default: True)",
)
build_parser.add_argument(
"--id-scheme",
choices=["sequential", "content-hash"],
default="sequential",
help=(
"How passage IDs are assigned. 'sequential' (default) keys by insertion "
"order; 'content-hash' uses sha256(text)[:16], stable across file moves "
"and reorderings. See #329."
),
)
# Slack indexing command
slack_parser = subparsers.add_parser("index-slack", help="Index Slack workspace via MCP")
slack_parser.add_argument(
"index_name", nargs="?", default="slack", help="Index name (default: slack)"
)
slack_parser.add_argument(
"--mcp-server",
type=str,
required=True,
help="MCP server command (e.g., 'slack-mcp-server')",
)
slack_parser.add_argument("--workspace-name", type=str, help="Slack workspace name")
slack_parser.add_argument(
"--channels",
type=str,
nargs="+",
default=[],
help="Specific channels to index (optional)",
)
add_embedding_args(slack_parser)
# Watch command
watch_parser = subparsers.add_parser(
"watch",
help="Monitor source files and auto-rebuild index when changes are detected",
)
watch_parser.add_argument("index_name", help="Index name")
watch_parser.add_argument(
"--interval",
type=int,
default=30,
help="Poll interval in seconds (default: 30)",
)
watch_parser.add_argument(
"--once",
action="store_true",
help="Check once for changes and exit (do not loop)",
)
watch_parser.add_argument(
"--dry-run",
action="store_true",
help="Report changes without rebuilding (original watch behavior)",
)
migrate_parser = subparsers.add_parser(
"migrate-ids",
help=(
"Rewrite an existing index's passage IDs to content-hash form. "
"Irreversible; back up the index first."
),
)
migrate_parser.add_argument("index_name", help="Index name")
migrate_parser.add_argument(
"--dry-run",
action="store_true",
help="Show what would change without writing anything.",
)
migrate_parser.add_argument(
"-y",
"--yes",
action="store_true",
help="Skip the interactive confirmation prompt.",
)
rebuild_parser = subparsers.add_parser(
"rebuild",
help="Rebuild an existing index using its stored config (delta by default; --force for full rebuild)",
)
rebuild_parser.add_argument("index_name", help="Index name")
rebuild_parser.add_argument(
"-f",
"--force",
action="store_true",
help="Full rebuild from scratch instead of incremental delta",
)
# Search command
search_parser = subparsers.add_parser("search", help="Search documents")
search_parser.add_argument("index_name", help="Index name")
search_parser.add_argument("query", help="Search query")
search_parser.add_argument(
"--top-k", type=int, default=5, help="Number of results (default: 5)"
)
search_parser.add_argument(
"--complexity", type=int, default=64, help="Search complexity (default: 64)"
)
search_parser.add_argument("--beam-width", type=int, default=1)
search_parser.add_argument("--prune-ratio", type=float, default=0.0)
search_parser.add_argument(
"--recompute",
dest="recompute_embeddings",
action=argparse.BooleanOptionalAction,
default=True,
help="Enable/disable embedding recomputation (default: enabled). Should not do a `no-recompute` search in a `recompute` build.",
)
search_parser.add_argument(
"--pruning-strategy",
choices=["global", "local", "proportional"],
default="global",
help="Pruning strategy (default: global)",
)
search_parser.add_argument(
"--json",
action="store_true",
help="Output results as JSON array (machine-readable)",
)
search_parser.add_argument(
"--non-interactive",
action="store_true",
help="Non-interactive mode: automatically select index without prompting",
)
search_parser.add_argument(
"--show-metadata",
action="store_true",
help="Display file paths and metadata in search results",
)
search_parser.add_argument(
"--embedding-prompt-template",
type=str,
default=None,
help="Prompt template to prepend to query for embedding (e.g., 'query: ' for search)",
)
search_parser.add_argument(
"--daemon",
dest="use_daemon",
action=argparse.BooleanOptionalAction,
default=True,
help="Enable/disable cross-process daemon reuse for embedding server (default: enabled)",
)
search_parser.add_argument(
"--daemon-ttl",
type=int,
default=900,
help="Daemon idle TTL in seconds (default: 900, 0 = never expire)",
)
search_parser.add_argument(
"--warmup",
dest="enable_warmup",
action=argparse.BooleanOptionalAction,
default=True,
help="Enable/disable warmup when starting embedding server (default: enabled)",
)
search_parser.add_argument(
"--metadata-filters",
type=str,
default=None,
help=(
"Filter results by metadata fields (JSON string). "
'Format: \'{"field": {"operator": value}}\'. '
"Operators: ==, !=, <, <=, >, >=, in, not_in, contains, starts_with, ends_with. "
'Example: \'{"chapter": {"<=": 5}, "genre": {"==": "fiction"}}\''
),
)
# Warmup command
warmup_parser = subparsers.add_parser("warmup", help="Warm up an index embedding server")
warmup_parser.add_argument("index_name", help="Index name")
warmup_parser.add_argument(
"--daemon",
dest="use_daemon",
action=argparse.BooleanOptionalAction,
default=True,
help="Enable/disable daemon mode for warmup (default: enabled)",
)
warmup_parser.add_argument(
"--daemon-ttl",
type=int,
default=900,
help="Daemon idle TTL in seconds (default: 900, 0 = never expire)",
)
warmup_parser.add_argument(
"--warmup",
dest="enable_warmup",
action=argparse.BooleanOptionalAction,
default=True,
help="Enable/disable warmup request itself (default: enabled)",
)
# Daemon command
daemon_parser = subparsers.add_parser("daemon", help="Manage embedding daemons")
daemon_subparsers = daemon_parser.add_subparsers(dest="daemon_command")
daemon_start = daemon_subparsers.add_parser("start", help="Start daemon for an index")
daemon_start.add_argument("index_name", help="Index name")
daemon_start.add_argument(
"--daemon-ttl",
type=int,
default=900,
help="Daemon idle TTL in seconds (default: 900, 0 = never expire)",
)
daemon_start.add_argument(
"--warmup",
dest="enable_warmup",
action=argparse.BooleanOptionalAction,
default=True,
help="Enable/disable startup warmup (default: enabled)",
)
daemon_stop = daemon_subparsers.add_parser("stop", help="Stop daemon(s)")
daemon_stop.add_argument("index_name", nargs="?", help="Index name to stop")
daemon_stop.add_argument(
"--all",
action="store_true",
help="Stop all LEANN embedding daemons",
)
daemon_status = daemon_subparsers.add_parser("status", help="Show daemon status")
daemon_status.add_argument("index_name", nargs="?", help="Optional index name filter")
# Ask command
ask_parser = subparsers.add_parser("ask", help="Ask questions")
ask_parser.add_argument("index_name", help="Index name")
ask_parser.add_argument(
"query",
nargs="?",
help="Question to ask (omit for prompt or when using --interactive)",
)
ask_parser.add_argument(
"--llm",
type=str,
default="ollama",
choices=[
"simulated",
"ollama",
"hf",
"openai",
"anthropic",
"litellm",
"minimax",
"novita",
"atlascloud",
"atlas-cloud",
"atlas",
],
help="LLM provider (default: ollama)",
)
ask_parser.add_argument(
"--model", type=str, default="qwen3:8b", help="Model name (default: qwen3:8b)"
)
ask_parser.add_argument(
"--host",
type=str,
default=None,
help="Override Ollama-compatible host (defaults to LEANN_OLLAMA_HOST/OLLAMA_HOST)",
)
ask_parser.add_argument(
"--interactive", "-i", action="store_true", help="Interactive chat mode"
)
ask_parser.add_argument(
"--top-k", type=int, default=20, help="Retrieval count (default: 20)"
)
ask_parser.add_argument("--complexity", type=int, default=32)
ask_parser.add_argument("--beam-width", type=int, default=1)
ask_parser.add_argument("--prune-ratio", type=float, default=0.0)
ask_parser.add_argument(
"--recompute",
dest="recompute_embeddings",
action=argparse.BooleanOptionalAction,
default=True,
help="Enable/disable embedding recomputation during ask (default: enabled)",
)
ask_parser.add_argument(
"--pruning-strategy",
choices=["global", "local", "proportional"],
default="global",
)
ask_parser.add_argument(
"--thinking-budget",
type=str,
choices=["low", "medium", "high"],
default=None,
help="Thinking budget for reasoning models (low/medium/high). Supported by GPT-Oss:20b and other reasoning models.",
)
ask_parser.add_argument(
"--api-base",
type=str,
default=None,
help="Base URL for OpenAI-compatible APIs (e.g., http://localhost:10000/v1)",
)
ask_parser.add_argument(
"--api-key",
type=str,
default=None,
help="API key for cloud LLM providers",
)
ask_parser.add_argument(
"--metadata-filters",
type=str,
default=None,
help=(
"Filter retrieved chunks by metadata fields before answering (JSON string). "
'Format: \'{"field": {"operator": value}}\'. '
"Operators: ==, !=, <, <=, >, >=, in, not_in, contains, starts_with, ends_with. "
'Example: \'{"chapter": {"<=": 5}, "genre": {"==": "fiction"}}\''
),
)
# React command (multiturn retrieval agent)
react_parser = subparsers.add_parser(
"react", help="Use ReAct agent for multiturn retrieval and reasoning"
)
react_parser.add_argument("index_name", help="Index name")
react_parser.add_argument("query", help="Question to research")
react_parser.add_argument(
"--llm",
type=str,
default="ollama",
choices=[
"simulated",
"ollama",
"hf",
"openai",
"anthropic",
"litellm",
"minimax",
"novita",
"atlascloud",
"atlas-cloud",
"atlas",
],
help="LLM provider (default: ollama)",
)
react_parser.add_argument(
"--model", type=str, default="qwen3:8b", help="Model name (default: qwen3:8b)"
)
react_parser.add_argument(
"--host",
type=str,
default=None,
help="Override Ollama-compatible host (defaults to LEANN_OLLAMA_HOST/OLLAMA_HOST)",
)
react_parser.add_argument(
"--top-k", type=int, default=5, help="Number of results per search (default: 5)"
)
react_parser.add_argument(
"--max-iterations",
type=int,
default=5,
help="Maximum number of search iterations (default: 5)",
)
react_parser.add_argument(
"--api-base",
type=str,
default=None,
help="Base URL for OpenAI-compatible APIs (e.g., http://localhost:10000/v1)",
)
react_parser.add_argument(
"--api-key",
type=str,
default=None,
help="API key for cloud LLM providers",
)
react_parser.add_argument(
"--serper-api-key",
type=str,
default=None,
help="Serper API key for web search (or set SERPER_API_KEY env var)",
)
react_parser.add_argument(
"--jina-api-key",
type=str,
default=None,
help="Jina API key for page content fetching (or set JINA_API_KEY env var)",
)
# ── index-* commands: data source indexing ──────────────────────
def _add_index_args(p, default_name):
"""Add common embedding and index args to an index-* subparser."""
p.add_argument(
"--index-name",
type=str,
default=default_name,
help=f"Index name (default: {default_name})",
)
p.add_argument(
"--embedding-model", type=str, default="facebook/contriever", help="Embedding model"
)
p.add_argument(
"--embedding-mode",
type=str,
default="sentence-transformers",
choices=["sentence-transformers", "openai", "mlx", "ollama"],
help="Embedding backend",
)
p.add_argument("--embedding-host", type=str, default=None, help="Ollama embedding host")
p.add_argument(
"--embedding-api-base",
type=str,
default=None,
help="OpenAI-compatible embedding base URL",
)
p.add_argument("--embedding-api-key", type=str, default=None, help="Embedding API key")
p.add_argument(
"--embedding-batch-size",
type=int,
default=None,
help="Embedding batch size for sentence-transformers",
)
p.add_argument(
"--max-count", type=int, default=1000, help="Max items to index (default: 1000)"
)
p.add_argument(
"--no-recompute",
action="store_true",
help="Disable embedding recomputation (stores full embeddings)",
)
idx_browser = subparsers.add_parser(
"index-browser", help="Index browser history (Chrome/Brave)"
)
_add_index_args(idx_browser, "browser_history")
idx_browser.add_argument(
"browser",
nargs="?",
default="chrome",
choices=["chrome", "brave"],
help="Browser to index (default: chrome)",
)
idx_email = subparsers.add_parser("index-email", help="Index Apple Mail")
_add_index_args(idx_email, "email")
idx_calendar = subparsers.add_parser("index-calendar", help="Index Apple Calendar events")
_add_index_args(idx_calendar, "calendar")
idx_imessage = subparsers.add_parser("index-imessage", help="Index iMessage conversations")
_add_index_args(idx_imessage, "imessage")
idx_wechat = subparsers.add_parser("index-wechat", help="Index WeChat chat history")
_add_index_args(idx_wechat, "wechat")
idx_wechat.add_argument(
"--export-dir", type=str, required=True, help="Path to WeChat JSON export directory"
)
idx_chatgpt = subparsers.add_parser("index-chatgpt", help="Index ChatGPT export")
_add_index_args(idx_chatgpt, "chatgpt")
idx_chatgpt.add_argument(
"--export-path",
type=str,
required=True,
help="Path to ChatGPT export (chat.html or .zip)",
)
idx_claude = subparsers.add_parser("index-claude", help="Index Claude export")
_add_index_args(idx_claude, "claude")
idx_claude.add_argument(
"--export-path", type=str, required=True, help="Path to Claude export (.json or .zip)"
)
# List command
list_parser = subparsers.add_parser("list", help="List all indexes")
list_parser.add_argument(
"--max-depth",
type=_non_negative_int,
default=DEFAULT_INDEX_SCAN_DEPTH,
help=(
"Maximum directory depth for app index discovery "
f"(default: {DEFAULT_INDEX_SCAN_DEPTH})"
),
)
# Remove command
remove_parser = subparsers.add_parser("remove", help="Remove an index")
remove_parser.add_argument("index_name", help="Index name to remove")
remove_parser.add_argument(
"--force", "-f", action="store_true", help="Force removal without confirmation"
)
# Serve command (HTTP API server)
serve_parser = subparsers.add_parser(
"serve", help="Start HTTP API server for LEANN vector DB"
)
serve_parser.add_argument(
"--host", type=str, default=None, help="Host to bind to (default: 127.0.0.1)"
)
serve_parser.add_argument(
"--port", type=int, default=None, help="Port to bind to (default: 8000)"
)
return parser
def register_project_dir(self):
"""Register current project directory in global registry"""
register_project_directory()
def _build_gitignore_parser(self, docs_dir: str):
"""Build gitignore parser using gitignore-parser library."""
from gitignore_parser import parse_gitignore
# Try to parse the root .gitignore
gitignore_path = Path(docs_dir) / ".gitignore"
if gitignore_path.exists():
try:
# gitignore-parser automatically handles all subdirectory .gitignore files!
matches = parse_gitignore(str(gitignore_path))
print(f"📋 Loaded .gitignore from {docs_dir} (includes all subdirectories)")
return matches
except Exception as e:
print(f"Warning: Could not parse .gitignore: {e}")
else:
print("📋 No .gitignore found")
# Fallback: basic pattern matching for essential files
essential_patterns = {".git", ".DS_Store", "__pycache__", "node_modules", ".venv", "venv"}
def basic_matches(file_path):
path_parts = Path(file_path).parts
return any(part in essential_patterns for part in path_parts)
return basic_matches
def _should_exclude_file(self, file_path: Path, gitignore_matches) -> bool:
"""Check if a file should be excluded using gitignore parser.
Always match against absolute, posix-style paths for consistency with
gitignore_parser expectations.
"""
try:
absolute_path = file_path.resolve()
except Exception:
absolute_path = Path(str(file_path))
return gitignore_matches(absolute_path.as_posix())
def _is_git_submodule(self, path: Path) -> bool:
"""Check if a path is a git submodule."""
try:
# Find the git repo root
current_dir = Path.cwd()
while current_dir != current_dir.parent:
if (current_dir / ".git").exists():
gitmodules_path = current_dir / ".gitmodules"
if gitmodules_path.exists():
# Read .gitmodules to check if this path is a submodule
gitmodules_content = gitmodules_path.read_text()
# Convert path to relative to git root
try:
relative_path = path.resolve().relative_to(current_dir)
# Check if this path appears in .gitmodules
return f"path = {relative_path}" in gitmodules_content
except ValueError:
# Path is not under git root
return False
break
current_dir = current_dir.parent
return False
except Exception:
# If anything goes wrong, assume it's not a submodule
return False
@staticmethod
def _project_has_discoverable_indexes(project_path: Path, max_depth: int) -> bool:
"""Return whether a project contains an index within the scan boundary."""
cli_indexes_dir = project_path / ".leann" / "indexes"
return cli_indexes_dir.exists() or any(
iter_index_meta_files(project_path, max_depth=max_depth)
)
@staticmethod
def _nested_project_dirs(project_path: Path, projects: list[Path]) -> list[Path]:
"""Return registered project roots nested below a project."""
return [
candidate
for candidate in projects
if candidate != project_path and candidate.is_relative_to(project_path)
]
def list_indexes(self, max_depth: int = DEFAULT_INDEX_SCAN_DEPTH):
# Get all project directories with .leann
global_registry = Path.home() / ".leann" / "projects.json"
all_projects = []
if global_registry.exists():
try:
import json
with open(global_registry) as f:
all_projects = json.load(f)
except Exception:
pass
# Filter to existing projects with a discoverable CLI- or App-format index.
valid_projects = []
for project_dir in all_projects:
project_path = Path(project_dir)
if project_path.exists() and self._project_has_discoverable_indexes(
project_path, max_depth
):
valid_projects.append(project_path)
# Add current project if it has a discoverable index but is not in registry.
current_path = Path.cwd()
if (
self._project_has_discoverable_indexes(current_path, max_depth)
and current_path not in valid_projects
):
valid_projects.append(current_path)
# Separate current and other projects
other_projects = []
for project_path in valid_projects:
if project_path != current_path:
other_projects.append(project_path)
# Exclude only registered projects nested below the current project.
# Registered ancestors must not hide indexes owned by the current directory.
current_exclude_dirs = self._nested_project_dirs(current_path, valid_projects)
print("📚 LEANN Indexes")
print("=" * 50)
total_indexes = 0
current_indexes_count = 0
# Show current project first (most important)
print("\n🏠 Current Project")
print(f" {current_path}")
print(" " + "─" * 45)
current_indexes = self._discover_indexes_in_project(
current_path, exclude_dirs=current_exclude_dirs, max_depth=max_depth
)
if current_indexes:
for idx in current_indexes:
total_indexes += 1
current_indexes_count += 1
type_icon = "📁" if idx["type"] == "cli" else "📄"
print(f" {current_indexes_count}. {type_icon} {idx['name']} {idx['status']}")
if idx["size_mb"] > 0:
print(f" 📦 Size: {idx['size_mb']:.1f} MB")
else:
print(" 📭 No indexes in current project")
# Show other projects (reference information)
if other_projects:
print("\n\n🗂️ Other Projects")
print(" " + "─" * 45)
for project_path in other_projects:
nested_projects = self._nested_project_dirs(project_path, valid_projects)
project_indexes = self._discover_indexes_in_project(
project_path, exclude_dirs=nested_projects, max_depth=max_depth
)
if not project_indexes:
continue
print(f"\n 📂 {project_path.name}")
print(f" {project_path}")
for idx in project_indexes:
total_indexes += 1
type_icon = "📁" if idx["type"] == "cli" else "📄"
print(f" • {type_icon} {idx['name']} {idx['status']}")
if idx["size_mb"] > 0:
print(f" 📦 {idx['size_mb']:.1f} MB")
# Summary and usage info
print("\n" + "=" * 50)
if total_indexes == 0:
print("💡 Get started:")
print(" leann build my-docs --docs ./documents")
else:
# Count only projects that have at least one discoverable index
projects_count = 0
for p in valid_projects:
if p == current_path:
discovered = self._discover_indexes_in_project(
p, exclude_dirs=current_exclude_dirs, max_depth=max_depth
)
else:
nested_projects = self._nested_project_dirs(p, valid_projects)
discovered = self._discover_indexes_in_project(
p, exclude_dirs=nested_projects, max_depth=max_depth
)
if len(discovered) > 0:
projects_count += 1
print(f"📊 Total: {total_indexes} indexes across {projects_count} projects")
if current_indexes_count > 0:
print("\n💫 Quick start (current project):")
# Get first index from current project for example
current_indexes_dir = current_path / ".leann" / "indexes"
if current_indexes_dir.exists():
current_index_dirs = [d for d in current_indexes_dir.iterdir() if d.is_dir()]
if current_index_dirs:
example_name = current_index_dirs[0].name
print(f' leann search {example_name} "your query"')
print(f" leann ask {example_name} --interactive")
else:
print("\n💡 Create your first index:")
print(" leann build my-docs --docs ./documents")
def _discover_indexes_in_project(
self,
project_path: Path,
exclude_dirs: Optional[list[Path]] = None,
max_depth: Optional[int] = None,
):
"""Discover all indexes in a project directory (both CLI and apps formats)
exclude_dirs: when provided, skip any APP-format index files that are
located under these directories. This prevents duplicates when the
current project is a parent directory of other registered projects.
max_depth: maximum number of directories to descend for APP-format indexes.
None preserves full-depth discovery for non-CLI callers.
"""
indexes = []
exclude_dirs = exclude_dirs or []
# normalize to resolved paths once for comparison
try:
exclude_dirs_resolved = [p.resolve() for p in exclude_dirs]
except Exception:
exclude_dirs_resolved = exclude_dirs
# 1. CLI format: .leann/indexes/index_name/
cli_indexes_dir = project_path / ".leann" / "indexes"
if cli_indexes_dir.exists():
for index_dir in cli_indexes_dir.iterdir():
if index_dir.is_dir():
meta_file = index_dir / "documents.leann.meta.json"
status = "✅" if meta_file.exists() else "❌"
size_mb = 0
if meta_file.exists():
try:
size_mb = sum(
f.stat().st_size for f in index_dir.iterdir() if f.is_file()
) / (1024 * 1024)
except (OSError, PermissionError):
pass
indexes.append(
{
"name": index_dir.name,
"type": "cli",
"status": status,
"size_mb": size_mb,
"path": index_dir,
}
)
# 2. Apps format: *.leann.meta.json files within the configured scan depth
cli_indexes_dir = project_path / ".leann" / "indexes"
for meta_file in iter_index_meta_files(project_path, max_depth=max_depth):
if meta_file.is_file():
# Skip CLI-built indexes (which store meta under .leann/indexes/<name>/)
try:
if cli_indexes_dir.exists() and cli_indexes_dir in meta_file.parents:
continue
except Exception:
pass
# Skip meta files that live under excluded directories
try:
meta_parent_resolved = meta_file.parent.resolve()
if any(
meta_parent_resolved.is_relative_to(ex_dir)
for ex_dir in exclude_dirs_resolved
):
continue
except Exception:
# best effort; if resolve or comparison fails, do not exclude
pass
# Use the parent directory name as the app index display name
display_name = meta_file.parent.name
# Extract file base used to store files
file_base = meta_file.name.replace(".leann.meta.json", "")
# Apps indexes are considered complete if the .leann.meta.json file exists
status = "✅"
# Calculate total size of all related files (use file base)
size_mb = 0
try:
index_dir = meta_file.parent
for related_file in index_dir.glob(f"{file_base}.leann*"):
size_mb += related_file.stat().st_size / (1024 * 1024)
except (OSError, PermissionError):
pass
indexes.append(
{
"name": display_name,
"type": "app",
"status": status,
"size_mb": size_mb,
"path": meta_file,
}
)
return indexes
def remove_index(self, index_name: str, force: bool = False):
"""Safely remove an index - always show all matches for transparency"""
# Always do a comprehensive search for safety
print(f"🔍 Searching for all indexes named '{index_name}'...")
all_matches = self._find_all_matching_indexes(index_name)
if not all_matches:
print(f"❌ Index '{index_name}' not found in any project.")
return False
if len(all_matches) == 1:
return self._remove_single_match(all_matches[0], index_name, force)
else:
return self._remove_from_multiple_matches(all_matches, index_name, force)
def _find_all_matching_indexes(self, index_name: str):
"""Find all indexes with the given name across all projects"""
matches = []
# Get all registered projects
global_registry = Path.home() / ".leann" / "projects.json"
all_projects = []
if global_registry.exists():
try:
import json
with open(global_registry) as f:
all_projects = json.load(f)
except Exception:
pass
# Always include current project
current_path = Path.cwd()
if str(current_path) not in all_projects:
all_projects.append(str(current_path))
# Search across all projects
for project_dir in all_projects:
project_path = Path(project_dir)
if not project_path.exists():
continue
# 1) CLI-format index under .leann/indexes/<name>
index_dir = project_path / ".leann" / "indexes" / index_name
if index_dir.exists():
is_current = project_path == current_path
matches.append(
{
"project_path": project_path,
"index_dir": index_dir,
"is_current": is_current,
"kind": "cli",
}
)
# 2) App-format indexes
# We support two ways of addressing apps:
# a) by the file base (e.g., `pdf_documents`)
# b) by the parent directory name (e.g., `new_txt`)
seen_app_meta = set()
# 2a) by file base
for meta_file in project_path.rglob(f"{index_name}.leann.meta.json"):
if meta_file.is_file():
# Skip CLI-built indexes' meta under .leann/indexes
try:
cli_indexes_dir = project_path / ".leann" / "indexes"
if cli_indexes_dir.exists() and cli_indexes_dir in meta_file.parents:
continue
except Exception:
pass
is_current = project_path == current_path
key = (str(project_path), str(meta_file))
if key in seen_app_meta:
continue
seen_app_meta.add(key)
matches.append(
{
"project_path": project_path,
"files_dir": meta_file.parent,
"meta_file": meta_file,
"is_current": is_current,
"kind": "app",
"display_name": meta_file.parent.name,
"file_base": meta_file.name.replace(".leann.meta.json", ""),
}
)
# 2b) by parent directory name
for meta_file in project_path.rglob("*.leann.meta.json"):
if meta_file.is_file() and meta_file.parent.name == index_name:
# Skip CLI-built indexes' meta under .leann/indexes
try:
cli_indexes_dir = project_path / ".leann" / "indexes"
if cli_indexes_dir.exists() and cli_indexes_dir in meta_file.parents:
continue
except Exception:
pass
is_current = project_path == current_path
key = (str(project_path), str(meta_file))
if key in seen_app_meta:
continue
seen_app_meta.add(key)
matches.append(
{
"project_path": project_path,
"files_dir": meta_file.parent,
"meta_file": meta_file,
"is_current": is_current,
"kind": "app",
"display_name": meta_file.parent.name,
"file_base": meta_file.name.replace(".leann.meta.json", ""),
}
)
# Sort: current project first, then by project name
matches.sort(key=lambda x: (not x["is_current"], x["project_path"].name))
return matches
def _remove_single_match(self, match, index_name: str, force: bool):
"""Handle removal when only one match is found"""
project_path = match["project_path"]
is_current = match["is_current"]
kind = match.get("kind", "cli")
if is_current:
location_info = "current project"
emoji = "🏠"
else:
location_info = f"other project '{project_path.name}'"
emoji = "📂"
print(f"✅ Found 1 index named '{index_name}':")
print(f" {emoji} Location: {location_info}")
if kind == "cli":
print(f" 📍 Path: {project_path / '.leann' / 'indexes' / index_name}")
else:
print(f" 📍 Meta: {match['meta_file']}")
if not force:
if not is_current:
print("\n⚠️ CROSS-PROJECT REMOVAL!")
print(" This will delete the index from another project.")
response = input(f" ❓ Confirm removal from {location_info}? (y/N): ").strip().lower()
if response not in ["y", "yes"]:
print(" ❌ Removal cancelled.")
return False
if kind == "cli":
return self._delete_index_directory(
match["index_dir"],
index_name,
project_path if not is_current else None,
is_app=False,
)
else:
return self._delete_index_directory(
match["files_dir"],
match.get("display_name", index_name),
project_path if not is_current else None,
is_app=True,
meta_file=match.get("meta_file"),
app_file_base=match.get("file_base"),
)
def _remove_from_multiple_matches(self, matches, index_name: str, force: bool):
"""Handle removal when multiple matches are found"""
print(f"⚠️ Found {len(matches)} indexes named '{index_name}':")
print(" " + "─" * 50)
for i, match in enumerate(matches, 1):
project_path = match["project_path"]
is_current = match["is_current"]
kind = match.get("kind", "cli")
if is_current:
print(f" {i}. 🏠 Current project ({'CLI' if kind == 'cli' else 'APP'})")
else:
print(f" {i}. 📂 {project_path.name} ({'CLI' if kind == 'cli' else 'APP'})")
# Show path details
if kind == "cli":
print(f" 📍 {project_path / '.leann' / 'indexes' / index_name}")
else:
print(f" 📍 {match['meta_file']}")
# Show size info
try:
if kind == "cli":
size_mb = sum(
f.stat().st_size for f in match["index_dir"].iterdir() if f.is_file()
) / (1024 * 1024)
else:
file_base = match.get("file_base")
size_mb = 0.0
if file_base:
size_mb = sum(
f.stat().st_size
for f in match["files_dir"].glob(f"{file_base}.leann*")
if f.is_file()
) / (1024 * 1024)
print(f" 📦 Size: {size_mb:.1f} MB")
except (OSError, PermissionError):
pass
print(" " + "─" * 50)
if force:
print(" ❌ Multiple matches found, but --force specified.")
print(" Please run without --force to choose which one to remove.")
return False
try:
choice = input(
f" ❓ Which one to remove? (1-{len(matches)}, or 'c' to cancel): "
).strip()
if choice.lower() == "c":
print(" ❌ Removal cancelled.")
return False
choice_idx = int(choice) - 1
if 0 <= choice_idx < len(matches):
selected_match = matches[choice_idx]
project_path = selected_match["project_path"]
is_current = selected_match["is_current"]
kind = selected_match.get("kind", "cli")
location = "current project" if is_current else f"'{project_path.name}' project"
print(f" 🎯 Selected: Remove from {location}")
# Final confirmation for safety
confirm = input(
f" ❓ FINAL CONFIRMATION - Type '{index_name}' to proceed: "
).strip()
if confirm != index_name:
print(" ❌ Confirmation failed. Removal cancelled.")
return False
if kind == "cli":
return self._delete_index_directory(
selected_match["index_dir"],
index_name,
project_path if not is_current else None,
is_app=False,
)
else:
return self._delete_index_directory(
selected_match["files_dir"],
selected_match.get("display_name", index_name),
project_path if not is_current else None,
is_app=True,
meta_file=selected_match.get("meta_file"),
app_file_base=selected_match.get("file_base"),
)
else:
print(" ❌ Invalid choice. Removal cancelled.")
return False
except (ValueError, KeyboardInterrupt):
print("\n ❌ Invalid input. Removal cancelled.")
return False
def _delete_index_directory(
self,
index_dir: Path,
index_display_name: str,
project_path: Optional[Path] = None,
is_app: bool = False,
meta_file: Optional[Path] = None,
app_file_base: Optional[str] = None,
):
"""Delete a CLI index directory or APP index files safely."""
try:
if is_app:
removed = 0
errors = 0
# Delete only files that belong to this app index (based on file base)
pattern_base = app_file_base or ""
for f in index_dir.glob(f"{pattern_base}.leann*"):
try:
f.unlink()
removed += 1
except Exception:
errors += 1
# Best-effort: also remove the meta file if specified and still exists
if meta_file and meta_file.exists():
try:
meta_file.unlink()
removed += 1
except Exception:
errors += 1
if removed > 0 and errors == 0:
if project_path:
print(
f"✅ App index '{index_display_name}' removed from {project_path.name}"
)
else:
print(f"✅ App index '{index_display_name}' removed successfully")
return True
elif removed > 0 and errors > 0:
print(
f"⚠️ App index '{index_display_name}' partially removed (some files couldn't be deleted)"
)
return True
else:
print(
f"❌ No files found to remove for app index '{index_display_name}' in {index_dir}"
)
return False
else:
import shutil
shutil.rmtree(index_dir)
if project_path:
print(f"✅ Index '{index_display_name}' removed from {project_path.name}")
else:
print(f"✅ Index '{index_display_name}' removed successfully")
return True
except Exception as e:
print(f"❌ Error removing index '{index_display_name}': {e}")
return False
def load_documents(
self,
docs_paths: Union[str, list],
custom_file_types: Union[str, None] = None,
include_hidden: bool = False,
args: Optional[dict[str, Any]] = None,
):
# Handle both single path (string) and multiple paths (list) for backward compatibility
if isinstance(docs_paths, str):
docs_paths = [docs_paths]
# Separate files and directories
files = []
directories = []
for path in docs_paths:
path_obj = Path(path)
if path_obj.is_file():
files.append(str(path_obj))
elif path_obj.is_dir():
# Check if this is a git submodule - if so, skip it
if self._is_git_submodule(path_obj):
print(f"⚠️ Skipping git submodule: {path}")
continue
directories.append(str(path_obj))
else:
print(f"⚠️ Warning: Path '{path}' does not exist, skipping...")
continue
# Print summary of what we're processing
total_items = len(files) + len(directories)
items_desc = []
if files:
items_desc.append(f"{len(files)} file{'s' if len(files) > 1 else ''}")
if directories:
items_desc.append(
f"{len(directories)} director{'ies' if len(directories) > 1 else 'y'}"
)
print(f"Loading documents from {' and '.join(items_desc)} ({total_items} total):")
if files:
print(f" 📄 Files: {', '.join([Path(f).name for f in files])}")
if directories:
print(f" 📁 Directories: {', '.join(directories)}")
if custom_file_types:
print(f"Using custom file types: {custom_file_types}")
all_documents = []
# Helper to detect hidden path components
def _path_has_hidden_segment(p: Path) -> bool:
return any(part.startswith(".") and part not in [".", ".."] for part in p.parts)
# First, process individual files if any
if files:
print(f"\n🔄 Processing {len(files)} individual file{'s' if len(files) > 1 else ''}...")
# Load individual files using SimpleDirectoryReader with input_files
# Note: We skip gitignore filtering for explicitly specified files
try:
# Group files by their parent directory for efficient loading
from collections import defaultdict
files_by_dir = defaultdict(list)
for file_path in files:
file_path_obj = Path(file_path)
# Judge an explicitly named file by its own name: testing every
# ancestor made the result depend on whether the user typed a
# relative or an absolute path.
if not include_hidden and file_path_obj.name.startswith("."):
print(f" ⚠️ Skipping hidden file: {file_path}")
continue
parent_dir = str(file_path_obj.parent)
files_by_dir[parent_dir].append(str(file_path_obj))
# Load files from each parent directory
for parent_dir, file_list in files_by_dir.items():
print(
f" Loading {len(file_list)} file{'s' if len(file_list) > 1 else ''} from {parent_dir}"
)
try:
file_docs = SimpleDirectoryReader(
parent_dir,
input_files=file_list,
# exclude_hidden only affects directory scans; input_files are explicit
filename_as_id=True,
).load_data()
for doc in file_docs:
if not doc.metadata.get("source"):
doc.metadata["source"] = doc.metadata.get("file_path", "")
all_documents.extend(file_docs)
print(
f" ✅ Loaded {len(file_docs)} document{'s' if len(file_docs) > 1 else ''}"
)
except Exception as e:
print(f" ❌ Warning: Could not load files from {parent_dir}: {e}")
except Exception as e:
print(f"❌ Error processing individual files: {e}")
# Define file extensions to process
if custom_file_types:
# Parse custom file types from comma-separated string
code_extensions = [ext.strip() for ext in custom_file_types.split(",") if ext.strip()]
# Ensure extensions start with a dot
code_extensions = [ext if ext.startswith(".") else f".{ext}" for ext in code_extensions]
else:
code_extensions = list(DEFAULT_INDEX_EXTENSIONS)
# Process each directory
if directories:
print(
f"\n🔄 Processing {len(directories)} director{'ies' if len(directories) > 1 else 'y'}..."
)
for docs_dir in directories:
print(f"Processing directory: {docs_dir}")
# Build gitignore parser for each directory
gitignore_matches = self._build_gitignore_parser(docs_dir)
# Try to use better PDF parsers first, but only if PDFs are requested
documents = []
# Use resolved absolute paths to avoid mismatches (symlinks, relative vs absolute)
docs_path = Path(docs_dir).resolve()
# Check if we should process PDFs
should_process_pdfs = custom_file_types is None or ".pdf" in custom_file_types
if should_process_pdfs:
for file_path in docs_path.rglob("*.pdf"):
# Check if file matches any exclude pattern
try:
# Ensure both paths are resolved before computing relativity
file_path_resolved = file_path.resolve()
# Determine directory scope using the non-resolved path to avoid
# misclassifying symlinked entries as outside the docs directory
relative_path = file_path.relative_to(docs_path)
if not include_hidden and _path_has_hidden_segment(relative_path):
continue
# Use absolute path for gitignore matching
if self._should_exclude_file(file_path_resolved, gitignore_matches):
continue
except ValueError:
# Skip files that can't be made relative to docs_path
print(f"⚠️ Skipping file outside directory scope: {file_path}")
continue
print(f"Processing PDF: {file_path}")
# Try PyMuPDF first (best quality)
text = extract_pdf_text_with_pymupdf(str(file_path))
if text is None:
# Try pdfplumber
text = extract_pdf_text_with_pdfplumber(str(file_path))
if text:
# Create a simple document structure
from llama_index.core import Document
doc = Document(text=text, metadata={"source": str(file_path)})
documents.append(doc)
else:
# Fallback to default reader
print(f"Using default reader for {file_path}")
try:
default_docs = SimpleDirectoryReader(
str(file_path.parent),
exclude_hidden=not include_hidden,
filename_as_id=True,
required_exts=[file_path.suffix],
).load_data()
documents.extend(default_docs)
except Exception as e:
print(f"Warning: Could not process {file_path}: {e}")
# Load other file types with default reader
# Exclude PDFs from code_extensions if they were already processed separately
other_file_extensions = code_extensions
if should_process_pdfs and ".pdf" in code_extensions:
other_file_extensions = [ext for ext in code_extensions if ext != ".pdf"]
try:
# Create a custom file filter function using our PathSpec
def file_filter(
file_path: str, docs_dir=docs_dir, gitignore_matches=gitignore_matches
) -> bool:
"""Return True if file should be included (not excluded)"""
try:
docs_path_obj = Path(docs_dir).resolve()
file_path_obj = Path(file_path).resolve()
# Use absolute path for gitignore matching
_ = file_path_obj.relative_to(docs_path_obj) # validate scope
return not self._should_exclude_file(file_path_obj, gitignore_matches)
except (ValueError, OSError):
return True # Include files that can't be processed
# Only load other file types if there are extensions to process
if other_file_extensions:
other_docs = SimpleDirectoryReader(
docs_dir,
recursive=True,
encoding="utf-8",
required_exts=other_file_extensions,
file_extractor={}, # Use default extractors
exclude_hidden=not include_hidden,
filename_as_id=True,
).load_data(show_progress=True)
else:
other_docs = []
# Filter documents after loading based on gitignore rules
filtered_docs = []
for doc in other_docs:
file_path = doc.metadata.get("file_path", "")
if file_filter(file_path):
doc.metadata["source"] = file_path
filtered_docs.append(doc)
documents.extend(filtered_docs)
except ValueError as e:
if "No files found" in str(e):
print(f"No additional files found for other supported types in {docs_dir}.")
else:
raise e
all_documents.extend(documents)
print(f"Loaded {len(documents)} documents from {docs_dir}")
documents = all_documents
all_texts = []
# Define code file extensions for intelligent chunking
code_file_exts = {
".py",
".js",
".ts",
".jsx",
".tsx",
".java",
".cpp",
".c",
".h",
".hpp",
".cs",
".go",
".rs",
".rb",
".php",
".swift",
".kt",
".scala",
".r",
".sql",
".sh",
".bash",
".zsh",
".fish",
".ps1",
".bat",
".json",
".yaml",
".yml",
".xml",
".toml",
".ini",
".cfg",
".conf",
".html",
".css",
".scss",
".less",
".vue",
".svelte",
".ipynb",
".R",
".jl",
}
print("start chunking documents")
# Check if AST chunking is requested
use_ast = getattr(args, "use_ast_chunking", False)
if use_ast:
print("🧠 Using AST-aware chunking for code files")
try:
# Import enhanced chunking utilities from packaged module
from .chunking_utils import create_text_chunks
# Use enhanced chunking with AST support
chunk_texts = create_text_chunks(
documents,
chunk_size=self.node_parser.chunk_size,
chunk_overlap=self.node_parser.chunk_overlap,
use_ast_chunking=True,
ast_chunk_size=getattr(args, "ast_chunk_size", 768),
ast_chunk_overlap=getattr(args, "ast_chunk_overlap", 96),
code_file_extensions=None, # Use defaults
ast_fallback_traditional=getattr(args, "ast_fallback_traditional", True),
)
# create_text_chunks now returns list[dict] with metadata preserved
all_texts.extend(chunk_texts)
except ImportError as e:
print(
f"⚠️ AST chunking utilities not available in package ({e}), falling back to traditional chunking"
)
use_ast = False
if not use_ast:
# Use traditional chunking logic
for doc in tqdm(documents, desc="Chunking documents", unit="doc"):
# Check if this is a code file based on source path
source_path = doc.metadata.get("source", "")
file_path = doc.metadata.get("file_path", "")
is_code_file = any(
(source_path or file_path).endswith(ext) for ext in code_file_exts
)
# For code files, prepend line numbers so chunks carry them
if is_code_file:
from llama_index.core.schema import MediaResource
original_text = doc.get_content()
lines = original_text.split("\n")
width = len(str(len(lines)))
numbered = "\n".join(f"{i + 1:>{width}}|{line}" for i, line in enumerate(lines))
doc.text_resource = MediaResource(text=numbered)
# Extract metadata to preserve with chunks
chunk_metadata = {
"file_path": file_path or source_path,
"file_name": doc.metadata.get("file_name", ""),
"source": source_path,
}
# Add optional metadata if available
if "creation_date" in doc.metadata:
chunk_metadata["creation_date"] = doc.metadata["creation_date"]
if "last_modified_date" in doc.metadata:
chunk_metadata["last_modified_date"] = doc.metadata["last_modified_date"]
# Use appropriate parser based on file type
parser = self.code_parser if is_code_file else self.node_parser
nodes = parser.get_nodes_from_documents([doc])
for node in nodes:
text = node.get_content()
# For code chunks, trim a partial first line left by overlap
# (a valid line starts with digits followed by '|')
if is_code_file and text:
first_line = text.split("\n", 1)[0]
if "|" in first_line and not re.match(r"^\s*\d+\|", first_line):
first_nl = text.find("\n")
if first_nl != -1:
text = text[first_nl + 1 :]
all_texts.append({"text": text, "metadata": chunk_metadata.copy()})
print(f"Loaded {len(documents)} documents, {len(all_texts)} chunks")
return all_texts
def _parse_file_types(self, custom_file_types: Optional[str]) -> list[str]:
return parse_include_extensions(custom_file_types)
def _sync_ignore_patterns(self, include_hidden: bool) -> Optional[list[str]]:
if include_hidden:
return None
return ["**/.*"]
def _build_embedding_options(self, args) -> dict[str, Any]:
"""Build embedding provider options dict from CLI args."""
opts: dict[str, Any] = {}
if args.embedding_mode == "ollama":
opts["host"] = resolve_ollama_host(args.embedding_host)
elif args.embedding_mode == "openai":
opts["base_url"] = resolve_openai_base_url(args.embedding_api_base)
resolved_key = resolve_openai_api_key(args.embedding_api_key)
if resolved_key:
opts["api_key"] = resolved_key
if args.query_prompt_template:
if args.embedding_prompt_template:
opts["build_prompt_template"] = args.embedding_prompt_template
opts["query_prompt_template"] = args.query_prompt_template
elif args.embedding_prompt_template:
opts["prompt_template"] = args.embedding_prompt_template
batch_size = getattr(args, "embedding_batch_size", None)
if batch_size is not None:
opts["batch_size"] = batch_size
return opts
def _build_config_from_args(
self,
args,
docs_paths: list[str],
*,
doc_chunk_size: Optional[int] = None,
doc_chunk_overlap: Optional[int] = None,
code_chunk_size: Optional[int] = None,
code_chunk_overlap: Optional[int] = None,
) -> dict[str, Any]:
"""Capture the CLI build settings needed to replay `leann build`."""
embedding_options = self._build_embedding_options(args)
config: dict[str, Any] = {
"docs": [str(Path(path).resolve()) for path in docs_paths],
"file_types": args.file_types,
"include_hidden": bool(args.include_hidden),
"doc_chunk_size": doc_chunk_size
if doc_chunk_size is not None
else int(args.doc_chunk_size),
"doc_chunk_overlap": doc_chunk_overlap
if doc_chunk_overlap is not None
else int(args.doc_chunk_overlap),
"code_chunk_size": code_chunk_size
if code_chunk_size is not None
else int(args.code_chunk_size),
"code_chunk_overlap": code_chunk_overlap
if code_chunk_overlap is not None
else int(args.code_chunk_overlap),
"use_ast_chunking": bool(args.use_ast_chunking),
"ast_chunk_size": int(args.ast_chunk_size),
"ast_chunk_overlap": int(args.ast_chunk_overlap),
"ast_fallback_traditional": bool(args.ast_fallback_traditional),
"graph_degree": int(args.graph_degree),
"complexity": int(args.complexity),
"num_threads": int(args.num_threads),
"compact": bool(args.compact),
"recompute": bool(args.recompute),
}
if "host" in embedding_options:
config["embedding_host"] = embedding_options["host"]
if "base_url" in embedding_options:
config["embedding_api_base"] = embedding_options["base_url"]
if args.embedding_prompt_template:
config["embedding_prompt_template"] = args.embedding_prompt_template
if args.query_prompt_template:
config["query_prompt_template"] = args.query_prompt_template
return config
def _resolve_sync_scope(self, docs_paths: list[str]) -> tuple[list[str], list[str]]:
directories: list[str] = []
files: list[str] = []
for path in docs_paths:
path_obj = Path(path).resolve()
if path_obj.is_dir():
directories.append(str(path_obj))
elif path_obj.is_file():
files.append(str(path_obj))
return sorted(directories), sorted(files)
def _resolve_sync_roots(self, docs_paths: list[str]) -> list[str]:
"""Directory roots only (legacy). Prefer _resolve_sync_scope."""
directories, _files = self._resolve_sync_scope(docs_paths)
return directories
def _create_synchronizers(
self,
index_dir: Path,
directories: list[str],
explicit_files: list[str],
include_extensions: list[str],
include_hidden: bool = False,
) -> list[FileSynchronizer]:
"""Create FileSynchronizers with snapshots stored in the index dir. Shared by build and watch."""
synchronizers: list[FileSynchronizer] = []
for root in directories:
tag = hashlib.sha256(root.encode()).hexdigest()[:12]
snapshot_path = str(index_dir / f"sync_{tag}.pickle")
try:
fs = FileSynchronizer(
root_dir=root,
include_extensions=include_extensions,
include_hidden=include_hidden,
snapshot_path=snapshot_path,
)
synchronizers.append(fs)
except Exception as exc:
print(f"Warning: Failed to init synchronizer for {root}: {exc}")
if explicit_files:
tag = hashlib.sha256("|".join(explicit_files).encode()).hexdigest()[:12]
snapshot_path = str(index_dir / f"sync_files_{tag}.pickle")
try:
fs = FileSynchronizer(
explicit_files=explicit_files,
include_extensions=include_extensions,
include_hidden=include_hidden,
snapshot_path=snapshot_path,
)
synchronizers.append(fs)
except Exception as exc:
print(f"Warning: Failed to init synchronizer for explicit files: {exc}")
return synchronizers
def _build_synchronizers(
self,
docs_paths: list[str],
index_dir: Path,
file_types: Optional[str] = None,
include_hidden: bool = False,
) -> list[FileSynchronizer]:
"""Create FileSynchronizers for build from docs_paths."""
directories, files = self._resolve_sync_scope(docs_paths)
include_extensions = self._parse_file_types(file_types)
return self._create_synchronizers(
index_dir, directories, files, include_extensions, include_hidden
)
def _detect_build_changes(
self,
synchronizers: list[FileSynchronizer],
) -> tuple[set[str], set[str], set[str]]:
"""Detect added/removed/modified files across all source roots using content hashes."""
all_added: set[str] = set()
all_removed: set[str] = set()
all_modified: set[str] = set()
for fs in synchronizers:
added, removed, modified = fs.detect_changes()
all_added.update(added)
all_removed.update(removed)
all_modified.update(modified)
return all_added, all_removed, all_modified
def _commit_synchronizers(self, synchronizers: list[FileSynchronizer]) -> None:
"""Persist all synchronizer snapshots after a successful build."""
for fs in synchronizers:
fs.commit()
@staticmethod
def _assign_chunk_ids(chunks: list[dict]) -> None:
"""Assign stable IDs to chunks based on their file path and position."""
from collections import defaultdict
by_path: dict[str, list] = defaultdict(list)
for c in chunks:
p = c.get("metadata", {}).get("file_path") or c.get("metadata", {}).get("source") or ""
by_path[_normalize_path(p)].append(c)
for path_key, path_chunks in by_path.items():
for idx, c in enumerate(path_chunks):
sid = hashlib.sha256(f"{path_key}:{idx}".encode()).hexdigest()[:16]
c.setdefault("metadata", {})["id"] = sid
c["id"] = sid
@staticmethod
def _assign_unique_chunk_ids(chunks: list[dict]) -> None:
"""Assign unique IDs for incremental (avoids collision when path lookup misses some old ids)."""
for c in chunks:
sid = uuid.uuid4().hex[:16]
c.setdefault("metadata", {})["id"] = sid
c["id"] = sid
def _chunks_for_paths(self, all_texts: list[dict], paths: set[str]) -> list[dict]:
"""Filter chunks belonging to the given file paths."""
return [
c
for c in all_texts
if _normalize_path(
c.get("metadata", {}).get("file_path") or c.get("metadata", {}).get("source") or ""
)
in paths
]
def _existing_index_id_scheme(self, index_path: str) -> Optional[str]:
"""Return the passage_id_scheme recorded in an existing index's meta.json.
Returns None when the index doesn't exist yet. Older indexes pre-#330
have a meta file without passage_id_scheme and must be treated as
sequential so incremental updates never mix ID schemes.
"""
meta_path = Path(index_path).with_suffix(".leann.meta.json")
if not meta_path.exists():
return None
try:
with open(meta_path, encoding="utf-8") as f:
return json.load(f).get("passage_id_scheme", "sequential")
except Exception:
return None
def _make_incremental_builder(self, args) -> "LeannBuilder":
# For incremental updates, the existing index's scheme wins. Otherwise
# IDs would mix schemes within one index, which breaks lookups.
existing_scheme = self._existing_index_id_scheme(self.get_index_path(args.index_name))
scheme = existing_scheme or getattr(args, "id_scheme", "sequential")
if existing_scheme and getattr(args, "id_scheme", existing_scheme) != existing_scheme:
print(
f"Note: --id-scheme={args.id_scheme} ignored — index '{args.index_name}' "
f"was built with passage_id_scheme={existing_scheme!r}, keeping that."
)
return LeannBuilder(
backend_name=args.backend_name,
embedding_model=args.embedding_model,
embedding_mode=args.embedding_mode,
embedding_options=self._build_embedding_options(args) or None,
graph_degree=args.graph_degree,
complexity=args.complexity,
is_compact=args.compact,
is_recompute=args.recompute,
num_threads=args.num_threads,
passage_id_scheme=scheme,
)
def _incremental_add_only(
self,
index_path: str,
all_texts: list[dict],
args,
new_paths: set[str],
) -> bool:
"""Add-only incremental update (works for HNSW and IVF)."""
new_chunks = self._chunks_for_paths(all_texts, new_paths)
if not new_chunks:
return False
self._assign_chunk_ids(new_chunks)
builder = self._make_incremental_builder(args)
for chunk in new_chunks:
builder.add_text(chunk["text"], metadata=chunk["metadata"])
print(
f"Incremental update: adding {len(new_chunks)} chunks from {len(new_paths)} new file(s)..."
)
builder.update_index(index_path)
print(f"Index updated at {index_path}")
return True
def _incremental_ivf_remove_only(
self, index_path: str, index_dir: Path, removed_paths: set[str], args
) -> bool:
"""IVF remove-only fast path: remove chunk IDs without loading or chunking documents."""
passages_file = index_dir / "documents.leann.passages.jsonl"
if not passages_file.exists():
return False
offset_file = index_dir / "documents.leann.passages.idx"
live_ids: set[str] | None = None
if offset_file.exists():
with open(offset_file, "rb") as f:
live_ids = set(pickle.load(f).keys())
chunk_ids_by_file = self._load_chunk_ids_by_file(passages_file, live_ids=live_ids)
roots = self._load_sync_roots(index_dir)
ids_to_remove: list[str] = []
seen_ids: set[str] = set()
for p in removed_paths:
for key in self._path_lookup_keys(p, roots):
for pid in chunk_ids_by_file.get(key, []):
if pid not in seen_ids:
seen_ids.add(pid)
ids_to_remove.append(pid)
if not ids_to_remove:
return False
print(
f"Incremental IVF update (-{len(removed_paths)} removed): removing {len(ids_to_remove)} old chunks..."
)
builder = self._make_incremental_builder(args)
builder.update_index(index_path, remove_passage_ids=ids_to_remove)
print(f"Index updated at {index_path}")
return True
def _path_lookup_keys(self, path: str, roots: list[str]) -> list[str]:
"""Return path variations for lookup (sync paths may be relative to roots)."""
keys = [path, _normalize_path(path)]
for root in roots:
candidate = str((Path(root) / path).resolve())
if candidate not in keys:
keys.append(candidate)
return keys
def _incremental_ivf_update(
self,
index_path: str,
index_dir: Path,
all_texts: list[dict],
args,
new_paths: set[str],
removed_paths: set[str],
modified_paths: set[str],
sync_roots: list[str],
) -> bool:
"""IVF incremental update: remove old chunks for modified/removed files, add new chunks."""
passages_file = index_dir / "documents.leann.passages.jsonl"
offset_file = index_dir / "documents.leann.passages.idx"
live_ids: set[str] | None = None
if offset_file.exists():
with open(offset_file, "rb") as f:
live_ids = set(pickle.load(f).keys())
chunk_ids_by_file = (
self._load_chunk_ids_by_file(passages_file, live_ids=live_ids)
if passages_file.exists()
else {}
)
# Collect old chunk IDs to remove (modified + removed files)
# Try all path variations: same file can have different path formats in passages
ids_to_remove: list[str] = []
seen_ids: set[str] = set()
for p in modified_paths | removed_paths:
for key in self._path_lookup_keys(p, sync_roots):
for pid in chunk_ids_by_file.get(key, []):
if pid not in seen_ids:
seen_ids.add(pid)
ids_to_remove.append(pid)
# Collect new chunks to add (modified + new files)
# Build path set for matching: chunks may have paths in different formats
changed_paths = new_paths | modified_paths
path_set: set[str] = set()
for p in changed_paths:
path_set.update(self._path_lookup_keys(p, sync_roots))
new_chunks = self._chunks_for_paths(all_texts, path_set)
# Use unique IDs: passages can have mixed path formats so we may miss some ids_to_remove
self._assign_unique_chunk_ids(new_chunks)
if not ids_to_remove and not new_chunks:
return False
builder = self._make_incremental_builder(args)
for chunk in new_chunks:
builder.add_text(chunk["text"], metadata=chunk["metadata"])
parts = []
if ids_to_remove:
parts.append(f"removing {len(ids_to_remove)} old chunks")
if new_chunks:
parts.append(f"adding {len(new_chunks)} new chunks")
file_parts = []
if new_paths:
file_parts.append(f"+{len(new_paths)} added")
if modified_paths:
file_parts.append(f"~{len(modified_paths)} modified")
if removed_paths:
file_parts.append(f"-{len(removed_paths)} removed")
print(f"Incremental IVF update ({', '.join(file_parts)}): {', '.join(parts)}...")
builder.update_index(
index_path, remove_passage_ids=ids_to_remove if ids_to_remove else None
)
print(f"Index updated at {index_path}")
return True
@staticmethod
def _log_rebuild_reason(
meta: dict, args, new_paths: set, removed_paths: set, modified_paths: set
) -> None:
"""Print a human-readable explanation of why incremental update is not possible."""
if removed_paths or modified_paths:
reasons = []
if removed_paths:
reasons.append(f"{len(removed_paths)} file(s) removed")
if modified_paths:
reasons.append(f"{len(modified_paths)} file(s) modified")
print(
f"Incremental update not possible ({', '.join(reasons)}); falling back to full rebuild."
)
return
blockers = []
if meta.get("backend_name") not in ("hnsw", "ivf"):
blockers.append(
f"backend '{meta.get('backend_name')}' does not support incremental updates"
)
if meta.get("is_compact", meta.get("backend_kwargs", {}).get("is_compact", True)):
blockers.append("index is compact (read-only); rebuild with --no-compact to enable")
if meta.get("embedding_model") != args.embedding_model:
blockers.append(
f"embedding model changed ('{meta.get('embedding_model')}' -> '{args.embedding_model}')"
)
if meta.get("embedding_mode") != args.embedding_mode:
blockers.append(
f"embedding mode changed ('{meta.get('embedding_mode')}' -> '{args.embedding_mode}')"
)
if blockers:
print(
f"Incremental update not possible: {'; '.join(blockers)}. Falling back to full rebuild."
)
else:
changes = []
if new_paths:
changes.append(f"+{len(new_paths)} added")
summary = ", ".join(changes) if changes else "unknown reason"
print(f"Full rebuild starting ({summary})...")
def _write_sync_config(
self,
index_dir: Path,
directories: list[str],
explicit_files: list[str],
include_extensions: list[str],
include_hidden: bool,
build_config: Optional[dict[str, Any]] = None,
) -> None:
sync_config_path = index_dir / "sync_roots.json"
config = {
"directories": directories,
"files": explicit_files,
"roots": directories,
"include_extensions": include_extensions,
"ignore_patterns": self._sync_ignore_patterns(include_hidden),
}
if build_config is not None:
config["build_config"] = build_config
with open(sync_config_path, "w", encoding="utf-8") as f:
json.dump(config, f, indent=2)
def _write_sync_config_for_docs(
self,
index_dir: Path,
docs_paths: list[str],
args,
build_config: Optional[dict[str, Any]] = None,
) -> None:
directories, files = self._resolve_sync_scope(docs_paths)
self._write_sync_config(
index_dir,
directories,
files,
self._parse_file_types(args.file_types),
args.include_hidden,
build_config,
)
def _load_sync_scope(self, index_dir: Path) -> tuple[list[str], list[str], list[str], bool]:
"""Load sync scope from index dir (directories, files, extensions, include_hidden)."""
sync_config_path = index_dir / "sync_roots.json"
if not sync_config_path.exists():
return [], [], list(DEFAULT_INDEX_EXTENSIONS), False
try:
with open(sync_config_path, encoding="utf-8") as f:
config = json.load(f)
except (json.JSONDecodeError, OSError):
return [], [], list(DEFAULT_INDEX_EXTENSIONS), False
directories = config.get("directories") or config.get("roots") or []
files = config.get("files") or []
include_extensions = config.get("include_extensions") or list(DEFAULT_INDEX_EXTENSIONS)
include_hidden = config.get("ignore_patterns") is None
return directories, files, include_extensions, include_hidden
def _load_sync_roots(self, index_dir: Path) -> list[str]:
"""Load directory + explicit file paths for chunk ID lookup."""
directories, files, _, _ = self._load_sync_scope(index_dir)
return directories + files
def _resolve_index_for_watch(self, index_name: str) -> Optional[dict[str, Path]]:
if self.index_exists(index_name):
index_dir = self.indexes_dir / index_name
passages_file = index_dir / "documents.leann.passages.jsonl"
return {"index_dir": index_dir, "passages_file": passages_file}
all_matches = self._find_all_matching_indexes(index_name)
if not all_matches:
print(
f"Index '{index_name}' not found. Use 'leann build {index_name} --docs <dir> [<dir2> ...]' to create it."
)
return None
if len(all_matches) == 1:
match = all_matches[0]
else:
current_matches = [m for m in all_matches if m.get("is_current")]
match = current_matches[0] if current_matches else all_matches[0]
location_desc = (
"current project"
if match.get("is_current")
else f"project '{match['project_path'].name}'"
)
print(
f"Found {len(all_matches)} indexes named '{index_name}', using index from {location_desc}"
)
if match.get("kind") == "cli":
index_dir = match["index_dir"]
passages_file = index_dir / "documents.leann.passages.jsonl"
else:
index_dir = match["meta_file"].parent
file_base = match["file_base"]
passages_file = index_dir / f"{file_base}.passages.jsonl"
return {"index_dir": index_dir, "passages_file": passages_file}
def _load_chunk_ids_by_file(
self, passages_file: Path, live_ids: set[str] | None = None
) -> dict[str, list[str]]:
"""Load chunk IDs grouped by file path from passages.jsonl.
If *live_ids* is provided, skip entries whose ID is not in the set
(filters out stale entries left by prior incremental updates).
"""
chunk_ids_by_file: dict[str, list[str]] = {}
with open(passages_file, encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
try:
data = json.loads(line)
except json.JSONDecodeError:
continue
metadata = data.get("metadata") or {}
file_path = metadata.get("file_path") or metadata.get("source")
if not file_path:
continue
chunk_id = data.get("id")
if chunk_id is None:
continue
if live_ids is not None and str(chunk_id) not in live_ids:
continue
normalized_path = str(Path(file_path).resolve())
chunk_ids_by_file.setdefault(normalized_path, []).append(str(chunk_id))
if file_path != normalized_path:
chunk_ids_by_file.setdefault(file_path, []).append(str(chunk_id))
return chunk_ids_by_file
async def index_slack(self, args):
"""Build an index from Slack workspace via MCP."""
from .readers import SlackMCPReader
index_name = args.index_name
index_dir = self.indexes_dir / index_name
index_path = str(index_dir / "documents.leann")
print("📱 Indexing Slack workspace...")
reader = SlackMCPReader(
mcp_server_command=args.mcp_server,
workspace_name=args.workspace_name,
)
documents = await reader.load_data(channels=args.channels)
if not documents:
print("❌ No Slack data found. Make sure MCP server is running.")
return
print(f"📚 Loaded {len(documents)} messages. Building index...")
index_dir.mkdir(parents=True, exist_ok=True)
embedding_options = {}
if args.embedding_mode == "ollama":
embedding_options["host"] = resolve_ollama_host(None)
builder = LeannBuilder(
backend_name="hnsw",
embedding_model=args.embedding_model,
embedding_mode=args.embedding_mode,
embedding_options=embedding_options or None,
is_recompute=False,
is_compact=False,
)
for doc in documents:
builder.add_text(doc.text, metadata=doc.metadata)
builder.build_index(index_path)
print(f"✅ Slack index built at: {index_path}")
print(f' Usage: leann search {index_name} "query"')
async def build_index(self, args):
docs_paths = args.docs
# Use current directory name if index_name not provided
if args.index_name:
index_name = args.index_name
else:
index_name = Path.cwd().name
print(f"Using current directory name as index: '{index_name}'")
index_dir = self.indexes_dir / index_name
index_path = self.get_index_path(index_name)
# Display all paths being indexed with file/directory distinction
files = [p for p in docs_paths if Path(p).is_file()]
directories = [p for p in docs_paths if Path(p).is_dir()]
print(f"📂 Indexing {len(docs_paths)} path{'s' if len(docs_paths) > 1 else ''}:")
if files:
print(f" 📄 Files ({len(files)}):")
for i, file_path in enumerate(files, 1):
print(f" {i}. {Path(file_path).resolve()}")
if directories:
print(f" 📁 Directories ({len(directories)}):")
for i, dir_path in enumerate(directories, 1):
print(f" {i}. {Path(dir_path).resolve()}")
# Configure chunking based on CLI args before loading documents
# Guard against invalid configurations
doc_chunk_size = max(1, int(args.doc_chunk_size))
doc_chunk_overlap = max(0, int(args.doc_chunk_overlap))
code_chunk_size = max(1, int(args.code_chunk_size))
code_chunk_overlap = max(0, int(args.code_chunk_overlap))
# Text beyond the embedding model's sequence limit is never embedded, so cap
# chunk sizes to fit it (e.g. all-MiniLM-L6-v2 reads only 256 tokens).
from .chunking_utils import cap_chunk_size
from .embedding_compute import get_model_token_limit
embedding_options = self._build_embedding_options(args)
token_limit = get_model_token_limit(
args.embedding_model,
base_url=embedding_options.get("host") or embedding_options.get("base_url"),
)
capped_doc = cap_chunk_size(doc_chunk_size, token_limit)
capped_code = cap_chunk_size(code_chunk_size, token_limit)
if capped_doc < doc_chunk_size or capped_code < code_chunk_size:
print(
f"📏 {args.embedding_model} reads {token_limit} tokens: chunk size "
f"doc {doc_chunk_size} → {capped_doc}, code {code_chunk_size} → {capped_code}"
)
doc_chunk_overlap = min(doc_chunk_overlap, capped_doc // 2)
code_chunk_overlap = min(code_chunk_overlap, capped_code // 2)
doc_chunk_size, code_chunk_size = capped_doc, capped_code
if doc_chunk_overlap >= doc_chunk_size:
print(
f"⚠️ Adjusting doc chunk overlap from {doc_chunk_overlap} to {doc_chunk_size - 1} (must be < chunk size)"
)
doc_chunk_overlap = doc_chunk_size - 1
if code_chunk_overlap >= code_chunk_size:
print(
f"⚠️ Adjusting code chunk overlap from {code_chunk_overlap} to {code_chunk_size - 1} (must be < chunk size)"
)
code_chunk_overlap = code_chunk_size - 1
self.node_parser = SentenceSplitter(
chunk_size=doc_chunk_size,
chunk_overlap=doc_chunk_overlap,
separator=" ",
paragraph_separator="\n\n",
)
self.code_parser = SentenceSplitter(
chunk_size=code_chunk_size,
chunk_overlap=code_chunk_overlap,
separator="\n",
paragraph_separator="\n\n",
)
build_config = self._build_config_from_args(
args,
docs_paths,
doc_chunk_size=doc_chunk_size,
doc_chunk_overlap=doc_chunk_overlap,
code_chunk_size=code_chunk_size,
code_chunk_overlap=code_chunk_overlap,
)
# Detect changes first so we can skip load_documents for remove-only
index_dir.mkdir(parents=True, exist_ok=True)
synchronizers = self._build_synchronizers(
docs_paths, index_dir, file_types=args.file_types, include_hidden=args.include_hidden
)
if index_dir.exists() and not args.force and synchronizers:
meta_path = index_dir / "documents.leann.meta.json"
new_paths, removed_paths, modified_paths = self._detect_build_changes(synchronizers)
if not new_paths and not removed_paths and not modified_paths:
print("Index up to date.")
return
# Show change summary (add / modify / delete) before build
change_parts = []
if new_paths:
change_parts.append(f"+{len(new_paths)} added")
if modified_paths:
change_parts.append(f"~{len(modified_paths)} modified")
if removed_paths:
change_parts.append(f"-{len(removed_paths)} removed")
if change_parts:
print(f"Changes: {', '.join(change_parts)}")
if meta_path.exists():
with open(meta_path, encoding="utf-8") as f:
meta = json.load(f)
backend_name = meta.get("backend_name")
is_compact = meta.get(
"is_compact", meta.get("backend_kwargs", {}).get("is_compact", True)
)
# Normalize model names for comparison — e.g. "all-MiniLM-L6-v2"
# should match "sentence-transformers/all-MiniLM-L6-v2"
def _normalize_model_name(name: str) -> str:
return name.rsplit("/", 1)[-1] if name else name
same_embedding = (
_normalize_model_name(meta.get("embedding_model", ""))
== _normalize_model_name(args.embedding_model)
and meta.get("embedding_mode") == args.embedding_mode
)
# IVF supports remove+add, so it can handle modified and removed files incrementally
can_ivf_update = backend_name == "ivf" and not is_compact and same_embedding
# HNSW only supports add (no remove), so it needs add-only changes
can_add_only = (
not removed_paths
and not modified_paths
and backend_name in ("hnsw", "ivf")
and not is_compact
and same_embedding
)
# Remove-only fast path: no load/chunk, just remove IDs from index
if can_ivf_update and removed_paths and not new_paths and not modified_paths:
result = self._incremental_ivf_remove_only(
index_path, index_dir, removed_paths, args
)
if result:
self._commit_synchronizers(synchronizers)
self._write_sync_config_for_docs(index_dir, docs_paths, args, build_config)
self.register_project_dir()
return
# Resolve paths relative to sync scope (directories + explicit files).
sync_dirs, sync_files = self._resolve_sync_scope(docs_paths)
scope_roots = sync_dirs + sync_files
paths_to_load = new_paths | modified_paths
resolved_paths: list[str] = []
for p in paths_to_load:
path_obj = Path(p)
if path_obj.is_absolute() and path_obj.exists():
resolved_paths.append(p)
else:
for root in scope_roots:
candidate = Path(root) / p
if candidate.exists():
resolved_paths.append(str(candidate.resolve()))
break
else:
resolved_paths.append(p) # fallback: pass as-is
all_texts = self.load_documents(
resolved_paths,
args.file_types,
include_hidden=args.include_hidden,
args=args,
)
# Proceed even when all_texts is empty (e.g. file emptied): we still need to remove old chunks
if not all_texts and not (can_ivf_update and (modified_paths or removed_paths)):
print("No documents found")
return
if can_ivf_update and (new_paths or modified_paths or removed_paths):
result = self._incremental_ivf_update(
index_path,
index_dir,
all_texts,
args,
new_paths,
removed_paths,
modified_paths,
scope_roots,
)
if result:
self._commit_synchronizers(synchronizers)
self._write_sync_config_for_docs(index_dir, docs_paths, args, build_config)
self.register_project_dir()
return
elif can_add_only and new_paths:
result = self._incremental_add_only(
index_path,
all_texts,
args,
new_paths,
)
if result:
self._commit_synchronizers(synchronizers)
self._write_sync_config_for_docs(index_dir, docs_paths, args, build_config)
self.register_project_dir()
return
else:
self._log_rebuild_reason(meta, args, new_paths, removed_paths, modified_paths)
# Force full reload — the partial all_texts from incremental
# path only contains changed files, not the full corpus.
all_texts = self.load_documents(
docs_paths, args.file_types, include_hidden=args.include_hidden, args=args
)
# Full rebuild: load documents if not already loaded (first build or force)
try:
_ = all_texts
except NameError:
all_texts = self.load_documents(
docs_paths, args.file_types, include_hidden=args.include_hidden, args=args
)
if not all_texts:
print("No documents found")
return
print(f"Building index '{index_name}' with {args.backend_name} backend...")
publish_from_staging = _existing_index_artifacts(index_dir)
target_index_dir = index_dir
target_index_path = index_path
if publish_from_staging:
target_index_dir = index_dir.with_name(f".{index_dir.name}.rebuild-{uuid.uuid4().hex}")
target_index_path = str(target_index_dir / "documents.leann")
builder = LeannBuilder(
backend_name=args.backend_name,
embedding_model=args.embedding_model,
embedding_mode=args.embedding_mode,
embedding_options=self._build_embedding_options(args) or None,
graph_degree=args.graph_degree,
complexity=args.complexity,
is_compact=args.compact,
is_recompute=args.recompute,
num_threads=args.num_threads,
passage_id_scheme=getattr(args, "id_scheme", "sequential"),
)
for chunk in all_texts:
builder.add_text(chunk["text"], metadata=chunk["metadata"])
try:
builder.build_index(target_index_path)
target_synchronizers = (
self._build_synchronizers(
docs_paths,
target_index_dir,
file_types=args.file_types,
include_hidden=args.include_hidden,
)
if publish_from_staging
else synchronizers
)
for fs in target_synchronizers:
fs.create_snapshot()
self._write_sync_config_for_docs(target_index_dir, docs_paths, args, build_config)
if publish_from_staging:
_publish_rebuilt_index(target_index_dir, index_dir)
except Exception:
if publish_from_staging and target_index_dir.exists():
import shutil
shutil.rmtree(target_index_dir)
raise
print(f"Index built at {index_path}")
self.register_project_dir()
def _watch_check_changes(self, index_name: str) -> tuple[set[str], set[str], set[str]]:
"""Check for file changes using the same snapshots as build (index_dir)."""
resolved = self._resolve_index_for_watch(index_name)
if not resolved:
return set(), set(), set()
index_dir = resolved["index_dir"]
sync_config_path = index_dir / "sync_roots.json"
if not sync_config_path.exists():
return set(), set(), set()
directories, files, include_extensions, include_hidden = self._load_sync_scope(index_dir)
if not directories and not files:
return set(), set(), set()
synchronizers = self._create_synchronizers(
index_dir,
directories,
files,
include_extensions,
include_hidden,
)
return self._detect_build_changes(synchronizers)
def _watch_report_changes(
self,
index_name: str,
added: set[str],
removed: set[str],
modified: set[str],
) -> None:
"""Print a summary of detected file changes."""
resolved = self._resolve_index_for_watch(index_name)
passages_file = resolved["passages_file"] if resolved else None
chunk_ids_by_file: dict[str, list[str]] = {}
if passages_file and passages_file.exists():
chunk_ids_by_file = self._load_chunk_ids_by_file(passages_file)
print("\n=== Changes detected ===")
for label, paths in (
("added", sorted(added)),
("removed", sorted(removed)),
("modified", sorted(modified)),
):
if not paths:
continue
print(f"\n{label} ({len(paths)}):")
for file_path in paths:
normalized_path = str(Path(file_path).resolve())
chunk_ids = chunk_ids_by_file.get(normalized_path) or chunk_ids_by_file.get(
file_path, []
)
chunk_display = ", ".join(chunk_ids) if chunk_ids else "(not in index)"
print(f" - {file_path}")
print(f" chunks: {chunk_display}")
def _reconstruct_build_args(
self, index_name: str, *, force: bool = False, verbose: bool = False
) -> Optional[list[str]]:
"""Reconstruct the `leann build` CLI args for an existing index from its
stored config (.meta.json + sync_roots.json).
Returns None when the index can't be rebuilt this way (no sync config,
missing metadata, etc.). With verbose=True the reason gets printed (for
the user-driven `leann rebuild` path); with verbose=False the failures
are silent (for the watch loop, which polls and shouldn't spam logs).
"""
resolved = self._resolve_index_for_watch(index_name)
if not resolved:
return None
index_dir = resolved["index_dir"]
sync_config_path = index_dir / "sync_roots.json"
if not sync_config_path.exists():
if verbose:
print(
f"Cannot rebuild '{index_name}': no sync config at {sync_config_path}. "
f"This usually means the index was built via the Python API rather than "
f"`leann build --docs <dir>`, so the original docs roots aren't recorded."
)
return None
with open(sync_config_path, encoding="utf-8") as f:
config = json.load(f)
roots = config.get("roots") or []
if not roots:
if verbose:
print(f"Cannot rebuild '{index_name}': sync config has no document roots.")
return None
meta_path = index_dir / "documents.leann.meta.json"
if not meta_path.exists():
if verbose:
print(f"Cannot rebuild '{index_name}': index metadata missing at {meta_path}.")
else:
print(f"Index metadata missing for '{index_name}', cannot rebuild.")
return None
with open(meta_path, encoding="utf-8") as f:
meta = json.load(f)
build_config = config.get("build_config") or meta.get("build_config") or {}
docs = build_config.get("docs") or roots
build_args_list = [
"build",
index_name,
"--docs",
*[str(doc) for doc in docs],
"--backend-name",
meta.get("backend_name", "hnsw"),
"--embedding-model",
meta.get("embedding_model", "all-MiniLM-L6-v2"),
"--embedding-mode",
meta.get("embedding_mode", "sentence-transformers"),
]
bkw = meta.get("backend_kwargs", {})
def add_option(flag: str, value: Any) -> None:
if value is not None:
build_args_list.extend([flag, str(value)])
def add_bool(true_flag: str, false_flag: str, value: Any) -> None:
if value is None:
return
build_args_list.append(true_flag if bool(value) else false_flag)
def config_or_backend(config_key: str, backend_key: str) -> Any:
if config_key in build_config:
return build_config[config_key]
return bkw.get(backend_key)
add_option("--graph-degree", config_or_backend("graph_degree", "graph_degree"))
add_option("--complexity", config_or_backend("complexity", "complexity"))
add_option("--num-threads", config_or_backend("num_threads", "num_threads"))
compact = (
build_config["compact"]
if "compact" in build_config
else meta.get("is_compact", bkw.get("is_compact"))
)
recompute = (
build_config["recompute"]
if "recompute" in build_config
else meta.get("is_recompute", bkw.get("is_recompute"))
)
add_bool("--compact", "--no-compact", compact)
add_bool("--recompute", "--no-recompute", recompute)
file_types = build_config.get("file_types")
if file_types is None:
include_extensions = config.get("include_extensions")
if include_extensions:
file_types = ",".join(include_extensions)
add_option("--file-types", file_types)
include_hidden = build_config.get("include_hidden")
if include_hidden is None:
include_hidden = config.get("ignore_patterns") is None
if include_hidden:
build_args_list.append("--include-hidden")
add_option("--doc-chunk-size", build_config.get("doc_chunk_size"))
add_option("--doc-chunk-overlap", build_config.get("doc_chunk_overlap"))
add_option("--code-chunk-size", build_config.get("code_chunk_size"))
add_option("--code-chunk-overlap", build_config.get("code_chunk_overlap"))
if build_config.get("use_ast_chunking"):
build_args_list.append("--use-ast-chunking")
add_option("--ast-chunk-size", build_config.get("ast_chunk_size"))
add_option("--ast-chunk-overlap", build_config.get("ast_chunk_overlap"))
if build_config.get("ast_fallback_traditional"):
build_args_list.append("--ast-fallback-traditional")
embedding_options = meta.get("embedding_options") or {}
embedding_host = build_config.get("embedding_host", embedding_options.get("host"))
embedding_api_base = build_config.get(
"embedding_api_base", embedding_options.get("base_url")
)
embedding_prompt_template = build_config.get("embedding_prompt_template")
if embedding_prompt_template is None:
embedding_prompt_template = embedding_options.get(
"build_prompt_template", embedding_options.get("prompt_template")
)
query_prompt_template = build_config.get(
"query_prompt_template", embedding_options.get("query_prompt_template")
)
add_option("--embedding-host", embedding_host)
add_option("--embedding-api-base", embedding_api_base)
add_option("--embedding-prompt-template", embedding_prompt_template)
add_option("--query-prompt-template", query_prompt_template)
if force:
build_args_list.append("--force")
return build_args_list
async def rebuild_index(self, args) -> None:
"""Rebuild an existing index using its stored config.
By default this is an incremental delta (leann build is idempotent and
only re-indexes added/changed/removed files). Pass --force to do a full
rebuild from scratch.
"""
build_args_list = self._reconstruct_build_args(
args.index_name, force=getattr(args, "force", False), verbose=True
)
if build_args_list is None:
return
parser = self.create_parser()
build_args = parser.parse_args(build_args_list)
await self.build_index(build_args)
async def _watch_trigger_build(self, index_name: str) -> None:
"""Trigger an idempotent build for the given index, reusing its stored config."""
build_args_list = self._reconstruct_build_args(index_name, verbose=False)
if build_args_list is None:
return
parser = self.create_parser()
build_args = parser.parse_args(build_args_list)
await self.build_index(build_args)
def migrate_ids(self, args) -> None:
"""Rewrite an existing index's passage IDs to content-hash form.
Migration is purely a Python-side rewrite — the vector graph isn't
touched, so FAISS labels stay valid. What gets rewritten:
- <index>.passages.jsonl : new IDs in each line's "id" field
- <index>.passages.idx : new offset map keyed by new IDs
- <index>.ids.txt : new label → ID mapping for FAISS
- <index>.meta.json : passage_id_scheme = "content-hash"
- <index>.bm25.sqlite : BM25 IDs, when an FTS5 artifact is configured
Identical-text chunks collide on the same sha256 prefix; the later
occurrence wins in the offset map (dedup). A `--preserve-duplicates`
knob to suffix collisions can land separately.
Irreversible. Prompts for confirmation unless --yes is passed.
"""
import hashlib
import shutil
index_name = args.index_name
index_path = self._resolve_index_path(index_name, purpose="migrate")
if not index_path:
return
meta_path = Path(index_path).with_suffix(".leann.meta.json")
if not meta_path.exists():
print(f"Cannot migrate '{index_name}': metadata missing at {meta_path}.")
return
with open(meta_path, encoding="utf-8") as f:
meta = json.load(f)
current_scheme = meta.get("passage_id_scheme", "sequential")
if current_scheme == "content-hash":
print(f"Index '{index_name}' already uses content-hash IDs. Nothing to do.")
return
# Locate the sibling artifacts using the same conventions as build_index.
index_dir = Path(index_path).parent
index_base = Path(index_path).name
passages_file = index_dir / f"{index_base}.passages.jsonl"
offset_file = index_dir / f"{index_base}.passages.idx"
base_no_leann = (
index_base[: -len(".leann")] if index_base.endswith(".leann") else index_base
)
idmap_file = index_dir / f"{base_no_leann}.ids.txt"
for p in (passages_file, offset_file):
if not p.exists():
print(f"Cannot migrate: required artifact missing at {p}.")
return
# Stream the passages to plan the rewrite and surface collision count
# before committing to anything irreversible.
old_ids: list[str] = []
new_ids: list[str] = []
with open(passages_file, encoding="utf-8") as f:
for line in f:
if not line.strip():
continue
data = json.loads(line)
old_ids.append(data["id"])
new_ids.append(hashlib.sha256(data["text"].encode("utf-8")).hexdigest()[:16])
unique_new = len(set(new_ids))
collisions = len(new_ids) - unique_new
print(
f"Migrating '{index_name}': {len(new_ids)} passages → {unique_new} unique "
f"content-hash IDs ({collisions} collision(s) will dedup)."
)
if args.dry_run:
print("(dry-run; not writing anything)")
return
if not args.yes:
confirm = input(
"Proceed? This rewrites passages.jsonl, .idx, .ids.txt, .meta.json. [y/N] "
)
if confirm.strip().lower() not in ("y", "yes"):
print("Aborted.")
return
# Stage writes into siblings, then atomically rename.
new_passages = passages_file.with_suffix(passages_file.suffix + ".migrate")
new_offsets: dict[str, int] = {}
rewritten_passages: list[dict[str, Any]] = []
with (
open(passages_file, encoding="utf-8") as src,
open(new_passages, "w", encoding="utf-8") as dst,
):
idx = 0
for line in src:
if not line.strip():
continue
data = json.loads(line)
data["id"] = new_ids[idx]
offset = dst.tell()
json.dump(data, dst, ensure_ascii=False)
dst.write("\n")
new_offsets[new_ids[idx]] = offset
rewritten_passages.append(data)
idx += 1
new_idx = offset_file.with_suffix(offset_file.suffix + ".migrate")
with open(new_idx, "wb") as f:
pickle.dump(new_offsets, f)
new_idmap: Optional[Path] = None
if idmap_file.exists():
new_idmap = idmap_file.with_suffix(idmap_file.suffix + ".migrate")
with open(new_idmap, "w", encoding="utf-8") as f:
for nid in new_ids:
f.write(nid + "\n")
bm25_db_name = meta.get("bm25_db")
should_rebuild_bm25 = meta.get("bm25_backend") == "fts5"
if should_rebuild_bm25 and not bm25_db_name:
bm25_db_name = f"{index_base}.bm25.sqlite"
new_bm25: Optional[Path] = None
bm25_file: Optional[Path] = None
if should_rebuild_bm25 and bm25_db_name:
bm25_file = index_dir / bm25_db_name
new_bm25 = bm25_file.with_suffix(bm25_file.suffix + ".migrate")
bm25_index = Fts5BM25Index(str(new_bm25))
bm25_index.fit(rewritten_passages)
bm25_index.close()
shutil.move(str(new_passages), str(passages_file))
shutil.move(str(new_idx), str(offset_file))
if new_idmap is not None:
shutil.move(str(new_idmap), str(idmap_file))
if new_bm25 is not None and bm25_file is not None:
shutil.move(str(new_bm25), str(bm25_file))
meta["passage_id_scheme"] = "content-hash"
meta["version"] = "1.1"
if should_rebuild_bm25 and bm25_db_name:
meta["bm25_backend"] = "fts5"
meta["bm25_db"] = bm25_db_name
with open(meta_path, "w", encoding="utf-8") as f:
json.dump(meta, f, indent=2)
print(
f"✓ Migrated '{index_name}' to content-hash IDs. {collisions} collisions were deduped."
)
async def watch_index(self, args):
index_name = args.index_name
resolved = self._resolve_index_for_watch(index_name)
if not resolved:
return
index_dir = resolved["index_dir"]
sync_config_path = index_dir / "sync_roots.json"
if not sync_config_path.exists():
print(
f"Sync config not found for index '{index_name}'. "
f"Run 'leann build {index_name} --docs <dir>' first."
)
return
dry_run = getattr(args, "dry_run", False)
once = getattr(args, "once", False)
interval = getattr(args, "interval", 5)
if once:
added, removed, modified = self._watch_check_changes(index_name)
if not added and not removed and not modified:
print("No changes detected.")
return
self._watch_report_changes(index_name, added, removed, modified)
if not dry_run:
await self._watch_trigger_build(index_name)
return
print(f"Watching index '{index_name}' (interval={interval}s, ctrl-c to stop)...")
try:
while True:
added, removed, modified = self._watch_check_changes(index_name)
if added or removed or modified:
self._watch_report_changes(index_name, added, removed, modified)
if not dry_run:
await self._watch_trigger_build(index_name)
await asyncio.sleep(interval)
except KeyboardInterrupt:
print("\nWatch stopped.")
def _resolve_index_path(
self,
index_name: str,
*,
non_interactive: bool = True,
purpose: str = "use",
quiet: bool = False,
) -> Optional[str]:
"""Resolve index path from current project or registered projects."""
_print = (lambda *a, **kw: print(*a, file=sys.stderr, **kw)) if quiet else print
if self.index_exists(index_name):
return self.get_index_path(index_name)
all_matches = self._find_all_matching_indexes(index_name)
if not all_matches:
_print(
f"Index '{index_name}' not found. Use 'leann build {index_name} --docs <dir> [<dir2> ...]' to create it."
)
return None
def _match_to_path(match: dict[str, Any]) -> str:
if match["kind"] == "cli":
return str(match["index_dir"] / "documents.leann")
meta_file = match["meta_file"]
file_base = match["file_base"]
return str(meta_file.parent / f"{file_base}.leann")
if len(all_matches) == 1:
match = all_matches[0]
project_info = (
"current project"
if match["is_current"]
else f"project '{match['project_path'].name}'"
)
_print(f"Using index '{index_name}' from {project_info}")
return _match_to_path(match)
if non_interactive:
current_matches = [m for m in all_matches if m["is_current"]]
match = current_matches[0] if current_matches else all_matches[0]
location_desc = (
"current project"
if match["is_current"]
else f"project '{match['project_path'].name}'"
)
_print(
f"Found {len(all_matches)} indexes named '{index_name}', using index from {location_desc}"
)
return _match_to_path(match)
print(f"Found {len(all_matches)} indexes named '{index_name}':")
for i, match in enumerate(all_matches, 1):
project_path = match["project_path"]
is_current = match["is_current"]
kind = match.get("kind", "cli")
if is_current:
print(f" {i}. 🏠 Current project ({'CLI' if kind == 'cli' else 'APP'})")
else:
print(f" {i}. 📂 {project_path.name} ({'CLI' if kind == 'cli' else 'APP'})")
try:
choice = input(f"Which index to {purpose}? (1-{len(all_matches)}): ").strip()
choice_idx = int(choice) - 1
if 0 <= choice_idx < len(all_matches):
match = all_matches[choice_idx]
project_info = (
"current project"
if match["is_current"]
else f"project '{match['project_path'].name}'"
)
print(f"Using index '{index_name}' from {project_info}")
return _match_to_path(match)
print("Invalid choice. Aborting.")
return None
except (ValueError, KeyboardInterrupt):
print("Invalid input. Aborting.")
return None
async def search_documents(self, args):
index_name = args.index_name
query = args.query
json_mode = getattr(args, "json", False)
index_path = self._resolve_index_path(
index_name,
non_interactive=args.non_interactive,
purpose="search",
quiet=json_mode,
)
if not index_path:
return
# Build provider_options for runtime override
provider_options = {}
if args.embedding_prompt_template:
provider_options["prompt_template"] = args.embedding_prompt_template
# Parse --metadata-filters JSON string
metadata_filters = None
raw_filters = getattr(args, "metadata_filters", None)
if raw_filters:
try:
metadata_filters = json.loads(raw_filters)
if not isinstance(metadata_filters, dict):
print(
"Error: --metadata-filters must be a JSON object, "
'e.g. \'{"chapter": {"<=": 5}}\''
)
return
except json.JSONDecodeError as e:
print(f"Error: --metadata-filters is not valid JSON: {e}")
return
saved_fd = None
if json_mode:
sys.stdout.flush()
try:
saved_fd = os.dup(1)
devnull = os.open(os.devnull, os.O_WRONLY)
os.dup2(devnull, 1)
os.close(devnull)
except OSError:
saved_fd = None
try:
searcher = LeannSearcher(
index_path=index_path,
enable_warmup=args.enable_warmup,
use_daemon=args.use_daemon,
daemon_ttl_seconds=args.daemon_ttl,
)
results = searcher.search(
query,
top_k=args.top_k,
complexity=args.complexity,
beam_width=args.beam_width,
prune_ratio=args.prune_ratio,
recompute_embeddings=args.recompute_embeddings,
pruning_strategy=args.pruning_strategy,
provider_options=provider_options if provider_options else None,
metadata_filters=metadata_filters,
)
finally:
if saved_fd is not None:
sys.stdout.flush()
os.dup2(saved_fd, 1)
os.close(saved_fd)
if json_mode:
json_results = [
{
"id": r.id,
"score": float(r.score),
"text": r.text,
"metadata": r.metadata,
}
for r in results
]
print(json.dumps(json_results, ensure_ascii=False, indent=2))
return
print(f"Search results for '{query}' (top {len(results)}):")
for i, result in enumerate(results, 1):
print(f"{i}. Score: {result.score:.3f}")
if args.show_metadata and result.metadata:
file_path = result.metadata.get("file_path", "")
if file_path:
print(f" File: {file_path}")
file_name = result.metadata.get("file_name", "")
if file_name and file_name != file_path:
print(f" Name: {file_name}")
if "creation_date" in result.metadata:
print(f" Created: {result.metadata['creation_date']}")
if "last_modified_date" in result.metadata:
print(f" Modified: {result.metadata['last_modified_date']}")
print(f" {result.text}")
print(f" Source: {result.metadata.get('source', '')}")
print()
async def warmup_index(self, args):
index_path = self._resolve_index_path(
args.index_name,
non_interactive=True,
purpose="warm up",
)
if not index_path:
return
searcher = LeannSearcher(
index_path=index_path,
recompute_embeddings=True,
enable_warmup=args.enable_warmup,
use_daemon=args.use_daemon,
daemon_ttl_seconds=args.daemon_ttl,
)
if args.enable_warmup:
searcher.warmup()
print(
f"Warmed index '{args.index_name}' (daemon={'on' if args.use_daemon else 'off'}, ttl={args.daemon_ttl}s)"
)
async def daemon_command(self, args):
if not args.daemon_command:
print("Please specify one of: start, stop, status")
return
if args.daemon_command == "status":
records = EmbeddingServerManager.list_daemons()
if args.index_name:
index_path = self._resolve_index_path(
args.index_name,
non_interactive=True,
purpose="check daemon status for",
)
if not index_path:
return
meta_path = str(Path(f"{index_path}.meta.json").resolve())
records = [
r
for r in records
if r.get("config_signature", {}).get("passages_file") == meta_path
]
if not records:
print("No active embedding daemons.")
return
print(f"Active embedding daemons: {len(records)}")
for record in records:
cfg = record.get("config_signature", {})
print(
f"- pid={record.get('pid')} port={record.get('port')} backend={record.get('backend_module_name')} model={cfg.get('model_name')}"
)
return
if args.daemon_command == "start":
index_path = self._resolve_index_path(
args.index_name,
non_interactive=True,
purpose="start daemon for",
)
if not index_path:
return
searcher = LeannSearcher(
index_path=index_path,
recompute_embeddings=True,
enable_warmup=args.enable_warmup,
use_daemon=True,
daemon_ttl_seconds=args.daemon_ttl,
)
searcher.warmup()
print(
f"Daemon started for '{args.index_name}' (ttl={args.daemon_ttl}s, warmup={'on' if args.enable_warmup else 'off'})"
)
return
if args.daemon_command == "stop":
if args.all:
stopped = EmbeddingServerManager.stop_daemons()
print(f"Stopped {stopped} daemon(s).")
return
if not args.index_name:
print("Provide an index name or pass --all.")
return
index_path = self._resolve_index_path(
args.index_name,
non_interactive=True,
purpose="stop daemon for",
)
if not index_path:
return
meta_path = str(Path(f"{index_path}.meta.json").resolve())
stopped = EmbeddingServerManager.stop_daemons(passages_file=meta_path)
print(f"Stopped {stopped} daemon(s) for index '{args.index_name}'.")
async def ask_questions(self, args):
index_name = args.index_name
index_path = self.get_index_path(index_name)
if not self.index_exists(index_name):
print(
f"Index '{index_name}' not found. Use 'leann build {index_name} --docs <dir> [<dir2> ...]' to create it."
)
return
print(f"Starting chat with index '{index_name}'...")
print(f"Using {args.model} ({args.llm})")
llm_config = {"type": args.llm, "model": args.model}
if args.llm == "ollama":
llm_config["host"] = resolve_ollama_host(args.host)
elif args.llm == "openai":
llm_config["base_url"] = resolve_openai_base_url(args.api_base)
resolved_api_key = resolve_openai_api_key(args.api_key)
if resolved_api_key:
llm_config["api_key"] = resolved_api_key
elif args.llm == "anthropic":
# For Anthropic, pass base_url and API key if provided
if args.api_base:
llm_config["base_url"] = resolve_anthropic_base_url(args.api_base)
if args.api_key:
llm_config["api_key"] = args.api_key
elif args.llm == "litellm":
# LiteLLM infers the provider from the model prefix and reads that
# provider's own env var; base_url/api_key are only for a proxy.
base_url = resolve_litellm_base_url(args.api_base)
if base_url:
llm_config["base_url"] = base_url
resolved_api_key = resolve_litellm_api_key(args.api_key)
if resolved_api_key:
llm_config["api_key"] = resolved_api_key
elif args.llm == "minimax":
llm_config["base_url"] = resolve_minimax_base_url(args.api_base)
resolved_api_key = resolve_minimax_api_key(args.api_key)
if resolved_api_key:
llm_config["api_key"] = resolved_api_key
elif args.llm in {"atlascloud", "atlas-cloud", "atlas"}:
llm_config["type"] = "atlascloud"
if args.model == "qwen3:8b":
llm_config["model"] = "deepseek-ai/deepseek-v4-pro"
llm_config["base_url"] = resolve_atlascloud_base_url(args.api_base)
resolved_api_key = resolve_atlascloud_api_key(args.api_key)
if resolved_api_key:
llm_config["api_key"] = resolved_api_key
# Parse --metadata-filters JSON string (mirrors `leann search`).
# Run before constructing LeannChat so invalid filters fail fast without
# spinning up an embedding server.
metadata_filters = None
raw_filters = getattr(args, "metadata_filters", None)
if raw_filters:
try:
metadata_filters = json.loads(raw_filters)
if not isinstance(metadata_filters, dict):
print(
"Error: --metadata-filters must be a JSON object, "
'e.g. \'{"chapter": {"<=": 5}}\''
)
return
except json.JSONDecodeError as e:
print(f"Error: --metadata-filters is not valid JSON: {e}")
return
chat = LeannChat(index_path=index_path, llm_config=llm_config)
llm_kwargs: dict[str, Any] = {}
if args.thinking_budget:
llm_kwargs["thinking_budget"] = args.thinking_budget
def _ask_once(prompt: str) -> None:
query_start_time = time.time()
response = chat.ask(
prompt,
top_k=args.top_k,
complexity=args.complexity,
beam_width=args.beam_width,
prune_ratio=args.prune_ratio,
recompute_embeddings=args.recompute_embeddings,
pruning_strategy=args.pruning_strategy,
metadata_filters=metadata_filters,
llm_kwargs=llm_kwargs,
)
query_completion_time = time.time() - query_start_time
print(f"LEANN: {response}")
print(f"The query took {query_completion_time:.3f} seconds to finish")
initial_query = (args.query or "").strip()
if args.interactive:
# Create interactive session
session = create_cli_session(index_name)
if initial_query:
_ask_once(initial_query)
session.run_interactive_loop(_ask_once)
else:
query = initial_query or input("Enter your question: ").strip()
if not query:
print("No question provided. Exiting.")
return
_ask_once(query)
# ── index-* shared handler ──────────────────────────────────────
async def _build_index_from_documents(self, args, documents: list):
"""Shared builder for all index-* commands.
Accepts a list of llama-index Document objects (or dicts with 'text' and
'metadata' keys), builds a LEANN index with proper embedding and
recomputation settings.
"""
if not documents:
print("No documents loaded — nothing to index.")
return
index_name = args.index_name
index_path = self.get_index_path(index_name)
is_recompute = not getattr(args, "no_recompute", False)
print(f"Building index '{index_name}' with {len(documents)} documents...")
print(f" Embedding: {args.embedding_model} ({args.embedding_mode})")
print(f" Recompute: {is_recompute}")
for attr in (
"embedding_prompt_template",
"query_prompt_template",
"embedding_host",
"embedding_api_base",
"embedding_api_key",
):
if not hasattr(args, attr):
setattr(args, attr, None)
embedding_options = self._build_embedding_options(args) or None
builder = LeannBuilder(
backend_name="hnsw",
embedding_model=args.embedding_model,
embedding_mode=args.embedding_mode,
embedding_options=embedding_options,
is_recompute=is_recompute,
)
for doc in documents:
if hasattr(doc, "text"):
builder.add_text(
doc.text, metadata=doc.metadata if hasattr(doc, "metadata") else {}
)
elif isinstance(doc, dict):
builder.add_text(doc["text"], metadata=doc.get("metadata", {}))
builder.build_index(index_path)
self.register_project_dir()
print(f"Index '{index_name}' built at {index_path}")
async def index_browser(self, args):
"""Index browser history (Chrome or Brave)."""
from apps.history_data.history import ChromeHistoryReader
browser = getattr(args, "browser", "chrome")
profile_paths = {
"chrome": "~/Library/Application Support/Google/Chrome/Default",
"brave": "~/Library/Application Support/BraveSoftware/Brave-Browser/Default",
}
profile = os.path.expanduser(profile_paths.get(browser, profile_paths["chrome"]))
reader = ChromeHistoryReader()
docs = reader.load_data(chrome_profile_path=profile, max_count=args.max_count)
print(f"Loaded {len(docs)} {browser} history entries")
await self._build_index_from_documents(args, docs)
async def index_email(self, args):
"""Index Apple Mail."""
from apps.email_data.LEANN_email_reader import EmlxReader, find_all_messages_directories
msg_dirs = find_all_messages_directories()
if not msg_dirs:
print("No Apple Mail Messages directories found.")
return
reader = EmlxReader()
docs = []
for msg_dir in msg_dirs:
docs.extend(reader.load_data(str(msg_dir), max_count=args.max_count))
await self._build_index_from_documents(args, docs)
async def index_calendar(self, args):
"""Index Apple Calendar events."""
import shutil
import sqlite3
from llama_index.core import Document
docs = []
calendar_cache = Path.home() / "Library/Calendars/Calendar Cache"
if not calendar_cache.exists():
print("Apple Calendar Cache not found.")
return
temp_db = "/tmp/leann_calendar_index_copy"
try:
shutil.copy2(calendar_cache, temp_db)
conn = sqlite3.connect(temp_db)
cursor = conn.cursor()
cursor.execute(
"""
SELECT summary, description, location,
datetime(start_date + 978307200, 'unixepoch', 'localtime') as start,
datetime(end_date + 978307200, 'unixepoch', 'localtime') as end_time
FROM CI_EVENT ORDER BY start_date DESC LIMIT ?
""",
(args.max_count,),
)
for summary, description, location, start, end_time in cursor.fetchall():
if not summary:
continue
text = (
f"Event: {summary}\nStart: {start}\nEnd: {end_time}\n"
f"Location: {location or ''}\nDescription: {description or ''}"
)
docs.append(Document(text=text, metadata={"event": summary, "start": start}))
conn.close()
except Exception as e:
print(f"Error reading Apple Calendar: {e}")
finally:
if os.path.exists(temp_db):
os.remove(temp_db)
await self._build_index_from_documents(args, docs)
async def index_imessage(self, args):
"""Index iMessage conversations."""
from apps.imessage_data.imessage_reader import IMessageReader
reader = IMessageReader(concatenate_conversations=True)
docs = reader.load_data()
print(f"Loaded {len(docs)} iMessage conversations")
await self._build_index_from_documents(args, docs)
async def index_wechat(self, args):
"""Index WeChat chat history from exported JSON."""
from apps.history_data.wechat_history import WeChatHistoryReader
reader = WeChatHistoryReader()
docs = reader.load_data(
input_dir=args.export_dir,
max_count=args.max_count,
concatenate_messages=True,
)
print(f"Loaded {len(docs)} WeChat conversations")
await self._build_index_from_documents(args, docs)
async def index_chatgpt(self, args):
"""Index ChatGPT export data."""
from apps.chatgpt_data.chatgpt_reader import ChatGPTReader
reader = ChatGPTReader(concatenate_conversations=True)
docs = reader.load_data(
input_dir=args.export_path,
max_count=args.max_count,
)
print(f"Loaded {len(docs)} ChatGPT conversations")
await self._build_index_from_documents(args, docs)
async def index_claude(self, args):
"""Index Claude export data."""
from apps.claude_data.claude_reader import ClaudeReader
reader = ClaudeReader(concatenate_conversations=True)
docs = reader.load_data(
input_dir=args.export_path,
max_count=args.max_count,
)
print(f"Loaded {len(docs)} Claude conversations")
await self._build_index_from_documents(args, docs)
async def react_agent(self, args):
"""Run ReAct agent for multiturn retrieval."""
index_name = args.index_name
query = args.query
# Find the index (similar to search_documents)
index_path = self.get_index_path(index_name)
if self.index_exists(index_name):
pass
else:
all_matches = self._find_all_matching_indexes(index_name)
if not all_matches:
print(
f"Index '{index_name}' not found. Use 'leann build {index_name} --docs <dir> [<dir2> ...]' to create it."
)
return
elif len(all_matches) == 1:
match = all_matches[0]
if match["kind"] == "cli":
index_path = str(match["index_dir"] / "documents.leann")
else:
meta_file = match["meta_file"]
file_base = match["file_base"]
index_path = str(meta_file.parent / f"{file_base}.leann")
else:
# Multiple matches - use first one for now
match = all_matches[0]
if match["kind"] == "cli":
index_path = str(match["index_dir"] / "documents.leann")
else:
meta_file = match["meta_file"]
file_base = match["file_base"]
index_path = str(meta_file.parent / f"{file_base}.leann")
print(f"Found {len(all_matches)} indexes named '{index_name}', using first match")
print(f"🤖 Starting ReAct agent with index '{index_name}'...")
print(f"Using {args.model} ({args.llm})")
llm_config = {"type": args.llm, "model": args.model}
if args.llm == "ollama":
llm_config["host"] = resolve_ollama_host(args.host)
elif args.llm == "openai":
llm_config["base_url"] = resolve_openai_base_url(args.api_base)
resolved_api_key = resolve_openai_api_key(args.api_key)
if resolved_api_key:
llm_config["api_key"] = resolved_api_key
elif args.llm == "anthropic":
if args.api_base:
llm_config["base_url"] = resolve_anthropic_base_url(args.api_base)
if args.api_key:
llm_config["api_key"] = args.api_key
elif args.llm == "litellm":
# LiteLLM infers the provider from the model prefix and reads that
# provider's own env var; base_url/api_key are only for a proxy.
base_url = resolve_litellm_base_url(args.api_base)
if base_url:
llm_config["base_url"] = base_url
resolved_api_key = resolve_litellm_api_key(args.api_key)
if resolved_api_key:
llm_config["api_key"] = resolved_api_key
elif args.llm == "minimax":
llm_config["base_url"] = resolve_minimax_base_url(args.api_base)
resolved_api_key = resolve_minimax_api_key(args.api_key)
if resolved_api_key:
llm_config["api_key"] = resolved_api_key
elif args.llm in {"atlascloud", "atlas-cloud", "atlas"}:
llm_config["type"] = "atlascloud"
if args.model == "qwen3:8b":
llm_config["model"] = "deepseek-ai/deepseek-v4-pro"
llm_config["base_url"] = resolve_atlascloud_base_url(args.api_base)
resolved_api_key = resolve_atlascloud_api_key(args.api_key)
if resolved_api_key:
llm_config["api_key"] = resolved_api_key
from .react_agent import create_react_agent
agent = create_react_agent(
index_path=index_path,
llm_config=llm_config,
max_iterations=args.max_iterations,
serper_api_key=getattr(args, "serper_api_key", None),
jina_api_key=getattr(args, "jina_api_key", None),
)
if agent.web_search_available:
print("🌐 Web search enabled (Serper API key detected)")
else:
print("📚 Local search only (no Serper API key)")
print(f"\n🔍 Question: {query}\n")
answer = agent.run(query, top_k=args.top_k)
print(f"\n✅ Final Answer:\n{answer}\n")
if agent.search_history:
print(f"\n📊 Search History ({len(agent.search_history)} iterations):")
for entry in agent.search_history:
source_tag = f"[{entry.get('source', 'local')}]"
print(
f" {entry['iteration']}. {source_tag} {entry['action']} "
f"({entry['results_count']} results)"
)
async def serve_api(self, args):
"""Start the HTTP API server."""
import os
try:
from .server import serve_async
# Override host/port if provided via CLI args
if args.host:
os.environ["LEANN_SERVER_HOST"] = args.host
if args.port:
os.environ["LEANN_SERVER_PORT"] = str(args.port)
# Must await uvicorn on this loop: sync uvicorn.run() starts a nested loop
# and fails with "Cannot run the event loop while another loop is running".
await serve_async()
except ImportError as e:
print(
"❌ HTTP server dependencies not installed.\n"
"Install them with:\n"
" uv pip install 'leann-core[server]'\n"
"or:\n"
" uv pip install 'fastapi>=0.115' 'pydantic>=2' 'uvicorn[standard]'\n"
)
raise SystemExit(1) from e
except Exception as e:
print(f"❌ Error starting server: {e}")
raise SystemExit(1) from e
async def run(self, args=None):
parser = self.create_parser()
if args is None:
args = parser.parse_args()
if not args.command:
parser.print_help()
return
# Determine whether to suppress C++ output
# Default is to suppress (quiet mode), unless --verbose is specified
suppress = not getattr(args, "verbose", False)
if args.command == "list":
self.list_indexes(args.max_depth)
elif args.command == "remove":
self.remove_index(args.index_name, args.force)
elif args.command == "build":
with suppress_cpp_output(suppress):
await self.build_index(args)
elif args.command == "watch":
await self.watch_index(args)
elif args.command == "migrate-ids":
self.migrate_ids(args)
elif args.command == "rebuild":
with suppress_cpp_output(suppress):
await self.rebuild_index(args)
elif args.command == "search":
with suppress_cpp_output(suppress):
await self.search_documents(args)
elif args.command == "warmup":
with suppress_cpp_output(suppress):
await self.warmup_index(args)
elif args.command == "daemon":
await self.daemon_command(args)
elif args.command == "ask":
with suppress_cpp_output(suppress):
await self.ask_questions(args)
elif args.command == "react":
with suppress_cpp_output(suppress):
await self.react_agent(args)
elif args.command == "serve":
await self.serve_api(args)
elif args.command == "index-browser":
with suppress_cpp_output(suppress):
await self.index_browser(args)
elif args.command == "index-email":
with suppress_cpp_output(suppress):
await self.index_email(args)
elif args.command == "index-calendar":
with suppress_cpp_output(suppress):
await self.index_calendar(args)
elif args.command == "index-wechat":
with suppress_cpp_output(suppress):
await self.index_wechat(args)
elif args.command == "index-imessage":
with suppress_cpp_output(suppress):
await self.index_imessage(args)
elif args.command == "index-slack":
with suppress_cpp_output(suppress):
await self.index_slack(args)
elif args.command == "index-chatgpt":
with suppress_cpp_output(suppress):
await self.index_chatgpt(args)
elif args.command == "index-claude":
with suppress_cpp_output(suppress):
await self.index_claude(args)
else:
parser.print_help()
def main():
import logging
import dotenv
dotenv.load_dotenv()
# Set clean logging for CLI usage
logging.getLogger().setLevel(logging.WARNING) # Only show warnings and errors
cli = LeannCLI()
asyncio.run(cli.run())
if __name__ == "__main__":
main()