[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
feat: support multiple input formats for --docs argument (#39)
Yichuan Wangeab13434efc0
diff --git a/README.md b/README.mdindex 112584c..06403ae 100755--- a/README.md+++ b/README.md@@ -468,7 +468,7 @@ leann --help### Usage Examples```bash-# build from a specific directory, and my_docs is the index name+# build from a specific directory, and my_docs is the index name(Here you can also build from multiple dict or multiple files)leann build my-docs --docs ./your_documents# Search your documentsdiff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.pyindex 31dca55..9f6911f 100644--- a/packages/leann-core/src/leann/cli.py+++ b/packages/leann-core/src/leann/cli.py@@ -5,6 +5,7 @@ from typing import Unionfrom llama_index.core import SimpleDirectoryReaderfrom llama_index.core.node_parser import SentenceSplitter+from tqdm import tqdmfrom .api import LeannBuilder, LeannChat, LeannSearcher@@ -75,11 +76,14 @@ class LeannCLI:formatter_class=argparse.RawDescriptionHelpFormatter,epilog="""Examples:- leann build my-docs --docs ./documents # Build index named my-docs- leann build my-ppts --docs ./ --file-types .pptx,.pdf # Index only PowerPoint and PDF files- leann search my-docs "query" # Search in my-docs index- leann ask my-docs "question" # Ask my-docs index- leann list # List all stored indexes+ leann build my-docs --docs ./documents # Build index from directory+ leann build my-code --docs ./src ./tests ./config # Build index from multiple directories+ leann build my-files --docs ./file1.py ./file2.txt ./docs/ # Build index from files and directories+ leann build my-mixed --docs ./readme.md ./src/ ./config.json # Build index from mixed files/dirs+ leann build my-ppts --docs ./ --file-types .pptx,.pdf # Index only PowerPoint and PDF files+ leann search my-docs "query" # Search in my-docs index+ leann ask my-docs "question" # Ask my-docs index+ leann list # List all stored indexes""",)@@ -91,7 +95,11 @@ Examples:"index_name", nargs="?", help="Index name (default: current directory name)")build_parser.add_argument(- "--docs", type=str, default=".", help="Documents directory (default: current directory)"+ "--docs",+ type=str,+ nargs="+",+ default=["."],+ help="Documents directories and/or files (default: current directory)",)build_parser.add_argument("--backend", type=str, default="hnsw", choices=["hnsw", "diskann"]@@ -235,6 +243,32 @@ Examples:"""Check if a file should be excluded using gitignore parser."""return gitignore_matches(str(relative_path))+ def _is_git_submodule(self, path: Path) -> bool:+ """Check if a path is a git submodule."""+ try:+ # Find the git repo root+ current_dir = Path.cwd()+ while current_dir != current_dir.parent:+ if (current_dir / ".git").exists():+ gitmodules_path = current_dir / ".gitmodules"+ if gitmodules_path.exists():+ # Read .gitmodules to check if this path is a submodule+ gitmodules_content = gitmodules_path.read_text()+ # Convert path to relative to git root+ try:+ relative_path = path.resolve().relative_to(current_dir)+ # Check if this path appears in .gitmodules+ return f"path = {relative_path}" in gitmodules_content+ except ValueError:+ # Path is not under git root+ return False+ break+ current_dir = current_dir.parent+ return False+ except Exception:+ # If anything goes wrong, assume it's not a submodule+ return False+def list_indexes(self):print("Stored LEANN indexes:")@@ -264,7 +298,9 @@ Examples:valid_projects.append(current_path)if not valid_projects:- print("No indexes found. Use 'leann build <name> --docs <dir>' to create one.")+ print(+ "No indexes found. Use 'leann build <name> --docs <dir> [<dir2> ...]' to create one."+ )returntotal_indexes = 0@@ -311,56 +347,88 @@ Examples:print(f' leann search {example_name} "your query"')print(f" leann ask {example_name} --interactive")- def load_documents(self, docs_dir: str, custom_file_types: Union[str, None] = None):- print(f"Loading documents from {docs_dir}...")- if custom_file_types:- print(f"Using custom file types: {custom_file_types}")-- # Build gitignore parser- gitignore_matches = self._build_gitignore_parser(docs_dir)-- # Try to use better PDF parsers first, but only if PDFs are requested- documents = []- docs_path = Path(docs_dir)+ def load_documents(+ self, docs_paths: Union[str, list], custom_file_types: Union[str, None] = None+ ):+ # Handle both single path (string) and multiple paths (list) for backward compatibility+ if isinstance(docs_paths, str):+ docs_paths = [docs_paths]++ # Separate files and directories+ files = []+ directories = []+ for path in docs_paths:+ path_obj = Path(path)+ if path_obj.is_file():+ files.append(str(path_obj))+ elif path_obj.is_dir():+ # Check if this is a git submodule - if so, skip it+ if self._is_git_submodule(path_obj):+ print(f"⚠️ Skipping git submodule: {path}")+ continue+ directories.append(str(path_obj))+ else:+ print(f"⚠️ Warning: Path '{path}' does not exist, skipping...")+ continue- # Check if we should process PDFs- should_process_pdfs = custom_file_types is None or ".pdf" in custom_file_types+ # Print summary of what we're processing+ total_items = len(files) + len(directories)+ items_desc = []+ if files:+ items_desc.append(f"{len(files)} file{'s' if len(files) > 1 else ''}")+ if directories:+ items_desc.append(+ f"{len(directories)} director{'ies' if len(directories) > 1 else 'y'}"+ )- if should_process_pdfs:- for file_path in docs_path.rglob("*.pdf"):- # Check if file matches any exclude pattern- relative_path = file_path.relative_to(docs_path)- if self._should_exclude_file(relative_path, gitignore_matches):- continue+ print(f"Loading documents from {' and '.join(items_desc)} ({total_items} total):")+ if files:+ print(f" 📄 Files: {', '.join([Path(f).name for f in files])}")+ if directories:+ print(f" 📁 Directories: {', '.join(directories)}")- print(f"Processing PDF: {file_path}")+ if custom_file_types:+ print(f"Using custom file types: {custom_file_types}")- # Try PyMuPDF first (best quality)- text = extract_pdf_text_with_pymupdf(str(file_path))- if text is None:- # Try pdfplumber- text = extract_pdf_text_with_pdfplumber(str(file_path))+ all_documents = []- if text:- # Create a simple document structure- from llama_index.core import Document+ # First, process individual files if any+ if files:+ print(f"\n🔄 Processing {len(files)} individual file{'s' if len(files) > 1 else ''}...")- doc = Document(text=text, metadata={"source": str(file_path)})- documents.append(doc)- else:- # Fallback to default reader- print(f"Using default reader for {file_path}")+ # Load individual files using SimpleDirectoryReader with input_files+ # Note: We skip gitignore filtering for explicitly specified files+ try:+ # Group files by their parent directory for efficient loading+ from collections import defaultdict++ files_by_dir = defaultdict(list)+ for file_path in files:+ parent_dir = str(Path(file_path).parent)+ files_by_dir[parent_dir].append(file_path)++ # Load files from each parent directory+ for parent_dir, file_list in files_by_dir.items():+ print(+ f" Loading {len(file_list)} file{'s' if len(file_list) > 1 else ''} from {parent_dir}"+ )try:- default_docs = SimpleDirectoryReader(- str(file_path.parent),+ file_docs = SimpleDirectoryReader(+ parent_dir,+ input_files=file_list,filename_as_id=True,- required_exts=[file_path.suffix],).load_data()- documents.extend(default_docs)+ all_documents.extend(file_docs)+ print(+ f" ✅ Loaded {len(file_docs)} document{'s' if len(file_docs) > 1 else ''}"+ )except Exception as e:- print(f"Warning: Could not process {file_path}: {e}")+ print(f" ❌ Warning: Could not load files from {parent_dir}: {e}")++ except Exception as e:+ print(f"❌ Error processing individual files: {e}")- # Load other file types with default reader+ # Define file extensions to processif custom_file_types:# Parse custom file types from comma-separated stringcode_extensions = [ext.strip() for ext in custom_file_types.split(",") if ext.strip()]@@ -422,41 +490,106 @@ Examples:".py",".jl",]- # Try to load other file types, but don't fail if none are found- try:- # Create a custom file filter function using our PathSpec- def file_filter(file_path: str) -> bool:- """Return True if file should be included (not excluded)"""- try:- docs_path_obj = Path(docs_dir)- file_path_obj = Path(file_path)- relative_path = file_path_obj.relative_to(docs_path_obj)- return not self._should_exclude_file(relative_path, gitignore_matches)- except (ValueError, OSError):- return True # Include files that can't be processed-- other_docs = SimpleDirectoryReader(- docs_dir,- recursive=True,- encoding="utf-8",- required_exts=code_extensions,- file_extractor={}, # Use default extractors- filename_as_id=True,- ).load_data(show_progress=True)-- # Filter documents after loading based on gitignore rules- filtered_docs = []- for doc in other_docs:- file_path = doc.metadata.get("file_path", "")- if file_filter(file_path):- filtered_docs.append(doc)-- documents.extend(filtered_docs)- except ValueError as e:- if "No files found" in str(e):- print("No additional files found for other supported types.")- else:- raise e++ # Process each directory+ if directories:+ print(+ f"\n🔄 Processing {len(directories)} director{'ies' if len(directories) > 1 else 'y'}..."+ )++ for docs_dir in directories:+ print(f"Processing directory: {docs_dir}")+ # Build gitignore parser for each directory+ gitignore_matches = self._build_gitignore_parser(docs_dir)++ # Try to use better PDF parsers first, but only if PDFs are requested+ documents = []+ docs_path = Path(docs_dir)++ # Check if we should process PDFs+ should_process_pdfs = custom_file_types is None or ".pdf" in custom_file_types++ if should_process_pdfs:+ for file_path in docs_path.rglob("*.pdf"):+ # Check if file matches any exclude pattern+ try:+ relative_path = file_path.relative_to(docs_path)+ if self._should_exclude_file(relative_path, gitignore_matches):+ continue+ except ValueError:+ # Skip files that can't be made relative to docs_path+ print(f"⚠️ Skipping file outside directory scope: {file_path}")+ continue++ print(f"Processing PDF: {file_path}")++ # Try PyMuPDF first (best quality)+ text = extract_pdf_text_with_pymupdf(str(file_path))+ if text is None:+ # Try pdfplumber+ text = extract_pdf_text_with_pdfplumber(str(file_path))++ if text:+ # Create a simple document structure+ from llama_index.core import Document++ doc = Document(text=text, metadata={"source": str(file_path)})+ documents.append(doc)+ else:+ # Fallback to default reader+ print(f"Using default reader for {file_path}")+ try:+ default_docs = SimpleDirectoryReader(+ str(file_path.parent),+ filename_as_id=True,+ required_exts=[file_path.suffix],+ ).load_data()+ documents.extend(default_docs)+ except Exception as e:+ print(f"Warning: Could not process {file_path}: {e}")++ # Load other file types with default reader+ try:+ # Create a custom file filter function using our PathSpec+ def file_filter(+ file_path: str, docs_dir=docs_dir, gitignore_matches=gitignore_matches+ ) -> bool:+ """Return True if file should be included (not excluded)"""+ try:+ docs_path_obj = Path(docs_dir)+ file_path_obj = Path(file_path)+ relative_path = file_path_obj.relative_to(docs_path_obj)+ return not self._should_exclude_file(relative_path, gitignore_matches)+ except (ValueError, OSError):+ return True # Include files that can't be processed++ other_docs = SimpleDirectoryReader(+ docs_dir,+ recursive=True,+ encoding="utf-8",+ required_exts=code_extensions,+ file_extractor={}, # Use default extractors+ filename_as_id=True,+ ).load_data(show_progress=True)++ # Filter documents after loading based on gitignore rules+ filtered_docs = []+ for doc in other_docs:+ file_path = doc.metadata.get("file_path", "")+ if file_filter(file_path):+ filtered_docs.append(doc)++ documents.extend(filtered_docs)+ except ValueError as e:+ if "No files found" in str(e):+ print(f"No additional files found for other supported types in {docs_dir}.")+ else:+ raise e++ all_documents.extend(documents)+ print(f"Loaded {len(documents)} documents from {docs_dir}")++ documents = all_documentsall_texts = []@@ -507,7 +640,9 @@ Examples:".jl",}- for doc in documents:+ print("start chunking documents")+ # Add progress bar for document chunking+ for doc in tqdm(documents, desc="Chunking documents", unit="doc"):# Check if this is a code file based on source pathsource_path = doc.metadata.get("source", "")is_code_file = any(source_path.endswith(ext) for ext in code_file_exts)@@ -523,7 +658,7 @@ Examples:return all_textsasync def build_index(self, args):- docs_dir = args.docs+ docs_paths = args.docs# Use current directory name if index_name not providedif args.index_name:index_name = args.index_name@@ -534,13 +669,25 @@ Examples:index_dir = self.indexes_dir / index_nameindex_path = self.get_index_path(index_name)- print(f"📂 Indexing: {Path(docs_dir).resolve()}")+ # Display all paths being indexed with file/directory distinction+ files = [p for p in docs_paths if Path(p).is_file()]+ directories = [p for p in docs_paths if Path(p).is_dir()]++ print(f"📂 Indexing {len(docs_paths)} path{'s' if len(docs_paths) > 1 else ''}:")+ if files:+ print(f" 📄 Files ({len(files)}):")+ for i, file_path in enumerate(files, 1):+ print(f" {i}. {Path(file_path).resolve()}")+ if directories:+ print(f" 📁 Directories ({len(directories)}):")+ for i, dir_path in enumerate(directories, 1):+ print(f" {i}. {Path(dir_path).resolve()}")if index_dir.exists() and not args.force:print(f"Index '{index_name}' already exists. Use --force to rebuild.")return- all_texts = self.load_documents(docs_dir, args.file_types)+ all_texts = self.load_documents(docs_paths, args.file_types)if not all_texts:print("No documents found")return@@ -576,7 +723,7 @@ Examples:if not self.index_exists(index_name):print(- f"Index '{index_name}' not found. Use 'leann build {index_name} --docs <dir>' to create it."+ f"Index '{index_name}' not found. Use 'leann build {index_name} --docs <dir> [<dir2> ...]' to create it.")return@@ -603,7 +750,7 @@ Examples:if not self.index_exists(index_name):print(- f"Index '{index_name}' not found. Use 'leann build {index_name} --docs <dir>' to create it."+ f"Index '{index_name}' not found. Use 'leann build {index_name} --docs <dir> [<dir2> ...]' to create it.")returndiff --git a/packages/leann-mcp/README.md b/packages/leann-mcp/README.mdindex b762ae9..eb86488 100644--- a/packages/leann-mcp/README.md+++ b/packages/leann-mcp/README.md@@ -45,6 +45,42 @@ leann build my-project --docs ./claude```+## 🚀 Advanced Usage Examples++### Index Entire Git Repository+```bash+# Index all tracked files in your git repository, note right now we will skip submodules, but we can add it back easily if you want+leann build my-repo --docs $(git ls-files) --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw++# Index only specific file types from git+leann build my-python-code --docs $(git ls-files "*.py") --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw+```++### Multiple Directories and Files+```bash+# Index multiple directories+leann build my-codebase --docs ./src ./tests ./docs ./config --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw++# Mix files and directories+leann build my-project --docs ./README.md ./src/ ./package.json ./docs/ --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw++# Specific files only+leann build my-configs --docs ./tsconfig.json ./package.json ./webpack.config.js --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw+```++### Advanced Git Integration+```bash+# Index recently modified files+leann build recent-changes --docs $(git diff --name-only HEAD~10..HEAD) --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw++# Index files matching pattern+leann build frontend --docs $(git ls-files "*.tsx" "*.ts" "*.jsx" "*.js") --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw++# Index documentation and config files+leann build docs-and-configs --docs $(git ls-files "*.md" "*.yml" "*.yaml" "*.json" "*.toml") --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw+```++**Try this in Claude Code:**```Help me understand this codebase. List available indexes and search for authentication patterns.