[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
Fix: Prevent duplicate PDF processing when using --file-types .pdf (#179)
Aakash Sureshe268392d5b0c
diff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.pyindex 122ae6b..79b5e1d 100644--- a/packages/leann-core/src/leann/cli.py+++ b/packages/leann-core/src/leann/cli.py@@ -1180,6 +1180,11 @@ Examples:print(f"Warning: Could not process {file_path}: {e}")# Load other file types with default reader+ # Exclude PDFs from code_extensions if they were already processed separately+ other_file_extensions = code_extensions+ if should_process_pdfs and ".pdf" in code_extensions:+ other_file_extensions = [ext for ext in code_extensions if ext != ".pdf"]+try:# Create a custom file filter function using our PathSpecdef file_filter(@@ -1195,15 +1200,19 @@ Examples:except (ValueError, OSError):return True # Include files that can't be processed- other_docs = SimpleDirectoryReader(- docs_dir,- recursive=True,- encoding="utf-8",- required_exts=code_extensions,- file_extractor={}, # Use default extractors- exclude_hidden=not include_hidden,- filename_as_id=True,- ).load_data(show_progress=True)+ # Only load other file types if there are extensions to process+ if other_file_extensions:+ other_docs = SimpleDirectoryReader(+ docs_dir,+ recursive=True,+ encoding="utf-8",+ required_exts=other_file_extensions,+ file_extractor={}, # Use default extractors+ exclude_hidden=not include_hidden,+ filename_as_id=True,+ ).load_data(show_progress=True)+ else:+ other_docs = []# Filter documents after loading based on gitignore rulesfiltered_docs = []