[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
perf: switch to tranditional pdf reader
Andy Lee53c58fa7559f
diff --git a/examples/main_cli_example.py b/examples/main_cli_example.pyindex fc87cfc..77b0bec 100644--- a/examples/main_cli_example.py+++ b/examples/main_cli_example.py@@ -3,10 +3,7 @@ faulthandler.enable()import argparsefrom llama_index.core import SimpleDirectoryReader, Settings-from llama_index.core.readers.base import BaseReader-from llama_index.node_parser.docling import DoclingNodeParser-from llama_index.readers.docling import DoclingReader-from docling_core.transforms.chunker.hybrid_chunker import HybridChunker+from llama_index.core.node_parser import SentenceSplitterimport asyncioimport dotenvfrom leann.api import LeannBuilder, LeannSearcher, LeannChat@@ -15,25 +12,18 @@ from pathlib import Pathdotenv.load_dotenv()-reader = DoclingReader(export_type=DoclingReader.ExportType.JSON)-file_extractor: dict[str, BaseReader] = {- ".docx": reader,- ".pptx": reader,- ".pdf": reader,- ".xlsx": reader,- ".txt": reader,- ".md": reader,-}-node_parser = DoclingNodeParser(- chunker=HybridChunker(tokenizer="facebook/contriever", max_tokens=128)+node_parser = SentenceSplitter(+ chunk_size=256,+ chunk_overlap=20,+ separator=" ",+ paragraph_separator="\n\n")print("Loading documents...")documents = SimpleDirectoryReader("examples/data",- recursive=True,- file_extractor=file_extractor,+ recursive=True,encoding="utf-8",- required_exts=[".pdf", ".docx", ".pptx", ".xlsx", ".txt", ".md"]+ required_exts=[".pdf", ".txt", ".md"]).load_data(show_progress=True)print("Documents loaded.")all_texts = []