Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

fix mem compare fix split

yichuan520030910320326783f7f140
diff --git a/examples/faiss_only.py b/examples/faiss_only.py
index bd2822a..28d9db6 100644
--- a/examples/faiss_only.py
+++ b/examples/faiss_only.py
@@ -77,13 +77,7 @@ def main():
chunk_size=256, chunk_overlap=20, separator=" ", paragraph_separator="\n\n"
)
- all_texts = []
- for doc in documents:
- nodes = node_parser.get_nodes_from_documents([doc])
- for node in nodes:
- all_texts.append(node.get_content())
-
- tracker.checkpoint("After text chunking")
+ tracker.checkpoint("After text splitter setup")
# Check if index already exists and try to load it
index_loaded = False
@@ -115,7 +109,9 @@ def main():
vector_store = FaissVectorStore(faiss_index=faiss_index)
storage_context = StorageContext.from_defaults(vector_store=vector_store)
index = VectorStoreIndex.from_documents(
- documents, storage_context=storage_context
+ documents,
+ storage_context=storage_context,
+ transformations=[node_parser]
)
tracker.checkpoint("After index building")