Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

fix: move line number prepending after AST chunking and fix index existence check (#245)

Yichuan Wangdd5efa6cce76
diff --git a/.gitignore b/.gitignore
index 8254b65..47d53ef 100755
--- a/.gitignore
+++ b/.gitignore
@@ -110,4 +110,5 @@ apps/multimodal/vision-based-pdf-multi-vector/multi-vector-colpali-native-weavia
# AUR build directory (Arch Linux)
paru-bin/
merkle-tree-test/
+test-code/
*.pickle
diff --git a/apps/base_rag_example.py b/apps/base_rag_example.py
index 1517191..caa0e93 100644
--- a/apps/base_rag_example.py
+++ b/apps/base_rag_example.py
@@ -391,7 +391,7 @@ class BaseRAGExample(ABC):
# Check if index exists
index_path = str(Path(args.index_dir) / f"{self.default_index_name}.leann")
- index_exists = Path(args.index_dir).exists()
+ index_exists = Path(f"{index_path}.meta.json").exists()
if not index_exists or args.force_rebuild:
# Load data and build index
diff --git a/packages/leann-core/src/leann/chunking_utils.py b/packages/leann-core/src/leann/chunking_utils.py
index 1f083a2..b220007 100644
--- a/packages/leann-core/src/leann/chunking_utils.py
+++ b/packages/leann-core/src/leann/chunking_utils.py
@@ -274,13 +274,7 @@ def create_ast_chunks(
# Merge document metadata + astchunk metadata
combined_metadata = {**doc_metadata, **astchunk_metadata}
- # Trim partial first line left by overlap
- stripped = chunk_text.strip()
- if stripped and not stripped[0].isdigit():
- first_nl = stripped.find("\n")
- if first_nl != -1:
- stripped = stripped[first_nl + 1 :]
- all_chunks.append({"text": stripped, "metadata": combined_metadata})
+ all_chunks.append({"text": chunk_text.strip(), "metadata": combined_metadata})
logger.info(
f"Created {len(chunks)} AST chunks from {language} file: {doc.metadata.get('file_name', 'unknown')}"
@@ -391,24 +385,22 @@ def create_text_chunks(
all_chunks = []
if use_ast_chunking:
code_docs, text_docs = detect_code_files(documents, local_code_extensions)
- # Prepend line numbers to code documents for navigation
- if code_docs:
- from llama_index.core.schema import MediaResource
-
- for doc in code_docs:
- original = doc.get_content()
- lines = original.split("\n")
- w = len(str(len(lines)))
- doc.text_resource = MediaResource(
- text="\n".join(f"{i + 1:>{w}}|{line}" for i, line in enumerate(lines))
- )
if code_docs:
try:
- all_chunks.extend(
- create_ast_chunks(
- code_docs, max_chunk_size=ast_chunk_size, chunk_overlap=ast_chunk_overlap
- )
+ ast_chunks = create_ast_chunks(
+ code_docs, max_chunk_size=ast_chunk_size, chunk_overlap=ast_chunk_overlap
)
+ # Prepend line numbers to code chunks for navigation
+ for chunk in ast_chunks:
+ start_line = chunk.get("metadata", {}).get("start_line_no")
+ if start_line is not None:
+ lines = chunk["text"].split("\n")
+ end_line = start_line + len(lines) - 1
+ w = len(str(end_line))
+ chunk["text"] = "\n".join(
+ f"{start_line + i:>{w}}|{line}" for i, line in enumerate(lines)
+ )
+ all_chunks.extend(ast_chunks)
except Exception as e:
logger.error(f"AST chunking failed: {e}")
if ast_fallback_traditional: