Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

fix(chunking): keep first line of AST chunks that contains '|' (#444)

ShryukGrandhib51ddc0b1f80
diff --git a/packages/leann-core/src/leann/chunking_utils.py b/packages/leann-core/src/leann/chunking_utils.py
index d79428a..0493c3c 100644
--- a/packages/leann-core/src/leann/chunking_utils.py
+++ b/packages/leann-core/src/leann/chunking_utils.py
@@ -4,7 +4,6 @@ Packaged within leann-core so installed wheels can import it reliably.
"""
import logging
-import re
from pathlib import Path
from typing import Any, Optional
@@ -291,16 +290,7 @@ def create_ast_chunks(
# Merge document metadata + astchunk metadata
combined_metadata = {**doc_metadata, **astchunk_metadata}
- # Trim partial first line left by overlap if using line numbers
- # (a valid line starts with digits followed by '|')
- stripped = chunk_text.strip()
- if stripped:
- first_line = stripped.split("\n", 1)[0]
- if "|" in first_line and not re.match(r"^\s*\d+\|", first_line):
- first_nl = stripped.find("\n")
- if first_nl != -1:
- stripped = stripped[first_nl + 1 :]
- all_chunks.append({"text": stripped, "metadata": combined_metadata})
+ all_chunks.append({"text": chunk_text.strip(), "metadata": combined_metadata})
logger.info(
f"Created {len(chunks)} AST chunks from {language} file: {doc.metadata.get('file_name', 'unknown')}"
diff --git a/tests/test_astchunk_integration.py b/tests/test_astchunk_integration.py
index 1f2010b..7051fb8 100644
--- a/tests/test_astchunk_integration.py
+++ b/tests/test_astchunk_integration.py
@@ -625,6 +625,22 @@ class TestASTContentExtraction:
# Should not have stringified the empty dict
assert "'content': ''" not in chunk["text"]
+ def test_first_line_containing_pipe_is_kept(self):
+ """A chunk whose first line contains '|' (e.g. a union type hint) must not lose it."""
+ code = "def merge(a: dict | None) -> dict:\n return a or {}"
+ mock_builder = Mock()
+ mock_builder.chunkify.return_value = [
+ {"content": code, "metadata": {"start_line_no": 0, "end_line_no": 1}}
+ ]
+ mock_astchunk = Mock()
+ mock_astchunk.ASTChunkBuilder = Mock(return_value=mock_builder)
+
+ doc = MockDocument(code, "/test/merge.py", {"language": "python"})
+ with patch.dict("sys.modules", {"astchunk": mock_astchunk}):
+ chunks = create_ast_chunks([doc])
+
+ assert [c["text"] for c in chunks] == [code]
+
class TestASTMetadataPreservation:
"""Test metadata preservation in AST chunk dictionaries.