[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
fix(chunking): keep first line of AST chunks that contains '|' (#444)
ShryukGrandhib51ddc0b1f80
diff --git a/packages/leann-core/src/leann/chunking_utils.py b/packages/leann-core/src/leann/chunking_utils.pyindex d79428a..0493c3c 100644--- a/packages/leann-core/src/leann/chunking_utils.py+++ b/packages/leann-core/src/leann/chunking_utils.py@@ -4,7 +4,6 @@ Packaged within leann-core so installed wheels can import it reliably."""import logging-import refrom pathlib import Pathfrom typing import Any, Optional@@ -291,16 +290,7 @@ def create_ast_chunks(# Merge document metadata + astchunk metadatacombined_metadata = {**doc_metadata, **astchunk_metadata}- # Trim partial first line left by overlap if using line numbers- # (a valid line starts with digits followed by '|')- stripped = chunk_text.strip()- if stripped:- first_line = stripped.split("\n", 1)[0]- if "|" in first_line and not re.match(r"^\s*\d+\|", first_line):- first_nl = stripped.find("\n")- if first_nl != -1:- stripped = stripped[first_nl + 1 :]- all_chunks.append({"text": stripped, "metadata": combined_metadata})+ all_chunks.append({"text": chunk_text.strip(), "metadata": combined_metadata})logger.info(f"Created {len(chunks)} AST chunks from {language} file: {doc.metadata.get('file_name', 'unknown')}"diff --git a/tests/test_astchunk_integration.py b/tests/test_astchunk_integration.pyindex 1f2010b..7051fb8 100644--- a/tests/test_astchunk_integration.py+++ b/tests/test_astchunk_integration.py@@ -625,6 +625,22 @@ class TestASTContentExtraction:# Should not have stringified the empty dictassert "'content': ''" not in chunk["text"]+ def test_first_line_containing_pipe_is_kept(self):+ """A chunk whose first line contains '|' (e.g. a union type hint) must not lose it."""+ code = "def merge(a: dict | None) -> dict:\n return a or {}"+ mock_builder = Mock()+ mock_builder.chunkify.return_value = [+ {"content": code, "metadata": {"start_line_no": 0, "end_line_no": 1}}+ ]+ mock_astchunk = Mock()+ mock_astchunk.ASTChunkBuilder = Mock(return_value=mock_builder)++ doc = MockDocument(code, "/test/merge.py", {"language": "python"})+ with patch.dict("sys.modules", {"astchunk": mock_astchunk}):+ chunks = create_ast_chunks([doc])++ assert [c["text"] for c in chunks] == [code]+class TestASTMetadataPreservation:"""Test metadata preservation in AST chunk dictionaries.