[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
[Fix] Enable AST chunking when installed (package chunking utils) (#101)
Andy Leee93c0dec6f67
diff --git a/apps/chunking/__init__.py b/apps/chunking/__init__.pyindex 3cd5c0d..2f323e3 100644--- a/apps/chunking/__init__.py+++ b/apps/chunking/__init__.py@@ -1,16 +1,38 @@+"""Unified chunking utilities facade.++This module re-exports the packaged utilities from `leann.chunking_utils` so+that both repo apps (importing `chunking`) and installed wheels share one+single implementation. When running from the repo without installation, it+adds the `packages/leann-core/src` directory to `sys.path` as a fallback."""-Chunking utilities for LEANN RAG applications.-Provides AST-aware and traditional text chunking functionality.-"""-from .utils import (- CODE_EXTENSIONS,- create_ast_chunks,- create_text_chunks,- create_traditional_chunks,- detect_code_files,- get_language_from_extension,-)+import sys+from pathlib import Path++try:+ from leann.chunking_utils import (+ CODE_EXTENSIONS,+ create_ast_chunks,+ create_text_chunks,+ create_traditional_chunks,+ detect_code_files,+ get_language_from_extension,+ )+except Exception: # pragma: no cover - best-effort fallback for dev environment+ repo_root = Path(__file__).resolve().parents[2]+ leann_src = repo_root / "packages" / "leann-core" / "src"+ if leann_src.exists():+ sys.path.insert(0, str(leann_src))+ from leann.chunking_utils import (+ CODE_EXTENSIONS,+ create_ast_chunks,+ create_text_chunks,+ create_traditional_chunks,+ detect_code_files,+ get_language_from_extension,+ )+ else:+ raise__all__ = ["CODE_EXTENSIONS",diff --git a/apps/history_data/history.py b/apps/history_data/history.pyindex bb2eac1..f7e23c7 100644--- a/apps/history_data/history.py+++ b/apps/history_data/history.py@@ -74,7 +74,7 @@ class ChromeHistoryReader(BaseReader):if count >= max_count and max_count > 0:break- last_visit, url, title, visit_count, typed_count, hidden = row+ last_visit, url, title, visit_count, typed_count, _hidden = row# Create document content with metadata embedded in textdoc_content = f"""diff --git a/apps/chunking/utils.py b/packages/leann-core/src/leann/chunking_utils.pysimilarity index 56%rename from apps/chunking/utils.pyrename to packages/leann-core/src/leann/chunking_utils.pyindex 9a19c63..db80a39 100644--- a/apps/chunking/utils.py+++ b/packages/leann-core/src/leann/chunking_utils.py@@ -1,6 +1,6 @@"""Enhanced chunking utilities with AST-aware code chunking support.-Provides unified interface for both traditional and AST-based text chunking.+Packaged within leann-core so installed wheels can import it reliably."""import logging@@ -22,30 +22,9 @@ CODE_EXTENSIONS = {".jsx": "typescript",}-# Default chunk parameters for different content types-DEFAULT_CHUNK_PARAMS = {- "code": {- "max_chunk_size": 512,- "chunk_overlap": 64,- },- "text": {- "chunk_size": 256,- "chunk_overlap": 128,- },-}-def detect_code_files(documents, code_extensions=None) -> tuple[list, list]:- """- Separate documents into code files and regular text files.-- Args:- documents: List of LlamaIndex Document objects- code_extensions: Dict mapping file extensions to languages (defaults to CODE_EXTENSIONS)-- Returns:- Tuple of (code_documents, text_documents)- """+ """Separate documents into code files and regular text files."""if code_extensions is None:code_extensions = CODE_EXTENSIONS@@ -53,16 +32,10 @@ def detect_code_files(documents, code_extensions=None) -> tuple[list, list]:text_docs = []for doc in documents:- # Get file path from metadata- file_path = doc.metadata.get("file_path", "")- if not file_path:- # Fallback to file_name- file_path = doc.metadata.get("file_name", "")-+ file_path = doc.metadata.get("file_path", "") or doc.metadata.get("file_name", "")if file_path:file_ext = Path(file_path).suffix.lower()if file_ext in code_extensions:- # Add language info to metadatadoc.metadata["language"] = code_extensions[file_ext]doc.metadata["is_code"] = Truecode_docs.append(doc)@@ -70,7 +43,6 @@ def detect_code_files(documents, code_extensions=None) -> tuple[list, list]:doc.metadata["is_code"] = Falsetext_docs.append(doc)else:- # If no file path, treat as textdoc.metadata["is_code"] = Falsetext_docs.append(doc)@@ -79,7 +51,7 @@ def detect_code_files(documents, code_extensions=None) -> tuple[list, list]:def get_language_from_extension(file_path: str) -> Optional[str]:- """Get the programming language from file extension."""+ """Return language string from a filename/extension using CODE_EXTENSIONS."""ext = Path(file_path).suffix.lower()return CODE_EXTENSIONS.get(ext)@@ -90,40 +62,26 @@ def create_ast_chunks(chunk_overlap: int = 64,metadata_template: str = "default",) -> list[str]:- """- Create AST-aware chunks from code documents using astchunk.-- Args:- documents: List of code documents- max_chunk_size: Maximum characters per chunk- chunk_overlap: Number of AST nodes to overlap between chunks- metadata_template: Template for chunk metadata+ """Create AST-aware chunks from code documents using astchunk.- Returns:- List of text chunks with preserved code structure+ Falls back to traditional chunking if astchunk is unavailable."""try:- from astchunk import ASTChunkBuilder+ from astchunk import ASTChunkBuilder # optional dependencyexcept ImportError as e:logger.error(f"astchunk not available: {e}")logger.info("Falling back to traditional chunking for code files")return create_traditional_chunks(documents, max_chunk_size, chunk_overlap)all_chunks = []-for doc in documents:- # Get language from metadata (set by detect_code_files)language = doc.metadata.get("language")if not language:- logger.warning(- "No language detected for document, falling back to traditional chunking"- )- traditional_chunks = create_traditional_chunks([doc], max_chunk_size, chunk_overlap)- all_chunks.extend(traditional_chunks)+ logger.warning("No language detected; falling back to traditional chunking")+ all_chunks.extend(create_traditional_chunks([doc], max_chunk_size, chunk_overlap))continuetry:- # Configure astchunkconfigs = {"max_chunk_size": max_chunk_size,"language": language,@@ -131,7 +89,6 @@ def create_ast_chunks("chunk_overlap": chunk_overlap if chunk_overlap > 0 else 0,}- # Add repository-level metadata if availablerepo_metadata = {"file_path": doc.metadata.get("file_path", ""),"file_name": doc.metadata.get("file_name", ""),@@ -140,17 +97,13 @@ def create_ast_chunks(}configs["repo_level_metadata"] = repo_metadata- # Create chunk builder and processchunk_builder = ASTChunkBuilder(**configs)code_content = doc.get_content()-if not code_content or not code_content.strip():logger.warning("Empty code content, skipping")continuechunks = chunk_builder.chunkify(code_content)-- # Extract text content from chunksfor chunk in chunks:if hasattr(chunk, "text"):chunk_text = chunk.text@@ -159,7 +112,6 @@ def create_ast_chunks(elif isinstance(chunk, str):chunk_text = chunkelse:- # Try to convert to stringchunk_text = str(chunk)if chunk_text and chunk_text.strip():@@ -168,12 +120,10 @@ def create_ast_chunks(logger.info(f"Created {len(chunks)} AST chunks from {language} file: {doc.metadata.get('file_name', 'unknown')}")-except Exception as e:logger.warning(f"AST chunking failed for {language} file: {e}")logger.info("Falling back to traditional chunking")- traditional_chunks = create_traditional_chunks([doc], max_chunk_size, chunk_overlap)- all_chunks.extend(traditional_chunks)+ all_chunks.extend(create_traditional_chunks([doc], max_chunk_size, chunk_overlap))return all_chunks@@ -181,23 +131,10 @@ def create_ast_chunks(def create_traditional_chunks(documents, chunk_size: int = 256, chunk_overlap: int = 128) -> list[str]:- """- Create traditional text chunks using LlamaIndex SentenceSplitter.-- Args:- documents: List of documents to chunk- chunk_size: Size of each chunk in characters- chunk_overlap: Overlap between chunks-- Returns:- List of text chunks- """- # Handle invalid chunk_size values+ """Create traditional text chunks using LlamaIndex SentenceSplitter."""if chunk_size <= 0:logger.warning(f"Invalid chunk_size={chunk_size}, using default value of 256")chunk_size = 256-- # Ensure chunk_overlap is not negative and not larger than chunk_sizeif chunk_overlap < 0:chunk_overlap = 0if chunk_overlap >= chunk_size:@@ -215,12 +152,9 @@ def create_traditional_chunks(try:nodes = node_parser.get_nodes_from_documents([doc])if nodes:- chunk_texts = [node.get_content() for node in nodes]- all_texts.extend(chunk_texts)- logger.debug(f"Created {len(chunk_texts)} traditional chunks from document")+ all_texts.extend(node.get_content() for node in nodes)except Exception as e:logger.error(f"Traditional chunking failed for document: {e}")- # As last resort, add the raw contentcontent = doc.get_content()if content and content.strip():all_texts.append(content.strip())@@ -238,32 +172,13 @@ def create_text_chunks(code_file_extensions: Optional[list[str]] = None,ast_fallback_traditional: bool = True,) -> list[str]:- """- Create text chunks from documents with optional AST support for code files.-- Args:- documents: List of LlamaIndex Document objects- chunk_size: Size for traditional text chunks- chunk_overlap: Overlap for traditional text chunks- use_ast_chunking: Whether to use AST chunking for code files- ast_chunk_size: Size for AST chunks- ast_chunk_overlap: Overlap for AST chunks- code_file_extensions: Custom list of code file extensions- ast_fallback_traditional: Fall back to traditional chunking on AST errors-- Returns:- List of text chunks- """+ """Create text chunks from documents with optional AST support for code files."""if not documents:logger.warning("No documents provided for chunking")return []- # Create a local copy of supported extensions for this function calllocal_code_extensions = CODE_EXTENSIONS.copy()-- # Update supported extensions if providedif code_file_extensions:- # Map extensions to languages (simplified mapping)ext_mapping = {".py": "python",".java": "java",@@ -273,47 +188,32 @@ def create_text_chunks(}for ext in code_file_extensions:if ext.lower() not in local_code_extensions:- # Try to guess language from extensionif ext.lower() in ext_mapping:local_code_extensions[ext.lower()] = ext_mapping[ext.lower()]else:logger.warning(f"Unsupported extension {ext}, will use traditional chunking")all_chunks = []-if use_ast_chunking:- # Separate code and text documents using local extensionscode_docs, text_docs = detect_code_files(documents, local_code_extensions)-- # Process code files with AST chunkingif code_docs:- logger.info(f"Processing {len(code_docs)} code files with AST chunking")try:- ast_chunks = create_ast_chunks(- code_docs, max_chunk_size=ast_chunk_size, chunk_overlap=ast_chunk_overlap+ all_chunks.extend(+ create_ast_chunks(+ code_docs, max_chunk_size=ast_chunk_size, chunk_overlap=ast_chunk_overlap+ ))- all_chunks.extend(ast_chunks)- logger.info(f"Created {len(ast_chunks)} AST chunks from code files")except Exception as e:logger.error(f"AST chunking failed: {e}")if ast_fallback_traditional:- logger.info("Falling back to traditional chunking for code files")- traditional_code_chunks = create_traditional_chunks(- code_docs, chunk_size, chunk_overlap+ all_chunks.extend(+ create_traditional_chunks(code_docs, chunk_size, chunk_overlap))- all_chunks.extend(traditional_code_chunks)else:raise-- # Process text files with traditional chunkingif text_docs:- logger.info(f"Processing {len(text_docs)} text files with traditional chunking")- text_chunks = create_traditional_chunks(text_docs, chunk_size, chunk_overlap)- all_chunks.extend(text_chunks)- logger.info(f"Created {len(text_chunks)} traditional chunks from text files")+ all_chunks.extend(create_traditional_chunks(text_docs, chunk_size, chunk_overlap))else:- # Use traditional chunking for all files- logger.info(f"Processing {len(documents)} documents with traditional chunking")all_chunks = create_traditional_chunks(documents, chunk_size, chunk_overlap)logger.info(f"Total chunks created: {len(all_chunks)}")diff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.pyindex caad276..2d514e2 100644--- a/packages/leann-core/src/leann/cli.py+++ b/packages/leann-core/src/leann/cli.py@@ -1,6 +1,5 @@import argparseimport asyncio-import sysfrom pathlib import Pathfrom typing import Any, Optional, Union@@ -1216,13 +1215,8 @@ Examples:if use_ast:print("🧠 Using AST-aware chunking for code files")try:- # Import enhanced chunking utilities- # Add apps directory to path to import chunking utilities- apps_dir = Path(__file__).parent.parent.parent.parent.parent / "apps"- if apps_dir.exists():- sys.path.insert(0, str(apps_dir))-- from chunking import create_text_chunks+ # Import enhanced chunking utilities from packaged module+ from .chunking_utils import create_text_chunks# Use enhanced chunking with AST supportall_texts = create_text_chunks(@@ -1237,7 +1231,9 @@ Examples:)except ImportError as e:- print(f"⚠️ AST chunking not available ({e}), falling back to traditional chunking")+ print(+ f"⚠️ AST chunking utilities not available in package ({e}), falling back to traditional chunking"+ )use_ast = Falseif not use_ast: