[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
Fix/pr 375 chunking import (#383)
Aakash Suresh7482054a1e92
diff --git a/apps/base_rag_example.py b/apps/base_rag_example.pyindex 25e3b59..67e198b 100644--- a/apps/base_rag_example.py+++ b/apps/base_rag_example.py@@ -282,6 +282,22 @@ class BaseRAGExample(ABC):return config+ @staticmethod+ def _resolve_chunk_token_limit(args) -> int | None:+ """Resolve the embedding model's token limit for token-aware chunking.++ Returns ``None`` if the limit cannot be determined (e.g. model unknown).+ Apps can pass the result as ``max_tokens_per_chunk=`` to+ ``create_text_chunks()``.+ """+ try:+ from leann.embedding_compute import get_model_token_limit++ base_url = getattr(args, "embedding_api_base", None)+ return get_model_token_limit(args.embedding_model, base_url)+ except Exception:+ return None+async def build_index(self, args, texts: list[dict[str, Any]]) -> str:"""Build LEANN index from text chunks (dicts with 'text' and 'metadata' keys)."""index_path = str(Path(args.index_dir) / f"{self.default_index_name}.leann")@@ -289,6 +305,17 @@ class BaseRAGExample(ABC):print(f"\n[Building Index] Creating {self.name} index...")print(f"Total text chunks: {len(texts)}")+ # Warn if any chunks may exceed the embedding model's token limit+ limit = self._resolve_chunk_token_limit(args)+ if limit:+ try:+ from leann.chunking_utils import validate_chunk_token_limits++ _texts = [t["text"] if isinstance(t, dict) else t for t in texts]+ validate_chunk_token_limits(_texts, limit)+ except Exception:+ pass+embedding_options: dict[str, Any] = {}if args.embedding_mode == "ollama":embedding_options["host"] = resolve_ollama_host(args.embedding_host)diff --git a/packages/leann-core/src/leann/chunking_utils.py b/packages/leann-core/src/leann/chunking_utils.pyindex 30171c8..003e362 100644--- a/packages/leann-core/src/leann/chunking_utils.py+++ b/packages/leann-core/src/leann/chunking_utils.py@@ -172,6 +172,27 @@ def get_language_from_extension(file_path: str) -> Optional[str]:return CODE_EXTENSIONS.get(ext)+def _parse_ast_chunk_output(chunk: Any) -> tuple[str | None, dict[str, Any]]:+ """Normalize the various chunk output formats from ASTChunkBuilder.++ astchunk can return objects (with a ``.text`` attr), plain strings, or+ dicts (``{"content": ..., "metadata": ...}``). This helper returns a+ uniform ``(text, metadata)`` pair regardless of the input shape.+ """+ if hasattr(chunk, "text"):+ return (str(chunk.text) if chunk.text else None, {})+ if isinstance(chunk, str):+ return (chunk, {})+ if isinstance(chunk, dict):+ meta = chunk.get("metadata", {})+ if "content" in chunk:+ return (chunk["content"], meta)+ if "text" in chunk:+ return (chunk["text"], meta)+ return (str(chunk), {})+ return (str(chunk), {})++def create_ast_chunks(documents,max_chunk_size: int = 512,@@ -190,14 +211,14 @@ def create_ast_chunks(except ImportError as e:logger.error(f"astchunk not available: {e}")logger.info("Falling back to traditional chunking for code files")- return _traditional_chunks_as_dicts(documents, max_chunk_size, chunk_overlap)+ return create_traditional_chunks(documents, max_chunk_size, chunk_overlap)all_chunks = []for doc in documents:language = doc.metadata.get("language")if not language:logger.warning("No language detected; falling back to traditional chunking")- all_chunks.extend(_traditional_chunks_as_dicts([doc], max_chunk_size, chunk_overlap))+ all_chunks.extend(create_traditional_chunks([doc], max_chunk_size, chunk_overlap))continuetry:@@ -240,24 +261,7 @@ def create_ast_chunks(chunks = chunk_builder.chunkify(code_content)for chunk in chunks:- chunk_text: str | None = None- astchunk_metadata: dict[str, Any] = {}-- if hasattr(chunk, "text"):- chunk_text = str(chunk.text) if chunk.text else None- elif isinstance(chunk, str):- chunk_text = chunk- elif isinstance(chunk, dict):- # Handle astchunk format: {"content": "...", "metadata": {...}}- if "content" in chunk:- chunk_text = chunk["content"]- astchunk_metadata = chunk.get("metadata", {})- elif "text" in chunk:- chunk_text = chunk["text"]- else:- chunk_text = str(chunk) # Last resort- else:- chunk_text = str(chunk)+ chunk_text, astchunk_metadata = _parse_ast_chunk_output(chunk)if chunk_text and chunk_text.strip():# Extract document-level metadata@@ -282,26 +286,62 @@ def create_ast_chunks(except Exception as e:logger.warning(f"AST chunking failed for {language} file: {e}")logger.info("Falling back to traditional chunking")- all_chunks.extend(_traditional_chunks_as_dicts([doc], max_chunk_size, chunk_overlap))+ all_chunks.extend(create_traditional_chunks([doc], max_chunk_size, chunk_overlap))return all_chunksdef create_traditional_chunks(- documents, chunk_size: int = 256, chunk_overlap: int = 128+ documents,+ chunk_size: int = 256,+ chunk_overlap: int = 128,+ max_tokens_per_chunk: int | None = None,) -> list[dict[str, Any]]:"""Create traditional text chunks using LlamaIndex SentenceSplitter.+ Args:+ documents: LlamaIndex Document list.+ chunk_size: Target chunk size in **characters** (approximate tokens).+ chunk_overlap: Overlap between adjacent chunks in characters.+ max_tokens_per_chunk: If set, auto-scale ``chunk_size`` so each+ chunk stays within the model's token budget. Uses+ ``calculate_safe_chunk_size`` with a 10 % safety margin.+ Additionaly runs ``validate_chunk_token_limits`` post-chunk.+Returns:- List of dicts with {"text": str, "metadata": dict}+ List of dicts with ``{"text": str, "metadata": dict}``."""if chunk_size <= 0:logger.warning(f"Invalid chunk_size={chunk_size}, using default value of 256")chunk_size = 256if chunk_overlap < 0:chunk_overlap = 0++ # ── Token-aware auto-scaling ──────────────────────────────────────+ if max_tokens_per_chunk and max_tokens_per_chunk > 0:+ chunk_size = calculate_safe_chunk_size(+ max_tokens_per_chunk, chunk_overlap, chunking_mode="traditional"+ )+ logger.info(+ "Token-aware chunking: model limit=%d tokens → safe chunk_size=%d chars "+ "(overlap=%d, safety=0.9)",+ max_tokens_per_chunk,+ chunk_size,+ chunk_overlap,+ )++ # Revalidate after scalingif chunk_overlap >= chunk_size:- chunk_overlap = chunk_size // 2+ old_overlap = chunk_overlap+ chunk_overlap = max(0, chunk_size // 2)+ logger.warning(+ "Token-aware scaling reduced chunk_size below chunk_overlap "+ "(%d → %d); overlap reduced %d → %d",+ chunk_size,+ old_overlap,+ old_overlap,+ chunk_overlap,+ )node_parser = SentenceSplitter(chunk_size=chunk_size,@@ -312,8 +352,6 @@ def create_traditional_chunks(result = []for doc in documents:- # Propagate all document-level metadata to each chunk so custom fields- # (e.g. url/domain for browser_rag) remain available for metadata_filters.doc_metadata = dict(doc.metadata) if doc.metadata else {}doc_metadata.setdefault("file_path", "")doc_metadata.setdefault("file_name", "")@@ -330,17 +368,37 @@ def create_traditional_chunks(if content and content.strip():result.append({"text": content.strip(), "metadata": doc_metadata})+ # ── Post-chunk validation ─────────────────────────────────────────+ if max_tokens_per_chunk and max_tokens_per_chunk > 0 and result:+ _texts = [c["text"] for c in result]+ _validated, _n = validate_chunk_token_limits(_texts, max_tokens_per_chunk)+ if _n > 0:+ logger.warning(+ "Token-aware chunking: %d/%d chunks truncated to fit %d-token limit",+ _n,+ len(result),+ max_tokens_per_chunk,+ )+ for i, chunk_dict in enumerate(result):+ if i < len(_validated):+ chunk_dict["text"] = _validated[i]+return resultdef _traditional_chunks_as_dicts(- documents, chunk_size: int = 256, chunk_overlap: int = 128+ documents,+ chunk_size: int = 256,+ chunk_overlap: int = 128,+ max_tokens_per_chunk: int | None = None,) -> list[dict[str, Any]]:- """Helper: Traditional chunking that returns dict format for consistency.-- This is now just an alias for create_traditional_chunks for backwards compatibility.- """- return create_traditional_chunks(documents, chunk_size, chunk_overlap)+ """Backward-compatible alias for create_traditional_chunks."""+ return create_traditional_chunks(+ documents,+ chunk_size,+ chunk_overlap,+ max_tokens_per_chunk=max_tokens_per_chunk,+ )def create_text_chunks(@@ -352,11 +410,20 @@ def create_text_chunks(ast_chunk_overlap: int = 64,code_file_extensions: Optional[list[str]] = None,ast_fallback_traditional: bool = True,+ max_tokens_per_chunk: int | None = None,) -> list[dict[str, Any]]:"""Create text chunks from documents with optional AST support for code files.+ Args:+ documents: LlamaIndex Document list.+ chunk_size: Characters per traditional chunk.+ chunk_overlap: Character overlap between traditional chunks.+ max_tokens_per_chunk: If set, auto-scale chunk_size to keep each+ chunk within the embedding model's token budget. Also runs+ post-chunk validation and truncation when necessary.+Returns:- List of dicts with {"text": str, "metadata": dict}+ List of dicts with ``{"text": str, "metadata": dict}``."""if not documents:logger.warning("No documents provided for chunking")@@ -383,8 +450,22 @@ def create_text_chunks(code_docs, text_docs = detect_code_files(documents, local_code_extensions)if code_docs:try:+ # AST chunking: auto-scale if token limit given+ ast_size = ast_chunk_size+ if max_tokens_per_chunk and max_tokens_per_chunk > 0:+ ast_size = calculate_safe_chunk_size(+ max_tokens_per_chunk, ast_chunk_overlap, chunking_mode="ast"+ )+ logger.info(+ "Token-aware AST chunking: limit=%d → safe ast_chunk_size=%d chars "+ "(overlap=%d, safety=0.9)",+ max_tokens_per_chunk,+ ast_size,+ ast_chunk_overlap,+ )+ast_chunks = create_ast_chunks(- code_docs, max_chunk_size=ast_chunk_size, chunk_overlap=ast_chunk_overlap+ code_docs, max_chunk_size=ast_size, chunk_overlap=ast_chunk_overlap)# Prepend line numbers to code chunks for navigationfor chunk in ast_chunks:@@ -401,17 +482,32 @@ def create_text_chunks(logger.error(f"AST chunking failed: {e}")if ast_fallback_traditional:all_chunks.extend(- _traditional_chunks_as_dicts(code_docs, chunk_size, chunk_overlap)+ create_traditional_chunks(+ code_docs,+ chunk_size,+ chunk_overlap,+ max_tokens_per_chunk=max_tokens_per_chunk,+ ))else:raiseif text_docs:- all_chunks.extend(_traditional_chunks_as_dicts(text_docs, chunk_size, chunk_overlap))+ all_chunks.extend(+ create_traditional_chunks(+ text_docs,+ chunk_size,+ chunk_overlap,+ max_tokens_per_chunk=max_tokens_per_chunk,+ )+ )else:- all_chunks = _traditional_chunks_as_dicts(documents, chunk_size, chunk_overlap)+ all_chunks = create_traditional_chunks(+ documents,+ chunk_size,+ chunk_overlap,+ max_tokens_per_chunk=max_tokens_per_chunk,+ )logger.info(f"Total chunks created: {len(all_chunks)}")- # Note: Token truncation is now handled at embedding time with dynamic model limits- # See get_model_token_limit() and truncate_to_token_limit() in embedding_compute.pyreturn all_chunks