[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
fix(core): fit chunk sizes to the embedding model's sequence limit (#434)
Yichuan Wang00710edff4ab
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.mdindex 57da7a0..8a69061 100644--- a/docs/CHANGELOG.md+++ b/docs/CHANGELOG.md@@ -76,3 +76,15 @@ fixes). Newest entries at the bottom.- Keep Latin words and numbers separate from adjacent Chinese, Japanese, and Korean n-grams when building SQLite FTS5 indexes and parsing keyword queries. For example, `Python数据库SQL` can be retrieved by `Python`, `SQL`, or a mixed-script query.- Existing BM25 artifacts remain readable. Rebuild indexes containing mixed-script text to regenerate tokens that were previously joined at script boundaries.++## 2026-10-12: Parallel Search MCP web reader (#433)++- Added `apps/parallel_data`: anonymous Parallel Search MCP search/fetch (no API key) exported as Markdown named `<url-slug>-<hash>.md` for `leann build`.+- Added `apps/parallel_rag.py`: one-command web search → chunk → index → ask, keeping each chunk's title and source URL.++## 2026-10-12: Fit chunk sizes to the embedding model's sequence limit++- `leann build` now caps `--doc-chunk-size` / `--code-chunk-size` at 70% of the embedding model's sequence limit. Previously the macOS default model `all-MiniLM-L6-v2` (reads 256 tokens) got 512-token code chunks, so ~43% of indexed code text was never embedded. On `leann-core` sources, code retrieval MRR@10 rose from 0.30 to 0.54 (18 queries, real `leann build` indexes).+- `get_model_token_limit()` reads `max_seq_length` from Hugging Face sentence-transformers repos instead of assuming 2048 tokens for them.+- `create_text_chunks(max_tokens_per_chunk=...)` now only shrinks `chunk_size`; it previously replaced it, growing 256-token chunks to ~1700 for unknown models.+- Existing indexes are unchanged; rebuild with `--force` to re-chunk with the new sizes.diff --git a/packages/leann-core/src/leann/chunking_utils.py b/packages/leann-core/src/leann/chunking_utils.pyindex 003e362..0493c3c 100644--- a/packages/leann-core/src/leann/chunking_utils.py+++ b/packages/leann-core/src/leann/chunking_utils.py@@ -15,6 +15,18 @@ logger = logging.getLogger(__name__)_ast_token_warning_shown = False+# Chunk sizes are counted in tiktoken tokens, but BERT-style WordPiece tokenizers+# count ~30% more on web text and code, and SentenceSplitter overshoots on spans it+# cannot split (tables, long lines). At 70% of the model limit, <1% of chunk text is+# truncated by all-MiniLM-L6-v2 (256 tokens) on web pages and Python sources.+CHUNK_TOKEN_BUDGET_RATIO = 0.7+++def cap_chunk_size(chunk_size: int, model_token_limit: int) -> int:+ """Cap a SentenceSplitter chunk size (tiktoken tokens) so chunks fit the embedding model."""+ return min(chunk_size, max(1, int(model_token_limit * CHUNK_TOKEN_BUDGET_RATIO)))++def estimate_token_count(text: str) -> int:"""Estimate token count for a text string.@@ -303,10 +315,9 @@ def create_traditional_chunks(documents: LlamaIndex Document list.chunk_size: Target chunk size in **characters** (approximate tokens).chunk_overlap: Overlap between adjacent chunks in characters.- max_tokens_per_chunk: If set, auto-scale ``chunk_size`` so each- chunk stays within the model's token budget. Uses- ``calculate_safe_chunk_size`` with a 10 % safety margin.- Additionaly runs ``validate_chunk_token_limits`` post-chunk.+ max_tokens_per_chunk: If set, cap ``chunk_size`` (see ``cap_chunk_size``)+ so each chunk fits the embedding model, and truncate any chunk that+ still exceeds the limit.Returns:List of dicts with ``{"text": str, "metadata": dict}``.@@ -317,18 +328,18 @@ def create_traditional_chunks(if chunk_overlap < 0:chunk_overlap = 0- # ── Token-aware auto-scaling ──────────────────────────────────────+ # ── Token-aware cap ───────────────────────────────────────────────if max_tokens_per_chunk and max_tokens_per_chunk > 0:- chunk_size = calculate_safe_chunk_size(- max_tokens_per_chunk, chunk_overlap, chunking_mode="traditional"- )- logger.info(- "Token-aware chunking: model limit=%d tokens → safe chunk_size=%d chars "- "(overlap=%d, safety=0.9)",- max_tokens_per_chunk,- chunk_size,- chunk_overlap,- )+ capped = cap_chunk_size(chunk_size, max_tokens_per_chunk)+ if capped < chunk_size:+ logger.info(+ "Token-aware chunking: model limit=%d tokens → chunk_size %d → %d",+ max_tokens_per_chunk,+ chunk_size,+ capped,+ )+ chunk_size = capped+ chunk_overlap = min(chunk_overlap, capped // 2)# Revalidate after scalingif chunk_overlap >= chunk_size:@@ -453,8 +464,11 @@ def create_text_chunks(# AST chunking: auto-scale if token limit givenast_size = ast_chunk_sizeif max_tokens_per_chunk and max_tokens_per_chunk > 0:- ast_size = calculate_safe_chunk_size(- max_tokens_per_chunk, ast_chunk_overlap, chunking_mode="ast"+ ast_size = min(+ ast_chunk_size,+ calculate_safe_chunk_size(+ max_tokens_per_chunk, ast_chunk_overlap, chunking_mode="ast"+ ),)logger.info("Token-aware AST chunking: limit=%d → safe ast_chunk_size=%d chars "diff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.pyindex fbe4027..26084bb 100644--- a/packages/leann-core/src/leann/cli.py+++ b/packages/leann-core/src/leann/cli.py@@ -432,7 +432,7 @@ Examples:"--doc-chunk-size",type=int,default=256,- help="Document chunk size in TOKENS (default: 256). Final chunks may be larger due to overlap. For 512 token models: recommended 350 tokens (350 + 128 overlap = 478 max)",+ help="Document chunk size in TOKENS (default: 256), including overlap. Capped to 70%% of the embedding model's sequence limit so chunks are fully embedded (e.g. 179 for all-MiniLM-L6-v2)",)build_parser.add_argument("--doc-chunk-overlap",@@ -444,7 +444,7 @@ Examples:"--code-chunk-size",type=int,default=512,- help="Code chunk size in TOKENS (default: 512). Final chunks may be larger due to overlap. For 512 token models: recommended 400 tokens (400 + 50 overlap = 450 max)",+ help="Code chunk size in TOKENS (default: 512), including overlap. Capped to 70%% of the embedding model's sequence limit so chunks are fully embedded (e.g. 179 for all-MiniLM-L6-v2)",)build_parser.add_argument("--code-chunk-overlap",@@ -2586,14 +2586,36 @@ Examples:# Guard against invalid configurationsdoc_chunk_size = max(1, int(args.doc_chunk_size))doc_chunk_overlap = max(0, int(args.doc_chunk_overlap))+ code_chunk_size = max(1, int(args.code_chunk_size))+ code_chunk_overlap = max(0, int(args.code_chunk_overlap))++ # Text beyond the embedding model's sequence limit is never embedded, so cap+ # chunk sizes to fit it (e.g. all-MiniLM-L6-v2 reads only 256 tokens).+ from .chunking_utils import cap_chunk_size+ from .embedding_compute import get_model_token_limit++ embedding_options = self._build_embedding_options(args)+ token_limit = get_model_token_limit(+ args.embedding_model,+ base_url=embedding_options.get("host") or embedding_options.get("base_url"),+ )+ capped_doc = cap_chunk_size(doc_chunk_size, token_limit)+ capped_code = cap_chunk_size(code_chunk_size, token_limit)+ if capped_doc < doc_chunk_size or capped_code < code_chunk_size:+ print(+ f"📏 {args.embedding_model} reads {token_limit} tokens: chunk size "+ f"doc {doc_chunk_size} → {capped_doc}, code {code_chunk_size} → {capped_code}"+ )+ doc_chunk_overlap = min(doc_chunk_overlap, capped_doc // 2)+ code_chunk_overlap = min(code_chunk_overlap, capped_code // 2)+ doc_chunk_size, code_chunk_size = capped_doc, capped_code+if doc_chunk_overlap >= doc_chunk_size:print(f"⚠️ Adjusting doc chunk overlap from {doc_chunk_overlap} to {doc_chunk_size - 1} (must be < chunk size)")doc_chunk_overlap = doc_chunk_size - 1- code_chunk_size = max(1, int(args.code_chunk_size))- code_chunk_overlap = max(0, int(args.code_chunk_overlap))if code_chunk_overlap >= code_chunk_size:print(f"⚠️ Adjusting code chunk overlap from {code_chunk_overlap} to {code_chunk_size - 1} (must be < chunk size)"diff --git a/packages/leann-core/src/leann/embedding_compute.py b/packages/leann-core/src/leann/embedding_compute.pyindex c7125d7..62da2d4 100644--- a/packages/leann-core/src/leann/embedding_compute.py+++ b/packages/leann-core/src/leann/embedding_compute.py@@ -121,6 +121,13 @@ def get_model_token_limit(_token_limit_cache[cache_key] = limitreturn limit+ # Local Hugging Face models (sentence-transformers) publish their sequence limit+ if not base_url and "/" in model_name:+ limit = _query_huggingface_seq_limit(model_name)+ if limit:+ _token_limit_cache[cache_key] = limit+ return limit+# Check partial matches for common patternsfor known_model, registry_limit in EMBEDDING_MODEL_LIMITS.items():if known_model in base_model_name or base_model_name in known_model:@@ -199,6 +206,31 @@ def truncate_to_token_limit(texts: list[str], token_limit: int) -> list[str]:return truncated_texts+def _query_huggingface_seq_limit(model_name: str) -> Optional[int]:+ """+ Read ``max_seq_length`` from a Hugging Face sentence-transformers repo.++ Args:+ model_name: Hugging Face repo id, e.g. ``sentence-transformers/all-MiniLM-L6-v2``++ Returns:+ Sequence limit in tokens if published, None otherwise+ """+ try:+ from huggingface_hub import hf_hub_download++ config_path = hf_hub_download(model_name, "sentence_bert_config.json")+ with open(config_path, encoding="utf-8") as f:+ limit = json.load(f).get("max_seq_length")+ if isinstance(limit, int) and limit > 0:+ logger.info(f"Detected {model_name} sequence limit: {limit} tokens")+ return limit+ except Exception as e:+ logger.debug(f"Failed to read Hugging Face sequence limit: {e}")++ return None++def _query_ollama_context_limit(model_name: str, base_url: str) -> Optional[int]:"""Query Ollama /api/show for model context limit.diff --git a/tests/test_chunk_size_cap.py b/tests/test_chunk_size_cap.pynew file mode 100644index 0000000..d481ba3--- /dev/null+++ b/tests/test_chunk_size_cap.py@@ -0,0 +1,77 @@+"""Chunk sizes are capped (never raised) to fit the embedding model's sequence limit."""++import json++import pytest+from leann import chunking_utils, embedding_compute+from leann.chunking_utils import cap_chunk_size, create_text_chunks+from llama_index.core import Document++[email protected](autouse=True)+def clear_token_limit_cache():+ embedding_compute._token_limit_cache.clear()+ yield+ embedding_compute._token_limit_cache.clear()+++ "chunk_size,limit,expected",+ [(256, 256, 179), (512, 256, 179), (256, 512, 256), (512, 512, 358), (256, 8192, 256)],+)+def test_cap_chunk_size_only_shrinks(chunk_size, limit, expected):+ assert cap_chunk_size(chunk_size, limit) == expected+++ "chunk_size,overlap,limit,expected",+ [+ (256, 64, None, (256, 64)),+ (256, 64, 2048, (256, 64)), # large limit never grows the chunk+ (512, 50, 256, (179, 50)),+ (256, 128, 256, (179, 89)), # overlap capped to half the new size+ ],+)+def test_create_text_chunks_caps_splitter_size(monkeypatch, chunk_size, overlap, limit, expected):+ used = []++ class RecordingSplitter:+ def __init__(self, chunk_size, chunk_overlap, **kwargs):+ used.append((chunk_size, chunk_overlap))++ def get_nodes_from_documents(self, docs):+ return []++ monkeypatch.setattr(chunking_utils, "SentenceSplitter", RecordingSplitter)+ docs = [Document(text="LEANN recomputes embeddings on demand.")]+ create_text_chunks(+ docs, chunk_size=chunk_size, chunk_overlap=overlap, max_tokens_per_chunk=limit+ )+ assert used == [expected]+++def test_huggingface_seq_limit_discovery(monkeypatch, tmp_path):+ config = tmp_path / "sentence_bert_config.json"+ config.write_text(json.dumps({"max_seq_length": 256}))+ calls = []++ def fake_download(repo_id, filename):+ calls.append((repo_id, filename))+ return str(config)++ monkeypatch.setattr("huggingface_hub.hf_hub_download", fake_download)+ model = "sentence-transformers/all-MiniLM-L6-v2"+ assert embedding_compute.get_model_token_limit(model) == 256+ assert calls == [(model, "sentence_bert_config.json")]+ # Remote servers and bare names never hit the Hugging Face Hub+ embedding_compute.get_model_token_limit(model, base_url="http://localhost:8000/v1")+ embedding_compute.get_model_token_limit("unknown-model-xyz")+ assert len(calls) == 1+++def test_huggingface_discovery_failure_uses_default(monkeypatch):+ def fail(repo_id, filename):+ raise OSError("offline")++ monkeypatch.setattr("huggingface_hub.hf_hub_download", fail)+ assert embedding_compute.get_model_token_limit("org/no-such-model", default=2048) == 2048