Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

fix(core): fit chunk sizes to the embedding model's sequence limit (#434)

Yichuan Wang00710edff4ab
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 57da7a0..8a69061 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -76,3 +76,15 @@ fixes). Newest entries at the bottom.
- Keep Latin words and numbers separate from adjacent Chinese, Japanese, and Korean n-grams when building SQLite FTS5 indexes and parsing keyword queries. For example, `Python数据库SQL` can be retrieved by `Python`, `SQL`, or a mixed-script query.
- Existing BM25 artifacts remain readable. Rebuild indexes containing mixed-script text to regenerate tokens that were previously joined at script boundaries.
+
+## 2026-10-12: Parallel Search MCP web reader (#433)
+
+- Added `apps/parallel_data`: anonymous Parallel Search MCP search/fetch (no API key) exported as Markdown named `<url-slug>-<hash>.md` for `leann build`.
+- Added `apps/parallel_rag.py`: one-command web search → chunk → index → ask, keeping each chunk's title and source URL.
+
+## 2026-10-12: Fit chunk sizes to the embedding model's sequence limit
+
+- `leann build` now caps `--doc-chunk-size` / `--code-chunk-size` at 70% of the embedding model's sequence limit. Previously the macOS default model `all-MiniLM-L6-v2` (reads 256 tokens) got 512-token code chunks, so ~43% of indexed code text was never embedded. On `leann-core` sources, code retrieval MRR@10 rose from 0.30 to 0.54 (18 queries, real `leann build` indexes).
+- `get_model_token_limit()` reads `max_seq_length` from Hugging Face sentence-transformers repos instead of assuming 2048 tokens for them.
+- `create_text_chunks(max_tokens_per_chunk=...)` now only shrinks `chunk_size`; it previously replaced it, growing 256-token chunks to ~1700 for unknown models.
+- Existing indexes are unchanged; rebuild with `--force` to re-chunk with the new sizes.
diff --git a/packages/leann-core/src/leann/chunking_utils.py b/packages/leann-core/src/leann/chunking_utils.py
index 003e362..0493c3c 100644
--- a/packages/leann-core/src/leann/chunking_utils.py
+++ b/packages/leann-core/src/leann/chunking_utils.py
@@ -15,6 +15,18 @@ logger = logging.getLogger(__name__)
_ast_token_warning_shown = False
+# Chunk sizes are counted in tiktoken tokens, but BERT-style WordPiece tokenizers
+# count ~30% more on web text and code, and SentenceSplitter overshoots on spans it
+# cannot split (tables, long lines). At 70% of the model limit, <1% of chunk text is
+# truncated by all-MiniLM-L6-v2 (256 tokens) on web pages and Python sources.
+CHUNK_TOKEN_BUDGET_RATIO = 0.7
+
+
+def cap_chunk_size(chunk_size: int, model_token_limit: int) -> int:
+ """Cap a SentenceSplitter chunk size (tiktoken tokens) so chunks fit the embedding model."""
+ return min(chunk_size, max(1, int(model_token_limit * CHUNK_TOKEN_BUDGET_RATIO)))
+
+
def estimate_token_count(text: str) -> int:
"""
Estimate token count for a text string.
@@ -303,10 +315,9 @@ def create_traditional_chunks(
documents: LlamaIndex Document list.
chunk_size: Target chunk size in **characters** (approximate tokens).
chunk_overlap: Overlap between adjacent chunks in characters.
- max_tokens_per_chunk: If set, auto-scale ``chunk_size`` so each
- chunk stays within the model's token budget. Uses
- ``calculate_safe_chunk_size`` with a 10 % safety margin.
- Additionaly runs ``validate_chunk_token_limits`` post-chunk.
+ max_tokens_per_chunk: If set, cap ``chunk_size`` (see ``cap_chunk_size``)
+ so each chunk fits the embedding model, and truncate any chunk that
+ still exceeds the limit.
Returns:
List of dicts with ``{"text": str, "metadata": dict}``.
@@ -317,18 +328,18 @@ def create_traditional_chunks(
if chunk_overlap < 0:
chunk_overlap = 0
- # ── Token-aware auto-scaling ──────────────────────────────────────
+ # ── Token-aware cap ───────────────────────────────────────────────
if max_tokens_per_chunk and max_tokens_per_chunk > 0:
- chunk_size = calculate_safe_chunk_size(
- max_tokens_per_chunk, chunk_overlap, chunking_mode="traditional"
- )
- logger.info(
- "Token-aware chunking: model limit=%d tokens → safe chunk_size=%d chars "
- "(overlap=%d, safety=0.9)",
- max_tokens_per_chunk,
- chunk_size,
- chunk_overlap,
- )
+ capped = cap_chunk_size(chunk_size, max_tokens_per_chunk)
+ if capped < chunk_size:
+ logger.info(
+ "Token-aware chunking: model limit=%d tokens → chunk_size %d → %d",
+ max_tokens_per_chunk,
+ chunk_size,
+ capped,
+ )
+ chunk_size = capped
+ chunk_overlap = min(chunk_overlap, capped // 2)
# Revalidate after scaling
if chunk_overlap >= chunk_size:
@@ -453,8 +464,11 @@ def create_text_chunks(
# AST chunking: auto-scale if token limit given
ast_size = ast_chunk_size
if max_tokens_per_chunk and max_tokens_per_chunk > 0:
- ast_size = calculate_safe_chunk_size(
- max_tokens_per_chunk, ast_chunk_overlap, chunking_mode="ast"
+ ast_size = min(
+ ast_chunk_size,
+ calculate_safe_chunk_size(
+ max_tokens_per_chunk, ast_chunk_overlap, chunking_mode="ast"
+ ),
)
logger.info(
"Token-aware AST chunking: limit=%d → safe ast_chunk_size=%d chars "
diff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.py
index fbe4027..26084bb 100644
--- a/packages/leann-core/src/leann/cli.py
+++ b/packages/leann-core/src/leann/cli.py
@@ -432,7 +432,7 @@ Examples:
"--doc-chunk-size",
type=int,
default=256,
- help="Document chunk size in TOKENS (default: 256). Final chunks may be larger due to overlap. For 512 token models: recommended 350 tokens (350 + 128 overlap = 478 max)",
+ help="Document chunk size in TOKENS (default: 256), including overlap. Capped to 70%% of the embedding model's sequence limit so chunks are fully embedded (e.g. 179 for all-MiniLM-L6-v2)",
)
build_parser.add_argument(
"--doc-chunk-overlap",
@@ -444,7 +444,7 @@ Examples:
"--code-chunk-size",
type=int,
default=512,
- help="Code chunk size in TOKENS (default: 512). Final chunks may be larger due to overlap. For 512 token models: recommended 400 tokens (400 + 50 overlap = 450 max)",
+ help="Code chunk size in TOKENS (default: 512), including overlap. Capped to 70%% of the embedding model's sequence limit so chunks are fully embedded (e.g. 179 for all-MiniLM-L6-v2)",
)
build_parser.add_argument(
"--code-chunk-overlap",
@@ -2586,14 +2586,36 @@ Examples:
# Guard against invalid configurations
doc_chunk_size = max(1, int(args.doc_chunk_size))
doc_chunk_overlap = max(0, int(args.doc_chunk_overlap))
+ code_chunk_size = max(1, int(args.code_chunk_size))
+ code_chunk_overlap = max(0, int(args.code_chunk_overlap))
+
+ # Text beyond the embedding model's sequence limit is never embedded, so cap
+ # chunk sizes to fit it (e.g. all-MiniLM-L6-v2 reads only 256 tokens).
+ from .chunking_utils import cap_chunk_size
+ from .embedding_compute import get_model_token_limit
+
+ embedding_options = self._build_embedding_options(args)
+ token_limit = get_model_token_limit(
+ args.embedding_model,
+ base_url=embedding_options.get("host") or embedding_options.get("base_url"),
+ )
+ capped_doc = cap_chunk_size(doc_chunk_size, token_limit)
+ capped_code = cap_chunk_size(code_chunk_size, token_limit)
+ if capped_doc < doc_chunk_size or capped_code < code_chunk_size:
+ print(
+ f"📏 {args.embedding_model} reads {token_limit} tokens: chunk size "
+ f"doc {doc_chunk_size} → {capped_doc}, code {code_chunk_size} → {capped_code}"
+ )
+ doc_chunk_overlap = min(doc_chunk_overlap, capped_doc // 2)
+ code_chunk_overlap = min(code_chunk_overlap, capped_code // 2)
+ doc_chunk_size, code_chunk_size = capped_doc, capped_code
+
if doc_chunk_overlap >= doc_chunk_size:
print(
f"⚠️ Adjusting doc chunk overlap from {doc_chunk_overlap} to {doc_chunk_size - 1} (must be < chunk size)"
)
doc_chunk_overlap = doc_chunk_size - 1
- code_chunk_size = max(1, int(args.code_chunk_size))
- code_chunk_overlap = max(0, int(args.code_chunk_overlap))
if code_chunk_overlap >= code_chunk_size:
print(
f"⚠️ Adjusting code chunk overlap from {code_chunk_overlap} to {code_chunk_size - 1} (must be < chunk size)"
diff --git a/packages/leann-core/src/leann/embedding_compute.py b/packages/leann-core/src/leann/embedding_compute.py
index c7125d7..62da2d4 100644
--- a/packages/leann-core/src/leann/embedding_compute.py
+++ b/packages/leann-core/src/leann/embedding_compute.py
@@ -121,6 +121,13 @@ def get_model_token_limit(
_token_limit_cache[cache_key] = limit
return limit
+ # Local Hugging Face models (sentence-transformers) publish their sequence limit
+ if not base_url and "/" in model_name:
+ limit = _query_huggingface_seq_limit(model_name)
+ if limit:
+ _token_limit_cache[cache_key] = limit
+ return limit
+
# Check partial matches for common patterns
for known_model, registry_limit in EMBEDDING_MODEL_LIMITS.items():
if known_model in base_model_name or base_model_name in known_model:
@@ -199,6 +206,31 @@ def truncate_to_token_limit(texts: list[str], token_limit: int) -> list[str]:
return truncated_texts
+def _query_huggingface_seq_limit(model_name: str) -> Optional[int]:
+ """
+ Read ``max_seq_length`` from a Hugging Face sentence-transformers repo.
+
+ Args:
+ model_name: Hugging Face repo id, e.g. ``sentence-transformers/all-MiniLM-L6-v2``
+
+ Returns:
+ Sequence limit in tokens if published, None otherwise
+ """
+ try:
+ from huggingface_hub import hf_hub_download
+
+ config_path = hf_hub_download(model_name, "sentence_bert_config.json")
+ with open(config_path, encoding="utf-8") as f:
+ limit = json.load(f).get("max_seq_length")
+ if isinstance(limit, int) and limit > 0:
+ logger.info(f"Detected {model_name} sequence limit: {limit} tokens")
+ return limit
+ except Exception as e:
+ logger.debug(f"Failed to read Hugging Face sequence limit: {e}")
+
+ return None
+
+
def _query_ollama_context_limit(model_name: str, base_url: str) -> Optional[int]:
"""
Query Ollama /api/show for model context limit.
diff --git a/tests/test_chunk_size_cap.py b/tests/test_chunk_size_cap.py
new file mode 100644
index 0000000..d481ba3
--- /dev/null
+++ b/tests/test_chunk_size_cap.py
@@ -0,0 +1,77 @@
+"""Chunk sizes are capped (never raised) to fit the embedding model's sequence limit."""
+
+import json
+
+import pytest
+from leann import chunking_utils, embedding_compute
+from leann.chunking_utils import cap_chunk_size, create_text_chunks
+from llama_index.core import Document
+
+
[email protected](autouse=True)
+def clear_token_limit_cache():
+ embedding_compute._token_limit_cache.clear()
+ yield
+ embedding_compute._token_limit_cache.clear()
+
+
+ "chunk_size,limit,expected",
+ [(256, 256, 179), (512, 256, 179), (256, 512, 256), (512, 512, 358), (256, 8192, 256)],
+)
+def test_cap_chunk_size_only_shrinks(chunk_size, limit, expected):
+ assert cap_chunk_size(chunk_size, limit) == expected
+
+
+ "chunk_size,overlap,limit,expected",
+ [
+ (256, 64, None, (256, 64)),
+ (256, 64, 2048, (256, 64)), # large limit never grows the chunk
+ (512, 50, 256, (179, 50)),
+ (256, 128, 256, (179, 89)), # overlap capped to half the new size
+ ],
+)
+def test_create_text_chunks_caps_splitter_size(monkeypatch, chunk_size, overlap, limit, expected):
+ used = []
+
+ class RecordingSplitter:
+ def __init__(self, chunk_size, chunk_overlap, **kwargs):
+ used.append((chunk_size, chunk_overlap))
+
+ def get_nodes_from_documents(self, docs):
+ return []
+
+ monkeypatch.setattr(chunking_utils, "SentenceSplitter", RecordingSplitter)
+ docs = [Document(text="LEANN recomputes embeddings on demand.")]
+ create_text_chunks(
+ docs, chunk_size=chunk_size, chunk_overlap=overlap, max_tokens_per_chunk=limit
+ )
+ assert used == [expected]
+
+
+def test_huggingface_seq_limit_discovery(monkeypatch, tmp_path):
+ config = tmp_path / "sentence_bert_config.json"
+ config.write_text(json.dumps({"max_seq_length": 256}))
+ calls = []
+
+ def fake_download(repo_id, filename):
+ calls.append((repo_id, filename))
+ return str(config)
+
+ monkeypatch.setattr("huggingface_hub.hf_hub_download", fake_download)
+ model = "sentence-transformers/all-MiniLM-L6-v2"
+ assert embedding_compute.get_model_token_limit(model) == 256
+ assert calls == [(model, "sentence_bert_config.json")]
+ # Remote servers and bare names never hit the Hugging Face Hub
+ embedding_compute.get_model_token_limit(model, base_url="http://localhost:8000/v1")
+ embedding_compute.get_model_token_limit("unknown-model-xyz")
+ assert len(calls) == 1
+
+
+def test_huggingface_discovery_failure_uses_default(monkeypatch):
+ def fail(repo_id, filename):
+ raise OSError("offline")
+
+ monkeypatch.setattr("huggingface_hub.hf_hub_download", fail)
+ assert embedding_compute.get_model_token_limit("org/no-such-model", default=2048) == 2048