Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: fit knowledge base chunks to the embedding model's input limit (#14045)

Nilay0b75841dae1b
diff --git a/studio/backend/core/rag/conversation_archive.py b/studio/backend/core/rag/conversation_archive.py
index 484644d8023..330409f1e6b 100644
--- a/studio/backend/core/rag/conversation_archive.py
+++ b/studio/backend/core/rag/conversation_archive.py
@@ -408,6 +408,7 @@ def archive_turns(
conn = None
try:
count = embeddings.token_counter(model)
+ chunk_limit = None
conn = rag_db.get_connection()
# What the query side will ask for. Only a prediction until the encode reports what it actually used, so the
# authoritative check happens under the write lock.
@@ -441,9 +442,11 @@ def archive_turns(
)
_widen_span(conn, scope, digest, span)
continue
+ if chunk_limit is None:
+ chunk_limit = embeddings.chunk_tokens(model)
chunks = chunk_pages(
[Page(text = text, page_number = None, char_count = len(text))],
- max_tokens = config.CHUNK_TOKENS,
+ max_tokens = chunk_limit,
overlap = config.CHUNK_OVERLAP,
count = count,
)
diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.py
index 7fc872b96fd..1f36a441df3 100644
--- a/studio/backend/core/rag/embeddings.py
+++ b/studio/backend/core/rag/embeddings.py
@@ -1504,6 +1504,11 @@ def max_tokens(model_name: str | None = None) -> int | None:
return _get_backend(model_name).max_tokens(model_name = model_name)
+def chunk_tokens(model_name: str | None = None) -> int:
+ limit = max_tokens(model_name)
+ return min(config.CHUNK_TOKENS, limit) if limit else config.CHUNK_TOKENS
+
+
def token_counter(model_name: str | None = None) -> Callable[[str], int]:
"""Callable counting tokens with the embedder's own tokenizer.
diff --git a/studio/backend/core/rag/ingestion.py b/studio/backend/core/rag/ingestion.py
index 67bbbce4d17..c8cfcd024cf 100644
--- a/studio/backend/core/rag/ingestion.py
+++ b/studio/backend/core/rag/ingestion.py
@@ -403,7 +403,7 @@ def _run(
count = embeddings.token_counter(model_name)
chunks = chunking.chunk_pages(
pages,
- max_tokens = config.CHUNK_TOKENS,
+ max_tokens = embeddings.chunk_tokens(model_name),
overlap = config.CHUNK_OVERLAP,
count = count,
)
diff --git a/studio/backend/core/rag/web_rank.py b/studio/backend/core/rag/web_rank.py
index 77cf4a81832..32f89a2a1ac 100644
--- a/studio/backend/core/rag/web_rank.py
+++ b/studio/backend/core/rag/web_rank.py
@@ -70,7 +70,6 @@ def retrieve_web_chunks(
if not query or top_n <= 0 or not pages or not rag_db.RAG_AVAILABLE:
return "", []
model = model_name or config.effective_embedding_model()
- max_tokens = max_tokens or config.CHUNK_TOKENS
overlap = config.CHUNK_OVERLAP if overlap is None else overlap
count = embeddings.token_counter(model)
@@ -82,6 +81,7 @@ def retrieve_web_chunks(
scope = f"research_scrape_{uuid.uuid4().hex}"
doc_ids: list[str] = []
try:
+ max_tokens = max_tokens or embeddings.chunk_tokens(model)
for page in pages:
text = str(page.get("text") or "").strip()
if not text:
diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.py
index 5febdc7f7a0..fd900f791e0 100644
--- a/studio/backend/tests/conftest.py
+++ b/studio/backend/tests/conftest.py
@@ -1110,6 +1110,7 @@ def stub_embeddings(monkeypatch):
monkeypatch.setattr(embeddings, "encode", fake_encode)
monkeypatch.setattr(embeddings, "dim", lambda model_name = None: dim)
+ monkeypatch.setattr(embeddings, "max_tokens", lambda model_name = None: None)
monkeypatch.setattr(
embeddings,
"token_counter",
diff --git a/studio/backend/tests/test_conversation_archive.py b/studio/backend/tests/test_conversation_archive.py
index a16aec7d082..373a17698d3 100644
--- a/studio/backend/tests/test_conversation_archive.py
+++ b/studio/backend/tests/test_conversation_archive.py
@@ -123,6 +123,20 @@ def _tool_part(
}
+def test_the_embedding_limit_is_read_once_per_pass_and_never_for_archived_turns(conn, monkeypatch):
+ from core.rag import embeddings
+
+ calls = []
+ monkeypatch.setattr(embeddings, "max_tokens", lambda model_name = None: calls.append(model_name))
+ turns = _turn("q1", "a1") + _turn("q2", "a2") + _turn("q3", "a3")
+ assert _archive(turns) == 3
+ assert len(calls) == 1
+
+ calls.clear()
+ assert conversation_archive.archive_turns(THREAD, [dict(m) for m in turns]) == 0
+ assert calls == []
+
+
def test_evicted_turns_are_archived_under_the_conversation_scope(conn):
written = _archive(_turn("what is a duck", "a waterfowl"))
diff --git a/studio/backend/tests/test_rag_embedding_identity.py b/studio/backend/tests/test_rag_embedding_identity.py
index fb2626ca19e..ec4e4454ba8 100644
--- a/studio/backend/tests/test_rag_embedding_identity.py
+++ b/studio/backend/tests/test_rag_embedding_identity.py
@@ -463,6 +463,7 @@ def test_the_web_ranker_labels_a_page_with_the_backend_that_encoded_it(rag_home,
monkeypatch.setattr(
embeddings, "token_counter", lambda model_name = None: lambda t: max(1, len(t.split()))
)
+ monkeypatch.setattr(embeddings, "max_tokens", lambda model_name = None: None)
class _SwapsMidEncode:
def encode(
diff --git a/studio/backend/tests/test_rag_ingestion.py b/studio/backend/tests/test_rag_ingestion.py
index 4c7bde2d9a3..30cbea5cf6a 100644
--- a/studio/backend/tests/test_rag_ingestion.py
+++ b/studio/backend/tests/test_rag_ingestion.py
@@ -104,6 +104,26 @@ def test_ingestion_lifecycle_pending_to_completed(rag_home, stub_embeddings, tmp
conn.close()
+def test_ingested_chunks_fit_the_embedding_model_input_limit(
+ rag_home, stub_embeddings, tmp_path, monkeypatch
+):
+ from core.rag import embeddings
+
+ monkeypatch.setattr(embeddings, "max_tokens", lambda model_name = None: 8)
+ path = _write(tmp_path, "doc.txt", " ".join(f"word{i}" for i in range(60)))
+ scope = store.kb_scope("K1")
+ doc_id, job_id = ingestion.start_ingestion(scope, "K1", None, "doc.txt", path)
+ assert _wait_completed(job_id)["status"] == "completed"
+
+ conn = rag_db.get_connection()
+ try:
+ rows = conn.execute("SELECT text FROM chunks WHERE document_id=?", (doc_id,)).fetchall()
+ finally:
+ conn.close()
+ assert len(rows) > 1
+ assert max(len(row["text"].split()) for row in rows) <= 8
+
+
def test_ingestion_skips_chunk_write_when_the_document_was_deleted(
rag_home, stub_embeddings, tmp_path, monkeypatch
):
diff --git a/studio/backend/tests/test_web_rank.py b/studio/backend/tests/test_web_rank.py
index cdb09ab5182..9542b779cf4 100644
--- a/studio/backend/tests/test_web_rank.py
+++ b/studio/backend/tests/test_web_rank.py
@@ -55,6 +55,7 @@ def fake_embeddings(monkeypatch):
return np.stack(rows)
monkeypatch.setattr(rag_embeddings, "encode", encode)
+ monkeypatch.setattr(rag_embeddings, "max_tokens", lambda model_name = None: None)
def _scope_rows(db_file):
@@ -117,6 +118,15 @@ def test_char_budget_caps_kept_chunks(rag_home):
assert 1 <= capped.count("<chunk id") < full.count("<chunk id")
+def test_web_chunks_fit_the_embedding_model_input_limit(rag_home, monkeypatch):
+ from core.rag import embeddings as rag_embeddings
+
+ monkeypatch.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 8)
+ pages = [{"text": " ".join(f"lora{i}" for i in range(60)), "url": "https://a"}]
+ rendered, _ = web_rank.retrieve_web_chunks(pages, "lora", top_n = 20, min_score = 0.0)
+ assert rendered.count("<chunk id") > 1
+
+
def test_empty_and_invalid_inputs_return_empty(rag_home):
assert web_rank.retrieve_web_chunks([], "lora", top_n = 5, min_score = 0.1) == ("", [])
assert web_rank.retrieve_web_chunks([{"text": " "}], "lora", top_n = 5, min_score = 0.1) == ("", [])