Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio RAG: clip re-embedded chunks and embeddings API inputs with batched token counts (#17032)

Daniel Han0efc5fc3caf5
diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.py
index 558be0c507d..9cd6a6b65cd 100644
--- a/studio/backend/core/rag/embeddings.py
+++ b/studio/backend/core/rag/embeddings.py
@@ -1525,6 +1525,31 @@ def clip_to_limit(text: str, count, limit: int | None) -> tuple[str, bool]:
return text[:lo], True
+def clip_many_to_limit(texts, count, limit: int | None) -> list[tuple[str, bool]]:
+ """``clip_to_limit`` per text, bisecting every over-limit text in lockstep: one ``count.many`` per round."""
+ texts = list(texts)
+ if not limit or getattr(count, "many", None) is None or len(texts) < 2:
+ return [clip_to_limit(t, count, limit) for t in texts]
+ out = [(t, False) for t in texts]
+ bounds = {
+ i: [0, len(t)] for i, (t, n) in enumerate(zip(texts, _many(count, texts))) if n > limit
+ }
+ active = list(bounds)
+ while active:
+ mids = [(bounds[i][0] + bounds[i][1] + 1) // 2 for i in active]
+ for i, mid, n in zip(
+ active, mids, _many(count, [texts[i][:m] for i, m in zip(active, mids)])
+ ):
+ if n <= limit:
+ bounds[i][0] = mid
+ else:
+ bounds[i][1] = mid - 1
+ active = [i for i in active if bounds[i][0] < bounds[i][1]]
+ for i, (lo, _) in bounds.items():
+ out[i] = (texts[i][:lo], True)
+ return out
+
+
def chunk_tokens(model_name: str | None = None) -> int:
limit = max_tokens(model_name)
return min(config.CHUNK_TOKENS, limit) if limit else config.CHUNK_TOKENS
diff --git a/studio/backend/core/rag/reembed.py b/studio/backend/core/rag/reembed.py
index 70bd52803bd..4e9b517dcaa 100644
--- a/studio/backend/core/rag/reembed.py
+++ b/studio/backend/core/rag/reembed.py
@@ -92,10 +92,6 @@ def stale_counts() -> dict:
return {"documents": len(rows) - archived, "archived": archived}
-# Chunks were sized with the previous model's tokenizer.
-_clip = embeddings.clip_to_limit
-
-
def _reembed_document(job: _Job, doc: dict, model: str) -> bool:
conn = rag_db.get_connection()
try:
@@ -106,12 +102,9 @@ def _reembed_document(job: _Job, doc: dict, model: str) -> bool:
conn.close()
# Per document: an ST failure can hand the model to llama-server, whose tokenizer and limit differ.
count, limit = embeddings.token_counter(model), embeddings.max_tokens(model)
- texts = []
- clipped = 0
- for row in rows:
- text, was_clipped = _clip(row["text"], count, limit)
- texts.append(text)
- clipped += was_clipped
+ clips = embeddings.clip_many_to_limit([row["text"] for row in rows], count, limit)
+ texts = [text for text, _ in clips]
+ clipped = sum(cut for _, cut in clips)
def on_progress(done: int, total: int) -> None:
if job.stop.is_set() or account_is_retired():
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 5cd67a41c88..5edc7efba0f 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -38461,7 +38461,7 @@ async def _studio_embeddings(
status_code = 400,
detail = f"'input' exceeds the {limit}-token limit of {label}.",
)
- inputs = [rag_embeddings.clip_to_limit(text, count, limit)[0] for text in inputs]
+ inputs = [text for text, _ in rag_embeddings.clip_many_to_limit(inputs, count, limit)]
logger.warning(
"Embedding input over the %d-token limit of %s truncated to fit", limit, label
)
diff --git a/studio/backend/tests/test_openai_embeddings_studio_fallback.py b/studio/backend/tests/test_openai_embeddings_studio_fallback.py
index 9b080e6d982..5cbdb874b73 100644
--- a/studio/backend/tests/test_openai_embeddings_studio_fallback.py
+++ b/studio/backend/tests/test_openai_embeddings_studio_fallback.py
@@ -246,6 +246,32 @@ def test_over_length_input_is_truncated_by_default(studio_embedder):
assert payload["usage"]["prompt_tokens"] == 5
+def test_over_length_input_is_truncated_with_batched_counts(studio_embedder):
+ encoded, singles, batches = [], [], []
+
+ def _count(text):
+ singles.append(text)
+ return len(text)
+
+ def _many(texts):
+ batches.append(list(texts))
+ return [len(t) for t in texts]
+
+ _count.many = _many
+ studio_embedder.setattr(
+ rag_embeddings, "encode", lambda texts, **_: encoded.append(list(texts)) or _vectors(texts)
+ )
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ studio_embedder.setattr(rag_embeddings, "token_counter", lambda model_name = None: _count)
+ studio_embedder.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 3)
+ payload = _call({"input": ["ab", "abcd", "abcdefgh"]})
+ assert encoded == [["ab", "abc", "abc"]]
+ assert payload["usage"]["prompt_tokens"] == 8
+ assert singles == [] and batches[0] == ["ab", "abcd", "abcdefgh"]
+
+
def test_over_length_input_is_rejected_when_truncate_is_false(studio_embedder):
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
diff --git a/studio/backend/tests/test_rag_reembed.py b/studio/backend/tests/test_rag_reembed.py
index 16233b7ee86..b0ff58fc49a 100644
--- a/studio/backend/tests/test_rag_reembed.py
+++ b/studio/backend/tests/test_rag_reembed.py
@@ -389,3 +389,54 @@ def test_a_swap_committed_mid_search_cannot_pass_an_old_space_score(models, tmp_
with _connect() as c:
hits = retrieval.retrieve_dense(c, scope, "text a", k = 5, model_name = NEWER)
assert hits == []
+
+
+def test_a_many_chunk_document_clips_its_chunks_in_batched_counts(models, tmp_path, monkeypatch):
+ words = [f"w{i}" for i in range(1700)]
+ document_id = _ingest(tmp_path, store.kb_scope("K9"), "long.txt", " ".join(words))
+ _, chunks = _row(document_id)
+ assert len(chunks) > 2
+ calls = {"many": [], "single": 0}
+
+ def _counter(model_name = None):
+ def _count(text):
+ calls["single"] += 1
+ return len(text.split())
+
+ def _many(texts):
+ calls["many"].append(len(texts))
+ return [len(t.split()) for t in texts]
+
+ _count.many = _many
+ return _count
+
+ sizes = sorted(len(text.split()) for _, text in chunks)
+ limit = sizes[0]
+ assert sizes[0] < sizes[-1]
+ monkeypatch.setattr(embeddings, "token_counter", _counter)
+ monkeypatch.setattr(embeddings, "max_tokens", lambda model_name = None: limit)
+ word_count = lambda t: len(t.split())
+ expected = [embeddings.clip_to_limit(text, word_count, limit) for _, text in chunks]
+ assert any(cut for _, cut in expected) and not all(cut for _, cut in expected)
+ models["model"] = NEW
+ status = _run_job()
+ sent = [t for m, t in models["encoded"] if m == NEW]
+ assert sent == [[text for text, _ in expected]]
+ assert status["clipped_chunks"] == sum(cut for _, cut in expected)
+ assert calls["single"] == 0
+ assert calls["many"][0] == len(chunks)
+ assert len(calls["many"]) <= 2 + max(len(text) for _, text in chunks).bit_length()
+
+
+def test_clip_many_matches_clip_to_limit_per_text():
+ import random
+
+ rng = random.Random(3407)
+ texts = [
+ " ".join(f"w{rng.randint(0, 99)}" for _ in range(rng.randint(0, 60))) for _ in range(40)
+ ]
+ word_count = lambda t: len(t.split())
+ word_count.many = lambda ts: [len(t.split()) for t in ts]
+ for limit in (None, 0, 1, 7, 30, 100):
+ expected = [embeddings.clip_to_limit(t, word_count, limit) for t in texts]
+ assert embeddings.clip_many_to_limit(texts, word_count, limit) == expected