Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio RAG: clip re-embedded chunks and embeddings API inputs with batched token counts (#17032)
Daniel Han0efc5fc3caf5
diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.pyindex 558be0c507d..9cd6a6b65cd 100644--- a/studio/backend/core/rag/embeddings.py+++ b/studio/backend/core/rag/embeddings.py@@ -1525,6 +1525,31 @@ def clip_to_limit(text: str, count, limit: int | None) -> tuple[str, bool]:return text[:lo], True+def clip_many_to_limit(texts, count, limit: int | None) -> list[tuple[str, bool]]:+ """``clip_to_limit`` per text, bisecting every over-limit text in lockstep: one ``count.many`` per round."""+ texts = list(texts)+ if not limit or getattr(count, "many", None) is None or len(texts) < 2:+ return [clip_to_limit(t, count, limit) for t in texts]+ out = [(t, False) for t in texts]+ bounds = {+ i: [0, len(t)] for i, (t, n) in enumerate(zip(texts, _many(count, texts))) if n > limit+ }+ active = list(bounds)+ while active:+ mids = [(bounds[i][0] + bounds[i][1] + 1) // 2 for i in active]+ for i, mid, n in zip(+ active, mids, _many(count, [texts[i][:m] for i, m in zip(active, mids)])+ ):+ if n <= limit:+ bounds[i][0] = mid+ else:+ bounds[i][1] = mid - 1+ active = [i for i in active if bounds[i][0] < bounds[i][1]]+ for i, (lo, _) in bounds.items():+ out[i] = (texts[i][:lo], True)+ return out++def chunk_tokens(model_name: str | None = None) -> int:limit = max_tokens(model_name)return min(config.CHUNK_TOKENS, limit) if limit else config.CHUNK_TOKENSdiff --git a/studio/backend/core/rag/reembed.py b/studio/backend/core/rag/reembed.pyindex 70bd52803bd..4e9b517dcaa 100644--- a/studio/backend/core/rag/reembed.py+++ b/studio/backend/core/rag/reembed.py@@ -92,10 +92,6 @@ def stale_counts() -> dict:return {"documents": len(rows) - archived, "archived": archived}-# Chunks were sized with the previous model's tokenizer.-_clip = embeddings.clip_to_limit--def _reembed_document(job: _Job, doc: dict, model: str) -> bool:conn = rag_db.get_connection()try:@@ -106,12 +102,9 @@ def _reembed_document(job: _Job, doc: dict, model: str) -> bool:conn.close()# Per document: an ST failure can hand the model to llama-server, whose tokenizer and limit differ.count, limit = embeddings.token_counter(model), embeddings.max_tokens(model)- texts = []- clipped = 0- for row in rows:- text, was_clipped = _clip(row["text"], count, limit)- texts.append(text)- clipped += was_clipped+ clips = embeddings.clip_many_to_limit([row["text"] for row in rows], count, limit)+ texts = [text for text, _ in clips]+ clipped = sum(cut for _, cut in clips)def on_progress(done: int, total: int) -> None:if job.stop.is_set() or account_is_retired():diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 5cd67a41c88..5edc7efba0f 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -38461,7 +38461,7 @@ async def _studio_embeddings(status_code = 400,detail = f"'input' exceeds the {limit}-token limit of {label}.",)- inputs = [rag_embeddings.clip_to_limit(text, count, limit)[0] for text in inputs]+ inputs = [text for text, _ in rag_embeddings.clip_many_to_limit(inputs, count, limit)]logger.warning("Embedding input over the %d-token limit of %s truncated to fit", limit, label)diff --git a/studio/backend/tests/test_openai_embeddings_studio_fallback.py b/studio/backend/tests/test_openai_embeddings_studio_fallback.pyindex 9b080e6d982..5cbdb874b73 100644--- a/studio/backend/tests/test_openai_embeddings_studio_fallback.py+++ b/studio/backend/tests/test_openai_embeddings_studio_fallback.py@@ -246,6 +246,32 @@ def test_over_length_input_is_truncated_by_default(studio_embedder):assert payload["usage"]["prompt_tokens"] == 5+def test_over_length_input_is_truncated_with_batched_counts(studio_embedder):+ encoded, singles, batches = [], [], []++ def _count(text):+ singles.append(text)+ return len(text)++ def _many(texts):+ batches.append(list(texts))+ return [len(t) for t in texts]++ _count.many = _many+ studio_embedder.setattr(+ rag_embeddings, "encode", lambda texts, **_: encoded.append(list(texts)) or _vectors(texts)+ )+ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ studio_embedder.setattr(rag_embeddings, "token_counter", lambda model_name = None: _count)+ studio_embedder.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 3)+ payload = _call({"input": ["ab", "abcd", "abcdefgh"]})+ assert encoded == [["ab", "abc", "abc"]]+ assert payload["usage"]["prompt_tokens"] == 8+ assert singles == [] and batches[0] == ["ab", "abcd", "abcdefgh"]++def test_over_length_input_is_rejected_when_truncate_is_false(studio_embedder):studio_embedder.setattr(inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)diff --git a/studio/backend/tests/test_rag_reembed.py b/studio/backend/tests/test_rag_reembed.pyindex 16233b7ee86..b0ff58fc49a 100644--- a/studio/backend/tests/test_rag_reembed.py+++ b/studio/backend/tests/test_rag_reembed.py@@ -389,3 +389,54 @@ def test_a_swap_committed_mid_search_cannot_pass_an_old_space_score(models, tmp_with _connect() as c:hits = retrieval.retrieve_dense(c, scope, "text a", k = 5, model_name = NEWER)assert hits == []+++def test_a_many_chunk_document_clips_its_chunks_in_batched_counts(models, tmp_path, monkeypatch):+ words = [f"w{i}" for i in range(1700)]+ document_id = _ingest(tmp_path, store.kb_scope("K9"), "long.txt", " ".join(words))+ _, chunks = _row(document_id)+ assert len(chunks) > 2+ calls = {"many": [], "single": 0}++ def _counter(model_name = None):+ def _count(text):+ calls["single"] += 1+ return len(text.split())++ def _many(texts):+ calls["many"].append(len(texts))+ return [len(t.split()) for t in texts]++ _count.many = _many+ return _count++ sizes = sorted(len(text.split()) for _, text in chunks)+ limit = sizes[0]+ assert sizes[0] < sizes[-1]+ monkeypatch.setattr(embeddings, "token_counter", _counter)+ monkeypatch.setattr(embeddings, "max_tokens", lambda model_name = None: limit)+ word_count = lambda t: len(t.split())+ expected = [embeddings.clip_to_limit(text, word_count, limit) for _, text in chunks]+ assert any(cut for _, cut in expected) and not all(cut for _, cut in expected)+ models["model"] = NEW+ status = _run_job()+ sent = [t for m, t in models["encoded"] if m == NEW]+ assert sent == [[text for text, _ in expected]]+ assert status["clipped_chunks"] == sum(cut for _, cut in expected)+ assert calls["single"] == 0+ assert calls["many"][0] == len(chunks)+ assert len(calls["many"]) <= 2 + max(len(text) for _, text in chunks).bit_length()+++def test_clip_many_matches_clip_to_limit_per_text():+ import random++ rng = random.Random(3407)+ texts = [+ " ".join(f"w{rng.randint(0, 99)}" for _ in range(rng.randint(0, 60))) for _ in range(40)+ ]+ word_count = lambda t: len(t.split())+ word_count.many = lambda ts: [len(t.split()) for t in ts]+ for limit in (None, 0, 1, 7, 30, 100):+ expected = [embeddings.clip_to_limit(t, word_count, limit) for t in texts]+ assert embeddings.clip_many_to_limit(texts, word_count, limit) == expected