Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: truncate over-length /v1/embeddings inputs instead of failing (#14364)
Daniel Han439411c65b32
diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.pyindex 558fa6f4009..b8d7e509d36 100644--- a/studio/backend/core/rag/embeddings.py+++ b/studio/backend/core/rag/embeddings.py@@ -1528,6 +1528,20 @@ def max_tokens(model_name: str | None = None) -> int | None:return _get_backend(model_name).max_tokens(model_name = model_name)+def clip_to_limit(text: str, count, limit: int | None) -> tuple[str, bool]:+ """The longest prefix of ``text`` within ``limit`` tokens by ``count``, and whether it was cut."""+ if not limit or count(text) <= limit:+ return text, False+ lo, hi = 0, len(text)+ while lo < hi:+ mid = (lo + hi + 1) // 2+ if count(text[:mid]) <= limit:+ lo = mid+ else:+ hi = mid - 1+ return text[:lo], True++def chunk_tokens(model_name: str | None = None) -> int:limit = max_tokens(model_name)return min(config.CHUNK_TOKENS, limit) if limit else config.CHUNK_TOKENSdiff --git a/studio/backend/core/rag/reembed.py b/studio/backend/core/rag/reembed.pyindex f62ce305e6d..e93c79b1f16 100644--- a/studio/backend/core/rag/reembed.py+++ b/studio/backend/core/rag/reembed.py@@ -94,19 +94,8 @@ def stale_counts() -> dict:return {"documents": len(rows) - archived, "archived": archived}-def _clip(text: str, count, limit: int | None) -> tuple[str, bool]:- """The longest prefix of ``text`` the embedder accepts. Chunks were sized with the previous- model's tokenizer, and llama-server refuses an input past its batch instead of truncating."""- if not limit or count(text) <= limit:- return text, False- lo, hi = 0, len(text)- while lo < hi:- mid = (lo + hi + 1) // 2- if count(text[:mid]) <= limit:- lo = mid- else:- hi = mid - 1- return text[:lo], True+# Chunks were sized with the previous model's tokenizer.+_clip = embeddings.clip_to_limitdef _reembed_document(job: _Job, doc: dict, model: str) -> bool:diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 830e89636c2..794fdb6f7dd 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -40118,9 +40118,18 @@ def _embeddings_items(body: dict, *, tokens_ok: bool) -> list:raise HTTPException(status_code = 400, detail = "'encoding_format' must be 'float' or 'base64'.")+ _embeddings_truncate(body)return items+def _embeddings_truncate(body: dict) -> bool:+ """``truncate`` (Ollama / TEI spelling), default on (#14309)."""+ truncate = body.get("truncate", True)+ if not isinstance(truncate, bool):+ raise HTTPException(status_code = 400, detail = "'truncate' must be a boolean.")+ return truncate++def _embeddings_texts(body: dict) -> list[str]:return _embeddings_items(body, tokens_ok = False)@@ -40361,12 +40370,19 @@ async def _studio_embeddings(# setting, mixing one model's limit and dimension with another's vectors.limit = rag_embeddings.max_tokens(model_name)count = rag_embeddings.token_counter(model_name)- token_counts = [count(text) for text in texts]+ inputs = texts+ token_counts = [count(text) for text in inputs]if limit and max(token_counts) > limit:- raise HTTPException(- status_code = 400,- detail = f"'input' exceeds the {limit}-token limit of {label}.",+ if not _embeddings_truncate(body):+ raise HTTPException(+ status_code = 400,+ detail = f"'input' exceeds the {limit}-token limit of {label}.",+ )+ inputs = [rag_embeddings.clip_to_limit(text, count, limit)[0] for text in inputs]+ logger.warning(+ "Embedding input over the %d-token limit of %s truncated to fit", limit, label)+ token_counts = [count(text) for text in inputs]if dimensions is not None and dimensions != rag_embeddings.dim(model_name):raise HTTPException(status_code = 400, detail = f"'dimensions' is not supported by {label}."@@ -40374,7 +40390,7 @@ async def _studio_embeddings(# encode_with_identity: an ST failure swaps the process to llama-server mid-encode, and# reporting the configured name anyway files two spaces under one label.vectors, identity = rag_embeddings.encode_with_identity(- texts, model_name = model_name, normalize = True+ inputs, model_name = model_name, normalize = True)return vectors, identity, sum(token_counts), limit@@ -40471,6 +40487,75 @@ async def _studio_embeddings(return Response(content = json.dumps(payload), media_type = "application/json")+# llama-server's refusals past one micro-batch (500) or the slot context (400), server-context.cpp;+# a splittable slot (LAST pooling, e.g. Qwen3-Embedding) refuses n_tokens >= n_ctx instead.+_LLAMA_EMBED_OVERFLOW_RE = re.compile(+ r"input \(\d+ tokens\) is (?:too large to process\. increase the physical batch size "+ r"\(current batch size: (\d+)\)|larger than the max context size \((\d+) tokens\))"+ r"|request \(\d+ tokens\) exceeds the available context size \((\d+) tokens\)"+)+++def _llama_embedding_overflow_limit(resp) -> Optional[int]:+ if resp.status_code not in (400, 500):+ return None+ try:+ match = _LLAMA_EMBED_OVERFLOW_RE.search(resp.json()["error"]["message"])+ except Exception: # noqa: BLE001+ return None+ if not match:+ return None+ if match.group(3):+ return int(match.group(3)) - 1+ return int(match.group(1) or match.group(2))+++async def _truncated_llama_embedding_inputs(+ client, base_url, headers, value, limit, cache: dict+) -> Optional[list]:+ """Strings past ``limit`` cut to their first tokens, keeping the special tokens llama-server+ adds around a string. None when nothing was cut. ``cache`` spans the retries."""+ items = [value] if isinstance(value, str) else value+ # A top-level array holding numbers is one prompt (tokens, or tokens mixed with strings).+ if not isinstance(items, list) or any(not isinstance(item, (str, list)) for item in items):+ return None++ async def tokenize(text: str, add_special: bool) -> Optional[list]:+ r = await client.post(+ f"{base_url}/tokenize",+ json = {"content": text, "add_special": add_special},+ headers = headers,+ timeout = _DEFAULT_FIRST_TOKEN_TIMEOUT_S,+ )+ try:+ tokens = r.json()["tokens"] if r.status_code == 200 else None+ except (ValueError, KeyError, TypeError):+ tokens = None+ return tokens if isinstance(tokens, list) else None++ out, cut = [], False+ for index, item in enumerate(items):+ # Short ASCII skips /tokenize (>= 1 byte per token); normalization can expand non-ASCII.+ if isinstance(item, str) and (not item.isascii() or len(item) + 8 > limit):+ if index not in cache:+ cache[index] = [await tokenize(item, True), None]+ full, bare = cache[index]+ if full is None:+ return None+ if len(full) > limit:+ if bare is None:+ bare = cache[index][1] = await tokenize(item, False)+ if bare is None:+ return None+ extra = len(full) - len(bare)+ head = next((h for h in range(extra + 1) if full[h : h + len(bare)] == bare), None)+ if head is not None and limit > extra:+ item = full[:head] + bare[: limit - extra] + full[head + len(bare) :]+ cut = True+ out.append(item)+ return out if cut else None++def _embedding_width(resp) -> Optional[int]:try:embedding = resp.json()["data"][0]["embedding"]@@ -40590,6 +40675,8 @@ async def openai_embeddings(request: Request, current_subject: str = Depends(get# llama-server fetches media URLs inside multimodal items; a managed account sends text or tokens only._embeddings_items(body, tokens_ok = True)dimensions = body.pop("dimensions", None)+ truncate = _embeddings_truncate(body)+ body.pop("truncate", None)target_url = f"{llama_backend.base_url}/v1/embeddings"prompt_text = _flatten_monitor_prompt(body.get("input", ""))monitor_id = None@@ -40628,13 +40715,35 @@ async def openai_embeddings(request: Request, current_subject: str = Depends(get_direct_llama_request_started()try:try:+ upstream_headers = _openai_passthrough_upstream_headers(llama_backend = llama_backend)resp = await _client.post(target_url,json = body,- headers = _openai_passthrough_upstream_headers(llama_backend = llama_backend),+ headers = upstream_headers,timeout = _DEFAULT_FIRST_TOKEN_TIMEOUT_S,)- except httpx.RequestError:+ original, cache, limit = body.get("input"), {}, None+ for _ in range(2):+ reported = _llama_embedding_overflow_limit(resp) if truncate else None+ if not reported:+ break+ # Cut the client's inputs, not the token arrays the first retry sent.+ limit = min(limit or reported, reported)+ inputs = await _truncated_llama_embedding_inputs(+ _client, llama_backend.base_url, upstream_headers, original, limit, cache+ )+ if not inputs:+ break+ logger.warning("Embedding input over the %d-token limit truncated to fit", limit)+ body["input"] = inputs+ resp = await _client.post(+ target_url,+ json = body,+ headers = upstream_headers,+ timeout = _DEFAULT_FIRST_TOKEN_TIMEOUT_S,+ )+ # RuntimeError: a retry or /tokenize call sent after the watcher already closed the client.+ except (httpx.RequestError, RuntimeError):# The watcher closed the client out from under the request: report the cancel, not a transport failure._raise_if_nonstreaming_request_cancelled(_cancel_event)raisediff --git a/studio/backend/tests/test_openai_embeddings_studio_fallback.py b/studio/backend/tests/test_openai_embeddings_studio_fallback.pyindex 12062632922..eaee4f76a4b 100644--- a/studio/backend/tests/test_openai_embeddings_studio_fallback.py+++ b/studio/backend/tests/test_openai_embeddings_studio_fallback.py@@ -3,6 +3,7 @@import asyncioimport base64+import copyimport osimport sysimport threading@@ -227,16 +228,173 @@ def test_invalid_requests_are_400(studio_embedder, body):assert _http_error(body).status_code == 400-def test_over_length_input_is_rejected_not_truncated(studio_embedder):+def test_over_length_input_is_truncated_by_default(studio_embedder):+ encoded = []++ def _encode(texts, **_):+ encoded.append(list(texts))+ return _vectors(texts)++ studio_embedder.setattr(rag_embeddings, "encode", _encode)+ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ studio_embedder.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 3)+ payload = _call({"input": ["ab", "abcd"]})+ assert encoded == [["ab", "abc"]]+ assert len(payload["data"]) == 2+ assert payload["usage"]["prompt_tokens"] == 5+++def test_over_length_input_is_rejected_when_truncate_is_false(studio_embedder):studio_embedder.setattr(inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False))studio_embedder.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 3)- error = _http_error({"input": ["ab", "abcd"]})+ error = _http_error({"input": ["ab", "abcd"], "truncate": False})assert error.status_code == 400assert "3-token limit" in error.detail+def test_non_boolean_truncate_is_400(studio_embedder):+ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ error = _http_error({"input": "ab", "truncate": "yes"})+ assert error.status_code == 400+ assert "'truncate'" in error.detail+++_CLS, _SEP = 101, 102+++def _llama_overflow_server(+ ubatch,+ n_ctx,+ can_split = False,+):+ import httpx++ embedded, tokenized = [], []++ def _tokens(item):+ if isinstance(item, list):+ return item+ return [_CLS, *(len(word) for word in item.split()), _SEP]++ class _Client:+ async def post(+ self,+ url,+ *_args,+ json = None,+ **_kwargs,+ ):+ if url.endswith("/tokenize"):+ tokenized.append(json["content"])+ tokens = _tokens(json["content"])+ return httpx.Response(+ 200, json = {"tokens": tokens if json["add_special"] else tokens[1:-1]}+ )+ # The route reuses the dict.+ embedded.append(copy.deepcopy(json))+ items = [json["input"]] if isinstance(json["input"], str) else json["input"]+ for item in items:+ n = len(_tokens(item))+ if can_split and n >= n_ctx:+ return httpx.Response(+ 400,+ json = {+ "error": {+ "message": f"request ({n} tokens) exceeds the available context "+ f"size ({n_ctx} tokens), try increasing it"+ }+ },+ )+ if can_split:+ continue+ if n > ubatch:+ return httpx.Response(+ 500,+ json = {+ "error": {+ "message": f"input ({n} tokens) is too large to process. increase "+ f"the physical batch size (current batch size: {ubatch})"+ }+ },+ )+ if n > n_ctx:+ return httpx.Response(+ 400,+ json = {+ "error": {+ "message": f"input ({n} tokens) is larger than the max context "+ f"size ({n_ctx} tokens). skipping"+ }+ },+ )+ return httpx.Response(200, json = {"data": [{"embedding": [0.5]} for _ in items]})++ async def aclose(self):+ return None++ return _Client, embedded, tokenized+++_OVERFLOW_LIMITS = [(4, 512), (512, 4)]+++def _resident_embedding_gguf(studio_embedder, client):+ studio_embedder.setattr(inference_route, "_cancelable_nonstreaming_client", client)+ studio_embedder.setattr(+ inference_route,+ "get_llama_cpp_backend",+ lambda: SimpleNamespace(+ is_loaded = True,+ is_embedding_gguf = True,+ base_url = "http://llama.test",+ context_length = 512,+ model_identifier = "org/E-GGUF",+ ),+ )++[email protected]("ubatch, n_ctx", _OVERFLOW_LIMITS)+def test_resident_embedding_gguf_truncates_an_oversized_input(studio_embedder, ubatch, n_ctx):+ client, embedded, _ = _llama_overflow_server(ubatch, n_ctx)+ _resident_embedding_gguf(studio_embedder, client)+ payload = _call({"input": ["aa bbb c dddd e", "x y"], "model": "org/E-GGUF"})+ assert len(payload["data"]) == 2+ assert len(embedded) == 2+ assert embedded[1]["input"] == [[_CLS, 2, 3, _SEP], "x y"]+ assert "truncate" not in embedded[0]+++def test_resident_embedding_gguf_truncates_for_a_splittable_slot(studio_embedder):+ client, embedded, _ = _llama_overflow_server(512, 5, can_split = True)+ _resident_embedding_gguf(studio_embedder, client)+ payload = _call({"input": ["aa bbb c dddd e", "x y"], "model": "org/E-GGUF"})+ assert len(payload["data"]) == 2+ assert embedded[1]["input"] == [[_CLS, 2, 3, _SEP], "x y"]++[email protected]("ubatch, n_ctx", _OVERFLOW_LIMITS)+def test_resident_embedding_gguf_keeps_the_error_when_truncate_is_false(+ studio_embedder, ubatch, n_ctx+):+ client, embedded, _ = _llama_overflow_server(ubatch, n_ctx)+ _resident_embedding_gguf(studio_embedder, client)+ response = asyncio.run(+ inference_route.openai_embeddings(+ _Request({"input": "aa bbb c dddd e", "model": "org/E-GGUF", "truncate": False}),+ "tester",+ )+ )+ assert response.status_code == (500 if ubatch < n_ctx else 400)+ assert len(embedded) == 1+ assert "truncate" not in embedded[0]++def test_embedder_failure_is_502(studio_embedder):def boom(*_args, **_kwargs):raise RuntimeError("llama-server embedder POST /v1/embeddings -> 500")@@ -1036,7 +1194,7 @@ def test_local_path_models_are_not_exposed(studio_embedder, tmp_path):reported = _call({"input": "alpha"})["model"]assert reported.startswith("sentence-transformers:bge-") and str(tmp_path) not in reportedstudio_embedder.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 3)- error = _http_error({"input": "alphabet"})+ error = _http_error({"input": "alphabet", "truncate": False})assert error.status_code == 400assert "bge" in error.detail and str(tmp_path) not in error.detail@@ -1458,3 +1616,59 @@ def test_stashed_gguf_embeds_reads_the_stashed_model_header(monkeypatch, tmp_patmonkeypatch.setattr(inference_route, "_probe_backend", _Probe)assert inference_route._stashed_gguf_embeds() is True+++def test_resident_embedding_gguf_recuts_for_a_context_below_the_batch(studio_embedder):+ client, embedded, tokenized = _llama_overflow_server(6, 4)+ _resident_embedding_gguf(studio_embedder, client)+ payload = _call({"input": ["aa bbb c dddd e ff gg hh"], "model": "org/E-GGUF"})+ assert len(payload["data"]) == 1+ assert [body["input"] for body in embedded[1:]] == [+ [[_CLS, 2, 3, 1, 4, _SEP]],+ [[_CLS, 2, 3, _SEP]],+ ]+ assert len(tokenized) == 2+++def test_resident_embedding_gguf_skips_tokenizing_strings_that_cannot_overflow(studio_embedder):+ client, embedded, tokenized = _llama_overflow_server(20, 512)+ _resident_embedding_gguf(studio_embedder, client)+ long = " ".join(["a"] * 30)+ _call({"input": [long, "x", "y z", "\ufdfa"], "model": "org/E-GGUF"})+ assert embedded[1]["input"][1:] == ["x", "y z", "\ufdfa"]+ assert set(tokenized) == {long, "\ufdfa"}+++def test_resident_embedding_gguf_cancel_during_the_retry_is_499(studio_embedder):+ from fastapi import HTTPException++ server, _, _ = _llama_overflow_server(4, 512)+ request = _Request({"input": ["aa bbb c dddd e"], "model": "org/E-GGUF"})+ disconnected = False++ async def _is_disconnected():+ return disconnected++ request.is_disconnected = _is_disconnected++ class _Client(server):+ closed = False++ async def post(self, url, *args, **kwargs):+ nonlocal disconnected+ if self.closed:+ raise RuntimeError("Cannot send a request, as the client has been closed.")+ response = await super().post(url, *args, **kwargs)+ if response.status_code == 500:+ disconnected = True+ while not self.closed:+ await asyncio.sleep(0.05)+ return response++ async def aclose(self):+ self.closed = True++ _resident_embedding_gguf(studio_embedder, _Client)+ with pytest.raises(HTTPException) as exc:+ asyncio.run(inference_route.openai_embeddings(request, "tester"))+ assert exc.value.status_code == 499