Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: truncate over-length /v1/embeddings inputs instead of failing (#14364)

Daniel Han439411c65b32
diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.py
index 558fa6f4009..b8d7e509d36 100644
--- a/studio/backend/core/rag/embeddings.py
+++ b/studio/backend/core/rag/embeddings.py
@@ -1528,6 +1528,20 @@ def max_tokens(model_name: str | None = None) -> int | None:
return _get_backend(model_name).max_tokens(model_name = model_name)
+def clip_to_limit(text: str, count, limit: int | None) -> tuple[str, bool]:
+ """The longest prefix of ``text`` within ``limit`` tokens by ``count``, and whether it was cut."""
+ if not limit or count(text) <= limit:
+ return text, False
+ lo, hi = 0, len(text)
+ while lo < hi:
+ mid = (lo + hi + 1) // 2
+ if count(text[:mid]) <= limit:
+ lo = mid
+ else:
+ hi = mid - 1
+ return text[:lo], True
+
+
def chunk_tokens(model_name: str | None = None) -> int:
limit = max_tokens(model_name)
return min(config.CHUNK_TOKENS, limit) if limit else config.CHUNK_TOKENS
diff --git a/studio/backend/core/rag/reembed.py b/studio/backend/core/rag/reembed.py
index f62ce305e6d..e93c79b1f16 100644
--- a/studio/backend/core/rag/reembed.py
+++ b/studio/backend/core/rag/reembed.py
@@ -94,19 +94,8 @@ def stale_counts() -> dict:
return {"documents": len(rows) - archived, "archived": archived}
-def _clip(text: str, count, limit: int | None) -> tuple[str, bool]:
- """The longest prefix of ``text`` the embedder accepts. Chunks were sized with the previous
- model's tokenizer, and llama-server refuses an input past its batch instead of truncating."""
- if not limit or count(text) <= limit:
- return text, False
- lo, hi = 0, len(text)
- while lo < hi:
- mid = (lo + hi + 1) // 2
- if count(text[:mid]) <= limit:
- lo = mid
- else:
- hi = mid - 1
- return text[:lo], True
+# Chunks were sized with the previous model's tokenizer.
+_clip = embeddings.clip_to_limit
def _reembed_document(job: _Job, doc: dict, model: str) -> bool:
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 830e89636c2..794fdb6f7dd 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -40118,9 +40118,18 @@ def _embeddings_items(body: dict, *, tokens_ok: bool) -> list:
raise HTTPException(
status_code = 400, detail = "'encoding_format' must be 'float' or 'base64'."
)
+ _embeddings_truncate(body)
return items
+def _embeddings_truncate(body: dict) -> bool:
+ """``truncate`` (Ollama / TEI spelling), default on (#14309)."""
+ truncate = body.get("truncate", True)
+ if not isinstance(truncate, bool):
+ raise HTTPException(status_code = 400, detail = "'truncate' must be a boolean.")
+ return truncate
+
+
def _embeddings_texts(body: dict) -> list[str]:
return _embeddings_items(body, tokens_ok = False)
@@ -40361,12 +40370,19 @@ async def _studio_embeddings(
# setting, mixing one model's limit and dimension with another's vectors.
limit = rag_embeddings.max_tokens(model_name)
count = rag_embeddings.token_counter(model_name)
- token_counts = [count(text) for text in texts]
+ inputs = texts
+ token_counts = [count(text) for text in inputs]
if limit and max(token_counts) > limit:
- raise HTTPException(
- status_code = 400,
- detail = f"'input' exceeds the {limit}-token limit of {label}.",
+ if not _embeddings_truncate(body):
+ raise HTTPException(
+ status_code = 400,
+ detail = f"'input' exceeds the {limit}-token limit of {label}.",
+ )
+ inputs = [rag_embeddings.clip_to_limit(text, count, limit)[0] for text in inputs]
+ logger.warning(
+ "Embedding input over the %d-token limit of %s truncated to fit", limit, label
)
+ token_counts = [count(text) for text in inputs]
if dimensions is not None and dimensions != rag_embeddings.dim(model_name):
raise HTTPException(
status_code = 400, detail = f"'dimensions' is not supported by {label}."
@@ -40374,7 +40390,7 @@ async def _studio_embeddings(
# encode_with_identity: an ST failure swaps the process to llama-server mid-encode, and
# reporting the configured name anyway files two spaces under one label.
vectors, identity = rag_embeddings.encode_with_identity(
- texts, model_name = model_name, normalize = True
+ inputs, model_name = model_name, normalize = True
)
return vectors, identity, sum(token_counts), limit
@@ -40471,6 +40487,75 @@ async def _studio_embeddings(
return Response(content = json.dumps(payload), media_type = "application/json")
+# llama-server's refusals past one micro-batch (500) or the slot context (400), server-context.cpp;
+# a splittable slot (LAST pooling, e.g. Qwen3-Embedding) refuses n_tokens >= n_ctx instead.
+_LLAMA_EMBED_OVERFLOW_RE = re.compile(
+ r"input \(\d+ tokens\) is (?:too large to process\. increase the physical batch size "
+ r"\(current batch size: (\d+)\)|larger than the max context size \((\d+) tokens\))"
+ r"|request \(\d+ tokens\) exceeds the available context size \((\d+) tokens\)"
+)
+
+
+def _llama_embedding_overflow_limit(resp) -> Optional[int]:
+ if resp.status_code not in (400, 500):
+ return None
+ try:
+ match = _LLAMA_EMBED_OVERFLOW_RE.search(resp.json()["error"]["message"])
+ except Exception: # noqa: BLE001
+ return None
+ if not match:
+ return None
+ if match.group(3):
+ return int(match.group(3)) - 1
+ return int(match.group(1) or match.group(2))
+
+
+async def _truncated_llama_embedding_inputs(
+ client, base_url, headers, value, limit, cache: dict
+) -> Optional[list]:
+ """Strings past ``limit`` cut to their first tokens, keeping the special tokens llama-server
+ adds around a string. None when nothing was cut. ``cache`` spans the retries."""
+ items = [value] if isinstance(value, str) else value
+ # A top-level array holding numbers is one prompt (tokens, or tokens mixed with strings).
+ if not isinstance(items, list) or any(not isinstance(item, (str, list)) for item in items):
+ return None
+
+ async def tokenize(text: str, add_special: bool) -> Optional[list]:
+ r = await client.post(
+ f"{base_url}/tokenize",
+ json = {"content": text, "add_special": add_special},
+ headers = headers,
+ timeout = _DEFAULT_FIRST_TOKEN_TIMEOUT_S,
+ )
+ try:
+ tokens = r.json()["tokens"] if r.status_code == 200 else None
+ except (ValueError, KeyError, TypeError):
+ tokens = None
+ return tokens if isinstance(tokens, list) else None
+
+ out, cut = [], False
+ for index, item in enumerate(items):
+ # Short ASCII skips /tokenize (>= 1 byte per token); normalization can expand non-ASCII.
+ if isinstance(item, str) and (not item.isascii() or len(item) + 8 > limit):
+ if index not in cache:
+ cache[index] = [await tokenize(item, True), None]
+ full, bare = cache[index]
+ if full is None:
+ return None
+ if len(full) > limit:
+ if bare is None:
+ bare = cache[index][1] = await tokenize(item, False)
+ if bare is None:
+ return None
+ extra = len(full) - len(bare)
+ head = next((h for h in range(extra + 1) if full[h : h + len(bare)] == bare), None)
+ if head is not None and limit > extra:
+ item = full[:head] + bare[: limit - extra] + full[head + len(bare) :]
+ cut = True
+ out.append(item)
+ return out if cut else None
+
+
def _embedding_width(resp) -> Optional[int]:
try:
embedding = resp.json()["data"][0]["embedding"]
@@ -40590,6 +40675,8 @@ async def openai_embeddings(request: Request, current_subject: str = Depends(get
# llama-server fetches media URLs inside multimodal items; a managed account sends text or tokens only.
_embeddings_items(body, tokens_ok = True)
dimensions = body.pop("dimensions", None)
+ truncate = _embeddings_truncate(body)
+ body.pop("truncate", None)
target_url = f"{llama_backend.base_url}/v1/embeddings"
prompt_text = _flatten_monitor_prompt(body.get("input", ""))
monitor_id = None
@@ -40628,13 +40715,35 @@ async def openai_embeddings(request: Request, current_subject: str = Depends(get
_direct_llama_request_started()
try:
try:
+ upstream_headers = _openai_passthrough_upstream_headers(llama_backend = llama_backend)
resp = await _client.post(
target_url,
json = body,
- headers = _openai_passthrough_upstream_headers(llama_backend = llama_backend),
+ headers = upstream_headers,
timeout = _DEFAULT_FIRST_TOKEN_TIMEOUT_S,
)
- except httpx.RequestError:
+ original, cache, limit = body.get("input"), {}, None
+ for _ in range(2):
+ reported = _llama_embedding_overflow_limit(resp) if truncate else None
+ if not reported:
+ break
+ # Cut the client's inputs, not the token arrays the first retry sent.
+ limit = min(limit or reported, reported)
+ inputs = await _truncated_llama_embedding_inputs(
+ _client, llama_backend.base_url, upstream_headers, original, limit, cache
+ )
+ if not inputs:
+ break
+ logger.warning("Embedding input over the %d-token limit truncated to fit", limit)
+ body["input"] = inputs
+ resp = await _client.post(
+ target_url,
+ json = body,
+ headers = upstream_headers,
+ timeout = _DEFAULT_FIRST_TOKEN_TIMEOUT_S,
+ )
+ # RuntimeError: a retry or /tokenize call sent after the watcher already closed the client.
+ except (httpx.RequestError, RuntimeError):
# The watcher closed the client out from under the request: report the cancel, not a transport failure.
_raise_if_nonstreaming_request_cancelled(_cancel_event)
raise
diff --git a/studio/backend/tests/test_openai_embeddings_studio_fallback.py b/studio/backend/tests/test_openai_embeddings_studio_fallback.py
index 12062632922..eaee4f76a4b 100644
--- a/studio/backend/tests/test_openai_embeddings_studio_fallback.py
+++ b/studio/backend/tests/test_openai_embeddings_studio_fallback.py
@@ -3,6 +3,7 @@
import asyncio
import base64
+import copy
import os
import sys
import threading
@@ -227,16 +228,173 @@ def test_invalid_requests_are_400(studio_embedder, body):
assert _http_error(body).status_code == 400
-def test_over_length_input_is_rejected_not_truncated(studio_embedder):
+def test_over_length_input_is_truncated_by_default(studio_embedder):
+ encoded = []
+
+ def _encode(texts, **_):
+ encoded.append(list(texts))
+ return _vectors(texts)
+
+ studio_embedder.setattr(rag_embeddings, "encode", _encode)
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ studio_embedder.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 3)
+ payload = _call({"input": ["ab", "abcd"]})
+ assert encoded == [["ab", "abc"]]
+ assert len(payload["data"]) == 2
+ assert payload["usage"]["prompt_tokens"] == 5
+
+
+def test_over_length_input_is_rejected_when_truncate_is_false(studio_embedder):
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
studio_embedder.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 3)
- error = _http_error({"input": ["ab", "abcd"]})
+ error = _http_error({"input": ["ab", "abcd"], "truncate": False})
assert error.status_code == 400
assert "3-token limit" in error.detail
+def test_non_boolean_truncate_is_400(studio_embedder):
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ error = _http_error({"input": "ab", "truncate": "yes"})
+ assert error.status_code == 400
+ assert "'truncate'" in error.detail
+
+
+_CLS, _SEP = 101, 102
+
+
+def _llama_overflow_server(
+ ubatch,
+ n_ctx,
+ can_split = False,
+):
+ import httpx
+
+ embedded, tokenized = [], []
+
+ def _tokens(item):
+ if isinstance(item, list):
+ return item
+ return [_CLS, *(len(word) for word in item.split()), _SEP]
+
+ class _Client:
+ async def post(
+ self,
+ url,
+ *_args,
+ json = None,
+ **_kwargs,
+ ):
+ if url.endswith("/tokenize"):
+ tokenized.append(json["content"])
+ tokens = _tokens(json["content"])
+ return httpx.Response(
+ 200, json = {"tokens": tokens if json["add_special"] else tokens[1:-1]}
+ )
+ # The route reuses the dict.
+ embedded.append(copy.deepcopy(json))
+ items = [json["input"]] if isinstance(json["input"], str) else json["input"]
+ for item in items:
+ n = len(_tokens(item))
+ if can_split and n >= n_ctx:
+ return httpx.Response(
+ 400,
+ json = {
+ "error": {
+ "message": f"request ({n} tokens) exceeds the available context "
+ f"size ({n_ctx} tokens), try increasing it"
+ }
+ },
+ )
+ if can_split:
+ continue
+ if n > ubatch:
+ return httpx.Response(
+ 500,
+ json = {
+ "error": {
+ "message": f"input ({n} tokens) is too large to process. increase "
+ f"the physical batch size (current batch size: {ubatch})"
+ }
+ },
+ )
+ if n > n_ctx:
+ return httpx.Response(
+ 400,
+ json = {
+ "error": {
+ "message": f"input ({n} tokens) is larger than the max context "
+ f"size ({n_ctx} tokens). skipping"
+ }
+ },
+ )
+ return httpx.Response(200, json = {"data": [{"embedding": [0.5]} for _ in items]})
+
+ async def aclose(self):
+ return None
+
+ return _Client, embedded, tokenized
+
+
+_OVERFLOW_LIMITS = [(4, 512), (512, 4)]
+
+
+def _resident_embedding_gguf(studio_embedder, client):
+ studio_embedder.setattr(inference_route, "_cancelable_nonstreaming_client", client)
+ studio_embedder.setattr(
+ inference_route,
+ "get_llama_cpp_backend",
+ lambda: SimpleNamespace(
+ is_loaded = True,
+ is_embedding_gguf = True,
+ base_url = "http://llama.test",
+ context_length = 512,
+ model_identifier = "org/E-GGUF",
+ ),
+ )
+
+
[email protected]("ubatch, n_ctx", _OVERFLOW_LIMITS)
+def test_resident_embedding_gguf_truncates_an_oversized_input(studio_embedder, ubatch, n_ctx):
+ client, embedded, _ = _llama_overflow_server(ubatch, n_ctx)
+ _resident_embedding_gguf(studio_embedder, client)
+ payload = _call({"input": ["aa bbb c dddd e", "x y"], "model": "org/E-GGUF"})
+ assert len(payload["data"]) == 2
+ assert len(embedded) == 2
+ assert embedded[1]["input"] == [[_CLS, 2, 3, _SEP], "x y"]
+ assert "truncate" not in embedded[0]
+
+
+def test_resident_embedding_gguf_truncates_for_a_splittable_slot(studio_embedder):
+ client, embedded, _ = _llama_overflow_server(512, 5, can_split = True)
+ _resident_embedding_gguf(studio_embedder, client)
+ payload = _call({"input": ["aa bbb c dddd e", "x y"], "model": "org/E-GGUF"})
+ assert len(payload["data"]) == 2
+ assert embedded[1]["input"] == [[_CLS, 2, 3, _SEP], "x y"]
+
+
[email protected]("ubatch, n_ctx", _OVERFLOW_LIMITS)
+def test_resident_embedding_gguf_keeps_the_error_when_truncate_is_false(
+ studio_embedder, ubatch, n_ctx
+):
+ client, embedded, _ = _llama_overflow_server(ubatch, n_ctx)
+ _resident_embedding_gguf(studio_embedder, client)
+ response = asyncio.run(
+ inference_route.openai_embeddings(
+ _Request({"input": "aa bbb c dddd e", "model": "org/E-GGUF", "truncate": False}),
+ "tester",
+ )
+ )
+ assert response.status_code == (500 if ubatch < n_ctx else 400)
+ assert len(embedded) == 1
+ assert "truncate" not in embedded[0]
+
+
def test_embedder_failure_is_502(studio_embedder):
def boom(*_args, **_kwargs):
raise RuntimeError("llama-server embedder POST /v1/embeddings -> 500")
@@ -1036,7 +1194,7 @@ def test_local_path_models_are_not_exposed(studio_embedder, tmp_path):
reported = _call({"input": "alpha"})["model"]
assert reported.startswith("sentence-transformers:bge-") and str(tmp_path) not in reported
studio_embedder.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 3)
- error = _http_error({"input": "alphabet"})
+ error = _http_error({"input": "alphabet", "truncate": False})
assert error.status_code == 400
assert "bge" in error.detail and str(tmp_path) not in error.detail
@@ -1458,3 +1616,59 @@ def test_stashed_gguf_embeds_reads_the_stashed_model_header(monkeypatch, tmp_pat
monkeypatch.setattr(inference_route, "_probe_backend", _Probe)
assert inference_route._stashed_gguf_embeds() is True
+
+
+def test_resident_embedding_gguf_recuts_for_a_context_below_the_batch(studio_embedder):
+ client, embedded, tokenized = _llama_overflow_server(6, 4)
+ _resident_embedding_gguf(studio_embedder, client)
+ payload = _call({"input": ["aa bbb c dddd e ff gg hh"], "model": "org/E-GGUF"})
+ assert len(payload["data"]) == 1
+ assert [body["input"] for body in embedded[1:]] == [
+ [[_CLS, 2, 3, 1, 4, _SEP]],
+ [[_CLS, 2, 3, _SEP]],
+ ]
+ assert len(tokenized) == 2
+
+
+def test_resident_embedding_gguf_skips_tokenizing_strings_that_cannot_overflow(studio_embedder):
+ client, embedded, tokenized = _llama_overflow_server(20, 512)
+ _resident_embedding_gguf(studio_embedder, client)
+ long = " ".join(["a"] * 30)
+ _call({"input": [long, "x", "y z", "\ufdfa"], "model": "org/E-GGUF"})
+ assert embedded[1]["input"][1:] == ["x", "y z", "\ufdfa"]
+ assert set(tokenized) == {long, "\ufdfa"}
+
+
+def test_resident_embedding_gguf_cancel_during_the_retry_is_499(studio_embedder):
+ from fastapi import HTTPException
+
+ server, _, _ = _llama_overflow_server(4, 512)
+ request = _Request({"input": ["aa bbb c dddd e"], "model": "org/E-GGUF"})
+ disconnected = False
+
+ async def _is_disconnected():
+ return disconnected
+
+ request.is_disconnected = _is_disconnected
+
+ class _Client(server):
+ closed = False
+
+ async def post(self, url, *args, **kwargs):
+ nonlocal disconnected
+ if self.closed:
+ raise RuntimeError("Cannot send a request, as the client has been closed.")
+ response = await super().post(url, *args, **kwargs)
+ if response.status_code == 500:
+ disconnected = True
+ while not self.closed:
+ await asyncio.sleep(0.05)
+ return response
+
+ async def aclose(self):
+ self.closed = True
+
+ _resident_embedding_gguf(studio_embedder, _Client)
+ with pytest.raises(HTTPException) as exc:
+ asyncio.run(inference_route.openai_embeddings(request, "tester"))
+ assert exc.value.status_code == 499