Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio /v1/embeddings: count tokens in one batch and serialize float rows with tolist (#17030)

Daniel Han5e3b9adf98dc
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index f66e7833f88..5577ab958a3 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -38273,6 +38273,11 @@ def _embedding_payload(vector, encoding_format: str):
import base64
import numpy as np
return base64.b64encode(np.asarray(vector, dtype = np.float32).tobytes()).decode("ascii")
+ # float16/32/64 numpy rows only: longdouble tolist() keeps numpy scalars.
+ if getattr(vector, "ndim", None) == 1 and getattr(
+ getattr(vector, "dtype", None), "char", None
+ ) in ("e", "f", "d"):
+ return vector.tolist()
return [float(x) for x in vector]
@@ -38442,7 +38447,7 @@ async def _studio_embeddings(
limit = rag_embeddings.max_tokens(model_name)
count = rag_embeddings.token_counter(model_name)
inputs = texts
- token_counts = [count(text) for text in inputs]
+ token_counts = rag_embeddings._many(count, inputs)
if limit and max(token_counts) > limit:
if not _embeddings_truncate(body):
raise HTTPException(
@@ -38453,7 +38458,7 @@ async def _studio_embeddings(
logger.warning(
"Embedding input over the %d-token limit of %s truncated to fit", limit, label
)
- token_counts = [count(text) for text in inputs]
+ token_counts = rag_embeddings._many(count, inputs)
if dimensions is not None and dimensions != rag_embeddings.dim(model_name):
raise HTTPException(
status_code = 400, detail = f"'dimensions' is not supported by {label}."
diff --git a/studio/backend/tests/test_openai_embeddings_studio_fallback.py b/studio/backend/tests/test_openai_embeddings_studio_fallback.py
index 1d1e04feb4f..9b080e6d982 100644
--- a/studio/backend/tests/test_openai_embeddings_studio_fallback.py
+++ b/studio/backend/tests/test_openai_embeddings_studio_fallback.py
@@ -4,6 +4,7 @@
import asyncio
import base64
import copy
+import json
import os
import sys
import threading
@@ -1621,3 +1622,39 @@ def test_resident_embedding_gguf_cancel_during_the_retry_is_499(studio_embedder)
with pytest.raises(HTTPException) as exc:
asyncio.run(inference_route.openai_embeddings(request, "tester"))
assert exc.value.status_code == 499
+
+
+def test_batch_counts_tokens_in_one_call_with_the_same_usage(studio_embedder):
+ calls = {"single": 0, "many": 0}
+
+ def _counter(model_name = None):
+ def _count(text):
+ calls["single"] += 1
+ return len(text)
+
+ def _many(texts):
+ calls["many"] += 1
+ return [len(text) for text in texts]
+
+ _count.many = _many
+ return _count
+
+ studio_embedder.setattr(rag_embeddings, "token_counter", _counter)
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ texts = [f"text {i} " * (i % 7 + 1) for i in range(64)]
+ payload = _call({"input": texts})
+ assert calls == {"single": 0, "many": 1}
+ assert payload["usage"]["prompt_tokens"] == sum(len(text) for text in texts)
+ assert [row["embedding"] for row in payload["data"]] == [[1.0, 0.0]] * 64
+
+
+def test_float_rows_serialize_like_the_element_loop():
+ rows = np.random.default_rng(0).standard_normal((4, 33)).astype(np.float32)
+ rows[0, :3] = [np.nan, np.inf, -0.0]
+ for row in rows:
+ fast = inference_route._embedding_payload(row, "float")
+ assert json.dumps(fast) == json.dumps([float(x) for x in row])
+ assert all(type(x) is float for x in fast)
+ assert inference_route._embedding_payload([0.5, 1], "float") == [0.5, 1.0]