Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio /v1/embeddings: count tokens in one batch and serialize float rows with tolist (#17030)
Daniel Han5e3b9adf98dc
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex f66e7833f88..5577ab958a3 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -38273,6 +38273,11 @@ def _embedding_payload(vector, encoding_format: str):import base64import numpy as npreturn base64.b64encode(np.asarray(vector, dtype = np.float32).tobytes()).decode("ascii")+ # float16/32/64 numpy rows only: longdouble tolist() keeps numpy scalars.+ if getattr(vector, "ndim", None) == 1 and getattr(+ getattr(vector, "dtype", None), "char", None+ ) in ("e", "f", "d"):+ return vector.tolist()return [float(x) for x in vector]@@ -38442,7 +38447,7 @@ async def _studio_embeddings(limit = rag_embeddings.max_tokens(model_name)count = rag_embeddings.token_counter(model_name)inputs = texts- token_counts = [count(text) for text in inputs]+ token_counts = rag_embeddings._many(count, inputs)if limit and max(token_counts) > limit:if not _embeddings_truncate(body):raise HTTPException(@@ -38453,7 +38458,7 @@ async def _studio_embeddings(logger.warning("Embedding input over the %d-token limit of %s truncated to fit", limit, label)- token_counts = [count(text) for text in inputs]+ token_counts = rag_embeddings._many(count, inputs)if dimensions is not None and dimensions != rag_embeddings.dim(model_name):raise HTTPException(status_code = 400, detail = f"'dimensions' is not supported by {label}."diff --git a/studio/backend/tests/test_openai_embeddings_studio_fallback.py b/studio/backend/tests/test_openai_embeddings_studio_fallback.pyindex 1d1e04feb4f..9b080e6d982 100644--- a/studio/backend/tests/test_openai_embeddings_studio_fallback.py+++ b/studio/backend/tests/test_openai_embeddings_studio_fallback.py@@ -4,6 +4,7 @@import asyncioimport base64import copy+import jsonimport osimport sysimport threading@@ -1621,3 +1622,39 @@ def test_resident_embedding_gguf_cancel_during_the_retry_is_499(studio_embedder)with pytest.raises(HTTPException) as exc:asyncio.run(inference_route.openai_embeddings(request, "tester"))assert exc.value.status_code == 499+++def test_batch_counts_tokens_in_one_call_with_the_same_usage(studio_embedder):+ calls = {"single": 0, "many": 0}++ def _counter(model_name = None):+ def _count(text):+ calls["single"] += 1+ return len(text)++ def _many(texts):+ calls["many"] += 1+ return [len(text) for text in texts]++ _count.many = _many+ return _count++ studio_embedder.setattr(rag_embeddings, "token_counter", _counter)+ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ texts = [f"text {i} " * (i % 7 + 1) for i in range(64)]+ payload = _call({"input": texts})+ assert calls == {"single": 0, "many": 1}+ assert payload["usage"]["prompt_tokens"] == sum(len(text) for text in texts)+ assert [row["embedding"] for row in payload["data"]] == [[1.0, 0.0]] * 64+++def test_float_rows_serialize_like_the_element_loop():+ rows = np.random.default_rng(0).standard_normal((4, 33)).astype(np.float32)+ rows[0, :3] = [np.nan, np.inf, -0.0]+ for row in rows:+ fast = inference_route._embedding_payload(row, "float")+ assert json.dumps(fast) == json.dumps([float(x) for x in row])+ assert all(type(x) is float for x in fast)+ assert inference_route._embedding_payload([0.5, 1], "float") == [0.5, 1.0]