Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio embeddings: parse the GGUF body only when read and once, and serialize the Studio embedder payload off the event loop (#17286)
Daniel Han947973bd86bc
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 38f3d217d1a..0534ec317d6 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -38817,23 +38817,35 @@ async def _studio_embeddings(raise HTTPException(status_code = 502, detail = f"Embedding model failed: {_friendly_error(exc)}") from exc- payload = {- "object": "list",- "data": [- {- "object": "embedding",- "index": index,- "embedding": _embedding_payload(vector, encoding_format),- }- for index, vector in enumerate(vectors)- ],- "model": _public_embedding_identity(identity, model_name, label),- "usage": {"prompt_tokens": prompt_tokens, "total_tokens": prompt_tokens},- }++ def _serialize():+ payload = {+ "object": "list",+ "data": [+ {+ "object": "embedding",+ "index": index,+ "embedding": _embedding_payload(vector, encoding_format),+ }+ for index, vector in enumerate(vectors)+ ],+ "model": _public_embedding_identity(identity, model_name, label),+ "usage": {"prompt_tokens": prompt_tokens, "total_tokens": prompt_tokens},+ }+ # json.dumps(payload) holds the GIL for the whole body; per row, the loop gets it back between rows.+ rows = ", ".join([json.dumps(row) for row in payload["data"]])+ content = json.dumps({**payload, "data": []}).replace('"data": []', f'"data": [{rows}]', 1)+ return payload, content++ try:+ payload, content = await asyncio.to_thread(_serialize)+ except asyncio.CancelledError:+ api_monitor.finish(monitor_id, "cancelled")+ raise# The gauge only makes sense for a single input; prompt_tokens sums the batch._monitor_openai_chunk(monitor_id, payload, limit if len(texts) == 1 else None)api_monitor.finish(monitor_id)- return Response(content = json.dumps(payload), media_type = "application/json")+ return Response(content = content, media_type = "application/json")# llama-server's refusals past one micro-batch (500) or the slot context (400), server-context.cpp;@@ -38904,9 +38916,41 @@ async def _truncated_llama_embedding_inputs(return out if cut else None-def _embedding_width(resp) -> Optional[int]:+def _embedding_response_json(resp):+ """The parsed body, or the exception parsing raised (each caller swallows it)."""try:- embedding = resp.json()["data"][0]["embedding"]+ return resp.json()+ except Exception as exc: # noqa: BLE001+ return exc+++def _embedding_tail_usage(content: bytes) -> Optional[dict]:+ """``usage`` when it is the body's last top-level key, else None.++ llama-server's json keeps keys sorted, so usage follows the data rows (format_embeddings_response_oaicompat).+ """+ start = content.rfind(b'"usage"')+ before = start - 1+ while before >= 0 and content[before] in b" \t\r\n":+ before -= 1+ if before < 0 or content[before] not in b",{":+ return None+ try:+ tail = content[start + 7 :].decode().lstrip()+ if not tail.startswith(":"):+ return None+ tail = tail[1:].lstrip()+ usage, end = json.JSONDecoder().raw_decode(tail)+ except ValueError:+ return None+ return usage if isinstance(usage, dict) and tail[end:].strip() == "}" else None+++def _embedding_width(data) -> Optional[int]:+ try:+ if isinstance(data, Exception):+ raise data+ embedding = data["data"][0]["embedding"]if isinstance(embedding, str):return len(base64.b64decode(embedding)) // 4return len(embedding)@@ -39039,17 +39083,29 @@ async def openai_embeddings(request: Request, current_subject: str = Depends(getlogger.warning("Embedding input over the %d-token limit truncated to fit", limit)body["input"] = inputsresp = await upstream.post(body, timeout = _DEFAULT_FIRST_TOKEN_TIMEOUT_S)- if resp.status_code == 200 and dimensions is not None and dimensions != _embedding_width(resp):+ # A full parse holds the GIL even in a thread; the monitor alone needs only the tail's usage.+ data = None+ if resp.status_code == 200 and monitor_id and dimensions is None:+ usage = _embedding_tail_usage(resp.content)+ data = None if usage is None else {"usage": usage}+ if resp.status_code == 200 and data is None and (dimensions is not None or monitor_id):+ try:+ data = await asyncio.to_thread(_embedding_response_json, resp)+ except asyncio.CancelledError:+ api_monitor.finish(monitor_id, "cancelled")+ raise+ if resp.status_code == 200 and dimensions is not None and dimensions != _embedding_width(data):detail = f"'dimensions' is not supported by {_llama_public_model_id(llama_backend)}."api_monitor.fail(monitor_id, detail)raise HTTPException(status_code = 400, detail = detail)if resp.status_code != 200:api_monitor.fail(monitor_id, resp.text[:500])else:- try:- _monitor_openai_chunk(monitor_id, resp.json(), _monitor_context_length())- except Exception:- pass+ if monitor_id and not isinstance(data, Exception):+ try:+ _monitor_openai_chunk(monitor_id, data, _monitor_context_length())+ except Exception:+ passapi_monitor.finish(monitor_id)return Response(content = resp.content,diff --git a/studio/backend/tests/test_openai_embeddings_studio_fallback.py b/studio/backend/tests/test_openai_embeddings_studio_fallback.pyindex 5cbdb874b73..5d7c8d555f0 100644--- a/studio/backend/tests/test_openai_embeddings_studio_fallback.py+++ b/studio/backend/tests/test_openai_embeddings_studio_fallback.py@@ -1684,3 +1684,154 @@ def test_float_rows_serialize_like_the_element_loop():assert json.dumps(fast) == json.dumps([float(x) for x in row])assert all(type(x) is float for x in fast)assert inference_route._embedding_payload([0.5, 1], "float") == [0.5, 1.0]+++class _Monitor:+ def __init__(self):+ self.calls = []++ def start(self, **_kwargs):+ return "mid"++ def fail(self, monitor_id, detail):+ self.calls.append(("fail", detail))++ def finish(self, monitor_id, *args):+ self.calls.append(("finish", *args))+++def _counting_gguf_proxy(+ studio_embedder,+ body,+ status = 200,+):+ import httpx++ parses = []++ class _Response(httpx.Response):+ def json(self, **kwargs):+ parses.append(threading.get_ident())+ return super().json(**kwargs)++ class _Client:+ async def post(self, *_args, **_kwargs):+ return _Response(status, content = body)++ async def aclose(self):+ return None++ _resident_embedding_gguf(studio_embedder, _Client)+ return parses+++def _upstream_body(dim = 4):+ rows = [{"object": "embedding", "index": 0, "embedding": [0.25] * dim}]+ usage = {"prompt_tokens": 3, "total_tokens": 3}+ return json.dumps({"object": "list", "data": rows, "usage": usage}).encode()+++def test_gguf_proxy_skips_the_parse_when_nothing_reads_it(studio_embedder):+ body = _upstream_body()+ parses = _counting_gguf_proxy(studio_embedder, body)+ response = asyncio.run(+ inference_route.openai_embeddings(_Request({"input": "alpha"}), "tester")+ )+ assert response.body == body+ assert parses == []++[email protected]("upstream", [_upstream_body(), b"{not json"], ids = ["json", "invalid"])+def test_gguf_proxy_parses_once_off_the_loop_for_monitor_and_dimensions(studio_embedder, upstream):+ parses = _counting_gguf_proxy(studio_embedder, upstream)+ monitor, seen = _Monitor(), []+ studio_embedder.setattr(inference_route, "api_monitor", monitor)+ studio_embedder.setattr(inference_route, "_request_used_api_key", lambda request: False)+ studio_embedder.setattr(+ inference_route,+ "_monitor_openai_chunk",+ lambda monitor_id, data, context_length = None, **_: seen.append(data),+ )+ request = _Request({"input": "alpha", "dimensions": 4})+ request.state.skip_api_monitor = False+ if upstream.startswith(b"{not"):+ assert _http_error_for(request).status_code == 400+ assert seen == [] and monitor.calls[0][0] == "fail"+ else:+ response = asyncio.run(inference_route.openai_embeddings(request, "tester"))+ assert response.body == upstream+ assert seen == [json.loads(upstream)]+ assert monitor.calls == [("finish",)]+ assert len(parses) == 1 and parses[0] != threading.get_ident()+++def _http_error_for(request):+ with pytest.raises(HTTPException) as exc:+ asyncio.run(inference_route.openai_embeddings(request, "tester"))+ return exc.value++[email protected]("encoding_format", ["float", "base64"])+def test_studio_payload_is_serialized_off_the_loop_with_the_same_bytes(+ studio_embedder, encoding_format+):+ rows = np.random.default_rng(0).standard_normal((3, 2)).astype(np.float32)+ rows[0, 0] = np.nan+ threads = []+ real_payload = inference_route._embedding_payload++ def _payload(vector, fmt):+ threads.append(threading.get_ident())+ return real_payload(vector, fmt)++ studio_embedder.setattr(inference_route, "_embedding_payload", _payload)+ studio_embedder.setattr(rag_embeddings, "encode", lambda texts, **_: rows[: len(texts)])+ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ body = {"input": ["a", "bb", "ccc"], "encoding_format": encoding_format}+ response = asyncio.run(inference_route.openai_embeddings(_Request(body), "tester"))+ expected = {+ "object": "list",+ "data": [+ {"object": "embedding", "index": i, "embedding": real_payload(row, encoding_format)}+ for i, row in enumerate(rows)+ ],+ "model": IDENTITY,+ "usage": {"prompt_tokens": 6, "total_tokens": 6},+ }+ assert response.body == json.dumps(expected).encode()+ assert threads and threading.get_ident() not in threads+++_USAGE = {"prompt_tokens": 3, "total_tokens": 3}+_ROWS = [{"embedding": [0.25] * 4, "index": 0, "object": "embedding"}]+++ "upstream, full_parses",+ [+ (json.dumps({"data": _ROWS, "model": "m", "object": "list", "usage": _USAGE}), 0),+ (json.dumps({"data": _ROWS, "model": '"usage": {}', "usage": _USAGE}, indent = 2), 0),+ (json.dumps({"usage": _USAGE, "data": _ROWS}), 1),+ (json.dumps({"data": _ROWS, "x": {"usage": _USAGE}}), 1),+ (json.dumps({"data": _ROWS}), 1),+ ],+)+def test_gguf_proxy_monitor_reads_usage_from_the_tail(studio_embedder, upstream, full_parses):+ upstream = upstream.encode()+ parses = _counting_gguf_proxy(studio_embedder, upstream)+ monitor, seen = _Monitor(), []+ studio_embedder.setattr(inference_route, "api_monitor", monitor)+ studio_embedder.setattr(inference_route, "_request_used_api_key", lambda request: False)+ studio_embedder.setattr(+ inference_route,+ "_monitor_openai_chunk",+ lambda monitor_id, data, context_length = None, **_: seen.append(data.get("usage")),+ )+ request = _Request({"input": "alpha"})+ request.state.skip_api_monitor = False+ response = asyncio.run(inference_route.openai_embeddings(request, "tester"))+ assert response.body == upstream+ assert seen == [json.loads(upstream).get("usage")]+ assert len(parses) == full_parses and monitor.calls == [("finish",)]