Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio embeddings: parse the GGUF body only when read and once, and serialize the Studio embedder payload off the event loop (#17286)

Daniel Han947973bd86bc
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 38f3d217d1a..0534ec317d6 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -38817,23 +38817,35 @@ async def _studio_embeddings(
raise HTTPException(
status_code = 502, detail = f"Embedding model failed: {_friendly_error(exc)}"
) from exc
- payload = {
- "object": "list",
- "data": [
- {
- "object": "embedding",
- "index": index,
- "embedding": _embedding_payload(vector, encoding_format),
- }
- for index, vector in enumerate(vectors)
- ],
- "model": _public_embedding_identity(identity, model_name, label),
- "usage": {"prompt_tokens": prompt_tokens, "total_tokens": prompt_tokens},
- }
+
+ def _serialize():
+ payload = {
+ "object": "list",
+ "data": [
+ {
+ "object": "embedding",
+ "index": index,
+ "embedding": _embedding_payload(vector, encoding_format),
+ }
+ for index, vector in enumerate(vectors)
+ ],
+ "model": _public_embedding_identity(identity, model_name, label),
+ "usage": {"prompt_tokens": prompt_tokens, "total_tokens": prompt_tokens},
+ }
+ # json.dumps(payload) holds the GIL for the whole body; per row, the loop gets it back between rows.
+ rows = ", ".join([json.dumps(row) for row in payload["data"]])
+ content = json.dumps({**payload, "data": []}).replace('"data": []', f'"data": [{rows}]', 1)
+ return payload, content
+
+ try:
+ payload, content = await asyncio.to_thread(_serialize)
+ except asyncio.CancelledError:
+ api_monitor.finish(monitor_id, "cancelled")
+ raise
# The gauge only makes sense for a single input; prompt_tokens sums the batch.
_monitor_openai_chunk(monitor_id, payload, limit if len(texts) == 1 else None)
api_monitor.finish(monitor_id)
- return Response(content = json.dumps(payload), media_type = "application/json")
+ return Response(content = content, media_type = "application/json")
# llama-server's refusals past one micro-batch (500) or the slot context (400), server-context.cpp;
@@ -38904,9 +38916,41 @@ async def _truncated_llama_embedding_inputs(
return out if cut else None
-def _embedding_width(resp) -> Optional[int]:
+def _embedding_response_json(resp):
+ """The parsed body, or the exception parsing raised (each caller swallows it)."""
try:
- embedding = resp.json()["data"][0]["embedding"]
+ return resp.json()
+ except Exception as exc: # noqa: BLE001
+ return exc
+
+
+def _embedding_tail_usage(content: bytes) -> Optional[dict]:
+ """``usage`` when it is the body's last top-level key, else None.
+
+ llama-server's json keeps keys sorted, so usage follows the data rows (format_embeddings_response_oaicompat).
+ """
+ start = content.rfind(b'"usage"')
+ before = start - 1
+ while before >= 0 and content[before] in b" \t\r\n":
+ before -= 1
+ if before < 0 or content[before] not in b",{":
+ return None
+ try:
+ tail = content[start + 7 :].decode().lstrip()
+ if not tail.startswith(":"):
+ return None
+ tail = tail[1:].lstrip()
+ usage, end = json.JSONDecoder().raw_decode(tail)
+ except ValueError:
+ return None
+ return usage if isinstance(usage, dict) and tail[end:].strip() == "}" else None
+
+
+def _embedding_width(data) -> Optional[int]:
+ try:
+ if isinstance(data, Exception):
+ raise data
+ embedding = data["data"][0]["embedding"]
if isinstance(embedding, str):
return len(base64.b64decode(embedding)) // 4
return len(embedding)
@@ -39039,17 +39083,29 @@ async def openai_embeddings(request: Request, current_subject: str = Depends(get
logger.warning("Embedding input over the %d-token limit truncated to fit", limit)
body["input"] = inputs
resp = await upstream.post(body, timeout = _DEFAULT_FIRST_TOKEN_TIMEOUT_S)
- if resp.status_code == 200 and dimensions is not None and dimensions != _embedding_width(resp):
+ # A full parse holds the GIL even in a thread; the monitor alone needs only the tail's usage.
+ data = None
+ if resp.status_code == 200 and monitor_id and dimensions is None:
+ usage = _embedding_tail_usage(resp.content)
+ data = None if usage is None else {"usage": usage}
+ if resp.status_code == 200 and data is None and (dimensions is not None or monitor_id):
+ try:
+ data = await asyncio.to_thread(_embedding_response_json, resp)
+ except asyncio.CancelledError:
+ api_monitor.finish(monitor_id, "cancelled")
+ raise
+ if resp.status_code == 200 and dimensions is not None and dimensions != _embedding_width(data):
detail = f"'dimensions' is not supported by {_llama_public_model_id(llama_backend)}."
api_monitor.fail(monitor_id, detail)
raise HTTPException(status_code = 400, detail = detail)
if resp.status_code != 200:
api_monitor.fail(monitor_id, resp.text[:500])
else:
- try:
- _monitor_openai_chunk(monitor_id, resp.json(), _monitor_context_length())
- except Exception:
- pass
+ if monitor_id and not isinstance(data, Exception):
+ try:
+ _monitor_openai_chunk(monitor_id, data, _monitor_context_length())
+ except Exception:
+ pass
api_monitor.finish(monitor_id)
return Response(
content = resp.content,
diff --git a/studio/backend/tests/test_openai_embeddings_studio_fallback.py b/studio/backend/tests/test_openai_embeddings_studio_fallback.py
index 5cbdb874b73..5d7c8d555f0 100644
--- a/studio/backend/tests/test_openai_embeddings_studio_fallback.py
+++ b/studio/backend/tests/test_openai_embeddings_studio_fallback.py
@@ -1684,3 +1684,154 @@ def test_float_rows_serialize_like_the_element_loop():
assert json.dumps(fast) == json.dumps([float(x) for x in row])
assert all(type(x) is float for x in fast)
assert inference_route._embedding_payload([0.5, 1], "float") == [0.5, 1.0]
+
+
+class _Monitor:
+ def __init__(self):
+ self.calls = []
+
+ def start(self, **_kwargs):
+ return "mid"
+
+ def fail(self, monitor_id, detail):
+ self.calls.append(("fail", detail))
+
+ def finish(self, monitor_id, *args):
+ self.calls.append(("finish", *args))
+
+
+def _counting_gguf_proxy(
+ studio_embedder,
+ body,
+ status = 200,
+):
+ import httpx
+
+ parses = []
+
+ class _Response(httpx.Response):
+ def json(self, **kwargs):
+ parses.append(threading.get_ident())
+ return super().json(**kwargs)
+
+ class _Client:
+ async def post(self, *_args, **_kwargs):
+ return _Response(status, content = body)
+
+ async def aclose(self):
+ return None
+
+ _resident_embedding_gguf(studio_embedder, _Client)
+ return parses
+
+
+def _upstream_body(dim = 4):
+ rows = [{"object": "embedding", "index": 0, "embedding": [0.25] * dim}]
+ usage = {"prompt_tokens": 3, "total_tokens": 3}
+ return json.dumps({"object": "list", "data": rows, "usage": usage}).encode()
+
+
+def test_gguf_proxy_skips_the_parse_when_nothing_reads_it(studio_embedder):
+ body = _upstream_body()
+ parses = _counting_gguf_proxy(studio_embedder, body)
+ response = asyncio.run(
+ inference_route.openai_embeddings(_Request({"input": "alpha"}), "tester")
+ )
+ assert response.body == body
+ assert parses == []
+
+
[email protected]("upstream", [_upstream_body(), b"{not json"], ids = ["json", "invalid"])
+def test_gguf_proxy_parses_once_off_the_loop_for_monitor_and_dimensions(studio_embedder, upstream):
+ parses = _counting_gguf_proxy(studio_embedder, upstream)
+ monitor, seen = _Monitor(), []
+ studio_embedder.setattr(inference_route, "api_monitor", monitor)
+ studio_embedder.setattr(inference_route, "_request_used_api_key", lambda request: False)
+ studio_embedder.setattr(
+ inference_route,
+ "_monitor_openai_chunk",
+ lambda monitor_id, data, context_length = None, **_: seen.append(data),
+ )
+ request = _Request({"input": "alpha", "dimensions": 4})
+ request.state.skip_api_monitor = False
+ if upstream.startswith(b"{not"):
+ assert _http_error_for(request).status_code == 400
+ assert seen == [] and monitor.calls[0][0] == "fail"
+ else:
+ response = asyncio.run(inference_route.openai_embeddings(request, "tester"))
+ assert response.body == upstream
+ assert seen == [json.loads(upstream)]
+ assert monitor.calls == [("finish",)]
+ assert len(parses) == 1 and parses[0] != threading.get_ident()
+
+
+def _http_error_for(request):
+ with pytest.raises(HTTPException) as exc:
+ asyncio.run(inference_route.openai_embeddings(request, "tester"))
+ return exc.value
+
+
[email protected]("encoding_format", ["float", "base64"])
+def test_studio_payload_is_serialized_off_the_loop_with_the_same_bytes(
+ studio_embedder, encoding_format
+):
+ rows = np.random.default_rng(0).standard_normal((3, 2)).astype(np.float32)
+ rows[0, 0] = np.nan
+ threads = []
+ real_payload = inference_route._embedding_payload
+
+ def _payload(vector, fmt):
+ threads.append(threading.get_ident())
+ return real_payload(vector, fmt)
+
+ studio_embedder.setattr(inference_route, "_embedding_payload", _payload)
+ studio_embedder.setattr(rag_embeddings, "encode", lambda texts, **_: rows[: len(texts)])
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ body = {"input": ["a", "bb", "ccc"], "encoding_format": encoding_format}
+ response = asyncio.run(inference_route.openai_embeddings(_Request(body), "tester"))
+ expected = {
+ "object": "list",
+ "data": [
+ {"object": "embedding", "index": i, "embedding": real_payload(row, encoding_format)}
+ for i, row in enumerate(rows)
+ ],
+ "model": IDENTITY,
+ "usage": {"prompt_tokens": 6, "total_tokens": 6},
+ }
+ assert response.body == json.dumps(expected).encode()
+ assert threads and threading.get_ident() not in threads
+
+
+_USAGE = {"prompt_tokens": 3, "total_tokens": 3}
+_ROWS = [{"embedding": [0.25] * 4, "index": 0, "object": "embedding"}]
+
+
+ "upstream, full_parses",
+ [
+ (json.dumps({"data": _ROWS, "model": "m", "object": "list", "usage": _USAGE}), 0),
+ (json.dumps({"data": _ROWS, "model": '"usage": {}', "usage": _USAGE}, indent = 2), 0),
+ (json.dumps({"usage": _USAGE, "data": _ROWS}), 1),
+ (json.dumps({"data": _ROWS, "x": {"usage": _USAGE}}), 1),
+ (json.dumps({"data": _ROWS}), 1),
+ ],
+)
+def test_gguf_proxy_monitor_reads_usage_from_the_tail(studio_embedder, upstream, full_parses):
+ upstream = upstream.encode()
+ parses = _counting_gguf_proxy(studio_embedder, upstream)
+ monitor, seen = _Monitor(), []
+ studio_embedder.setattr(inference_route, "api_monitor", monitor)
+ studio_embedder.setattr(inference_route, "_request_used_api_key", lambda request: False)
+ studio_embedder.setattr(
+ inference_route,
+ "_monitor_openai_chunk",
+ lambda monitor_id, data, context_length = None, **_: seen.append(data.get("usage")),
+ )
+ request = _Request({"input": "alpha"})
+ request.state.skip_api_monitor = False
+ response = asyncio.run(inference_route.openai_embeddings(request, "tester"))
+ assert response.body == upstream
+ assert seen == [json.loads(upstream).get("usage")]
+ assert len(parses) == full_parses and monitor.calls == [("finish",)]