Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: return an error when embeddings dimensions can't be honored (#11968)

Nilay7959a297f77e
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 10861871368..7d1b9984ee9 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -31680,6 +31680,16 @@ async def _studio_embeddings(
return Response(content = json.dumps(payload), media_type = "application/json")
+def _embedding_width(resp) -> Optional[int]:
+ try:
+ embedding = resp.json()["data"][0]["embedding"]
+ if isinstance(embedding, str):
+ return len(base64.b64decode(embedding)) // 4
+ return len(embedding)
+ except Exception: # noqa: BLE001
+ return None
+
+
def _embeddings_input_present(body: dict) -> bool:
"""Whether an embeddings body carries a usable ``input`` (non-empty)."""
inp = body.get("input")
@@ -31775,6 +31785,9 @@ async def openai_embeddings(request: Request, current_subject: str = Depends(get
# no-pooling error on /v1/embeddings against a non-embedding GGUF), so claiming before the
# upstream response would strand a preview-owned checkpoint as Unsloth-owned.
+ # llama-server ignores `dimensions`; checked against the returned width below.
+ body = dict(body)
+ dimensions = body.pop("dimensions", None)
target_url = f"{llama_backend.base_url}/v1/embeddings"
prompt_text = _flatten_monitor_prompt(body.get("input", ""))
monitor_id = None
@@ -31839,6 +31852,10 @@ async def openai_embeddings(request: Request, current_subject: str = Depends(get
finally:
_direct_llama_request_finished()
_tracker.__exit__(None, None, None)
+ if resp.status_code == 200 and dimensions is not None and dimensions != _embedding_width(resp):
+ detail = f"'dimensions' is not supported by {_llama_public_model_id(llama_backend)}."
+ api_monitor.fail(monitor_id, detail)
+ raise HTTPException(status_code = 400, detail = detail)
if resp.status_code != 200:
api_monitor.fail(monitor_id, resp.text[:500])
else:
diff --git a/studio/backend/tests/test_openai_embeddings_studio_fallback.py b/studio/backend/tests/test_openai_embeddings_studio_fallback.py
index a074c1c0d9e..19e9aa22479 100644
--- a/studio/backend/tests/test_openai_embeddings_studio_fallback.py
+++ b/studio/backend/tests/test_openai_embeddings_studio_fallback.py
@@ -144,6 +144,58 @@ def test_resident_embedding_gguf_still_uses_the_proxy(studio_embedder):
assert payload == {"data": [{"embedding": [0.5]}]}
[email protected]("encoding_format", ["float", "base64"])
[email protected]("dimensions, status", [(2, 200), (256, 400)])
+def test_resident_embedding_gguf_checks_dimensions_against_its_width(
+ studio_embedder, encoding_format, dimensions, status
+):
+ import httpx
+
+ forwarded = []
+
+ class _Client:
+ async def post(
+ self,
+ *_args,
+ json = None,
+ **_kwargs,
+ ):
+ forwarded.append(json)
+ vector = [0.5, 0.5]
+ if encoding_format == "base64":
+ vector = base64.b64encode(np.asarray(vector, dtype = np.float32).tobytes()).decode()
+ return httpx.Response(200, json = {"data": [{"embedding": vector}]})
+
+ async def aclose(self):
+ return None
+
+ studio_embedder.setattr(inference_route, "_cancelable_nonstreaming_client", _Client)
+ studio_embedder.setattr(
+ inference_route,
+ "get_llama_cpp_backend",
+ lambda: SimpleNamespace(
+ is_loaded = True,
+ is_embedding_gguf = True,
+ base_url = "http://llama.test",
+ context_length = 512,
+ model_identifier = "org/E-GGUF",
+ ),
+ )
+ body = {
+ "input": "alpha",
+ "model": "org/E-GGUF",
+ "encoding_format": encoding_format,
+ "dimensions": dimensions,
+ }
+ if status == 200:
+ _call(body)
+ else:
+ error = _http_error(body)
+ assert error.status_code == 400
+ assert "'dimensions' is not supported" in error.detail
+ assert "dimensions" not in forwarded[0]
+
+
def test_base64_encoding_format(studio_embedder):
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)