Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop loading EmbeddingGemma in float16 in the RAG embedder (#12864)

oobabooga62ea2d7b52b6
diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.py
index 8148828ca8a..ba393ba407a 100644
--- a/studio/backend/core/rag/embeddings.py
+++ b/studio/backend/core/rag/embeddings.py
@@ -106,6 +106,28 @@ def _load_device() -> str:
)
+# Matched as a substring of the lowercased model name. EmbeddingGemma's activations overflow float16 and every
+# vector comes back NaN; its model card says to use float32 or bfloat16.
+_FLOAT16_UNSAFE_MODELS = ("embeddinggemma",)
+
+
+def _load_dtype(device: str, name: str) -> str:
+ """float32 on CPU, where fp16 BERT raises "not implemented for Half" and encode() answers by swapping the
+ whole process to llama-server. float16 on an accelerator, except for models that cannot run in it: those
+ get bf16 where the device has it natively, else float32."""
+ if device == "cpu":
+ return "float32"
+ if not any(m in name.lower() for m in _FLOAT16_UNSAFE_MODELS):
+ return "float16"
+ from core.training.diffusion_train_common import (
+ native_bf16_supported,
+ native_bf16_supported_xpu,
+ )
+
+ native = native_bf16_supported_xpu() if device == "xpu" else native_bf16_supported()
+ return "bfloat16" if native else "float32"
+
+
_torchao_stub_done = False
# Its own lock, not _lock: that one is held across a whole model construction, so borrowing it made
# a preflight probe wait out someone else's download.
@@ -507,7 +529,8 @@ def _gate_st_custom_modules() -> None:
def _get(model_name: str | None = None):
"""Cached SentenceTransformer, (re)loading on a name change. Loaded in fp16 on an
- accelerator for a ~1.5x speedup at negligible accuracy loss, fp32 on CPU."""
+ accelerator for a ~1.5x speedup at negligible accuracy loss, fp32 on CPU; see ``_load_dtype``
+ for the models that cannot run in fp16."""
account_path(model_name, reference = True)
global _model, _name
name = model_name or config.effective_embedding_model()
@@ -534,9 +557,7 @@ def _get(model_name: str | None = None):
st_kwargs = dict(
device = device,
cache_folder = active_hf_hub_cache(),
- # Keyed on the device we load on: fp16 BERT on CPU raises "not implemented for Half", which encode()
- # answers by swapping the whole process to llama-server.
- model_kwargs = dtype_kwargs("float32" if device == "cpu" else "float16"),
+ model_kwargs = dtype_kwargs(_load_dtype(device, name)),
)
if managed_account():
st_kwargs["token"] = False
diff --git a/studio/backend/core/rag/store.py b/studio/backend/core/rag/store.py
index 969ca2ed87f..43b9c7632a6 100644
--- a/studio/backend/core/rag/store.py
+++ b/studio/backend/core/rag/store.py
@@ -12,6 +12,7 @@ partition key.
from __future__ import annotations
import json
+import math
import os
import re
import sqlite3
@@ -874,7 +875,13 @@ def search_dense(
).fetchall()
kept[s] = _drop_incompatible(
conn,
- [(r["chunk_id"], 1.0 - r["distance"]) for r in rows],
+ # A NaN vector written by an fp16 embedder reads back as a NULL distance: skip it, or the
+ # subtraction raises and the whole retrieval fails.
+ [
+ (r["chunk_id"], 1.0 - r["distance"])
+ for r in rows
+ if r["distance"] is not None and math.isfinite(r["distance"])
+ ],
embedding_model,
untagged,
)
diff --git a/studio/backend/tests/test_rag_embeddings.py b/studio/backend/tests/test_rag_embeddings.py
index df784079983..a423a1e6843 100644
--- a/studio/backend/tests/test_rag_embeddings.py
+++ b/studio/backend/tests/test_rag_embeddings.py
@@ -33,7 +33,11 @@ def _shared_setup_1(monkeypatch):
return observed
-def _shared_setup_2(monkeypatch, tmp_path):
+def _shared_setup_2(
+ monkeypatch,
+ tmp_path,
+ name = "Org/Embedder",
+):
monkeypatch.setattr(
"utils.hf_cache_settings.active_hf_hub_cache",
lambda: str(tmp_path / "selected-hub"),
@@ -47,7 +51,7 @@ def _shared_setup_2(monkeypatch, tmp_path):
monkeypatch.setattr(embeddings, "_model", None)
monkeypatch.setattr(embeddings, "_name", None)
- embeddings._get("Org/Embedder")
+ embeddings._get(name)
def _shared_setup_3(monkeypatch):
@@ -531,6 +535,28 @@ def test_opted_in_accelerator_loads_float16(monkeypatch, tmp_path):
assert list(observed["model_kwargs"].values()) == ["float16"]
+ "device, native, expected",
+ [
+ ("cuda", True, "bfloat16"),
+ ("cuda", False, "float32"),
+ ("xpu", True, "bfloat16"),
+ ("xpu", False, "float32"),
+ ],
+)
+def test_float16_unsafe_model_avoids_float16(monkeypatch, tmp_path, device, native, expected):
+ import core.training.diffusion_train_common as dtc
+
+ observed = _shared_setup_1(monkeypatch)
+ monkeypatch.setattr(embeddings, "_load_device", lambda: device)
+ monkeypatch.setattr(dtc, "native_bf16_supported", lambda: native)
+ monkeypatch.setattr(dtc, "native_bf16_supported_xpu", lambda: native)
+ _shared_setup_2(monkeypatch, tmp_path, name = "unsloth/EmbeddingGemma-300m")
+
+ assert observed["device"] == device
+ assert list(observed["model_kwargs"].values()) == [expected]
+
+
def test_accelerator_fallback_loads_float32_on_cpu(monkeypatch, tmp_path):
observed = _shared_setup_1(monkeypatch)
monkeypatch.setattr(embeddings, "_load_device", lambda: "cpu")
diff --git a/studio/backend/tests/test_rag_store.py b/studio/backend/tests/test_rag_store.py
index c4862712c0b..1b02322aa70 100644
--- a/studio/backend/tests/test_rag_store.py
+++ b/studio/backend/tests/test_rag_store.py
@@ -75,6 +75,15 @@ def test_dense_ranks_by_cosine(rag_conn):
assert ranked[0][0] == "d1:0" and ranked[0][1] > 0.99
+def test_dense_skips_nan_vectors_instead_of_raising(rag_conn):
+ # A float16 embedder wrote NaN vectors, which vec0 answers with a NULL distance.
+ _add_doc(rag_conn, "kb_a", "d1", "f", "h1", ["alpha alpha"])
+ store.create_document(rag_conn, scope = "kb_a", filename = "g", sha256 = "h2", document_id = "d2")
+ store.add_chunks(rag_conn, "kb_a", "d2", [_chunk("alpha")], [[math.nan] * len(VOCAB)])
+ ranked = store.search_dense(rag_conn, "kb_a", embed("alpha"), 10)
+ assert [cid for cid, _ in ranked] == ["d1:0"]
+
+
def test_dense_knn_binds_k_rather_than_limit(rag_conn):
"""vec0's KNN bound must arrive as ``k = ?``, not a bare ``LIMIT ?``, which SQLite forwards
to a virtual table's planner only from 3.41 on. CI's SQLite accepts both and returns the