Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop loading EmbeddingGemma in float16 in the RAG embedder (#12864)
oobabooga62ea2d7b52b6
diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.pyindex 8148828ca8a..ba393ba407a 100644--- a/studio/backend/core/rag/embeddings.py+++ b/studio/backend/core/rag/embeddings.py@@ -106,6 +106,28 @@ def _load_device() -> str:)+# Matched as a substring of the lowercased model name. EmbeddingGemma's activations overflow float16 and every+# vector comes back NaN; its model card says to use float32 or bfloat16.+_FLOAT16_UNSAFE_MODELS = ("embeddinggemma",)+++def _load_dtype(device: str, name: str) -> str:+ """float32 on CPU, where fp16 BERT raises "not implemented for Half" and encode() answers by swapping the+ whole process to llama-server. float16 on an accelerator, except for models that cannot run in it: those+ get bf16 where the device has it natively, else float32."""+ if device == "cpu":+ return "float32"+ if not any(m in name.lower() for m in _FLOAT16_UNSAFE_MODELS):+ return "float16"+ from core.training.diffusion_train_common import (+ native_bf16_supported,+ native_bf16_supported_xpu,+ )++ native = native_bf16_supported_xpu() if device == "xpu" else native_bf16_supported()+ return "bfloat16" if native else "float32"++_torchao_stub_done = False# Its own lock, not _lock: that one is held across a whole model construction, so borrowing it made# a preflight probe wait out someone else's download.@@ -507,7 +529,8 @@ def _gate_st_custom_modules() -> None:def _get(model_name: str | None = None):"""Cached SentenceTransformer, (re)loading on a name change. Loaded in fp16 on an- accelerator for a ~1.5x speedup at negligible accuracy loss, fp32 on CPU."""+ accelerator for a ~1.5x speedup at negligible accuracy loss, fp32 on CPU; see ``_load_dtype``+ for the models that cannot run in fp16."""account_path(model_name, reference = True)global _model, _namename = model_name or config.effective_embedding_model()@@ -534,9 +557,7 @@ def _get(model_name: str | None = None):st_kwargs = dict(device = device,cache_folder = active_hf_hub_cache(),- # Keyed on the device we load on: fp16 BERT on CPU raises "not implemented for Half", which encode()- # answers by swapping the whole process to llama-server.- model_kwargs = dtype_kwargs("float32" if device == "cpu" else "float16"),+ model_kwargs = dtype_kwargs(_load_dtype(device, name)),)if managed_account():st_kwargs["token"] = Falsediff --git a/studio/backend/core/rag/store.py b/studio/backend/core/rag/store.pyindex 969ca2ed87f..43b9c7632a6 100644--- a/studio/backend/core/rag/store.py+++ b/studio/backend/core/rag/store.py@@ -12,6 +12,7 @@ partition key.from __future__ import annotationsimport json+import mathimport osimport reimport sqlite3@@ -874,7 +875,13 @@ def search_dense().fetchall()kept[s] = _drop_incompatible(conn,- [(r["chunk_id"], 1.0 - r["distance"]) for r in rows],+ # A NaN vector written by an fp16 embedder reads back as a NULL distance: skip it, or the+ # subtraction raises and the whole retrieval fails.+ [+ (r["chunk_id"], 1.0 - r["distance"])+ for r in rows+ if r["distance"] is not None and math.isfinite(r["distance"])+ ],embedding_model,untagged,)diff --git a/studio/backend/tests/test_rag_embeddings.py b/studio/backend/tests/test_rag_embeddings.pyindex df784079983..a423a1e6843 100644--- a/studio/backend/tests/test_rag_embeddings.py+++ b/studio/backend/tests/test_rag_embeddings.py@@ -33,7 +33,11 @@ def _shared_setup_1(monkeypatch):return observed-def _shared_setup_2(monkeypatch, tmp_path):+def _shared_setup_2(+ monkeypatch,+ tmp_path,+ name = "Org/Embedder",+):monkeypatch.setattr("utils.hf_cache_settings.active_hf_hub_cache",lambda: str(tmp_path / "selected-hub"),@@ -47,7 +51,7 @@ def _shared_setup_2(monkeypatch, tmp_path):monkeypatch.setattr(embeddings, "_model", None)monkeypatch.setattr(embeddings, "_name", None)- embeddings._get("Org/Embedder")+ embeddings._get(name)def _shared_setup_3(monkeypatch):@@ -531,6 +535,28 @@ def test_opted_in_accelerator_loads_float16(monkeypatch, tmp_path):assert list(observed["model_kwargs"].values()) == ["float16"]+ "device, native, expected",+ [+ ("cuda", True, "bfloat16"),+ ("cuda", False, "float32"),+ ("xpu", True, "bfloat16"),+ ("xpu", False, "float32"),+ ],+)+def test_float16_unsafe_model_avoids_float16(monkeypatch, tmp_path, device, native, expected):+ import core.training.diffusion_train_common as dtc++ observed = _shared_setup_1(monkeypatch)+ monkeypatch.setattr(embeddings, "_load_device", lambda: device)+ monkeypatch.setattr(dtc, "native_bf16_supported", lambda: native)+ monkeypatch.setattr(dtc, "native_bf16_supported_xpu", lambda: native)+ _shared_setup_2(monkeypatch, tmp_path, name = "unsloth/EmbeddingGemma-300m")++ assert observed["device"] == device+ assert list(observed["model_kwargs"].values()) == [expected]++def test_accelerator_fallback_loads_float32_on_cpu(monkeypatch, tmp_path):observed = _shared_setup_1(monkeypatch)monkeypatch.setattr(embeddings, "_load_device", lambda: "cpu")diff --git a/studio/backend/tests/test_rag_store.py b/studio/backend/tests/test_rag_store.pyindex c4862712c0b..1b02322aa70 100644--- a/studio/backend/tests/test_rag_store.py+++ b/studio/backend/tests/test_rag_store.py@@ -75,6 +75,15 @@ def test_dense_ranks_by_cosine(rag_conn):assert ranked[0][0] == "d1:0" and ranked[0][1] > 0.99+def test_dense_skips_nan_vectors_instead_of_raising(rag_conn):+ # A float16 embedder wrote NaN vectors, which vec0 answers with a NULL distance.+ _add_doc(rag_conn, "kb_a", "d1", "f", "h1", ["alpha alpha"])+ store.create_document(rag_conn, scope = "kb_a", filename = "g", sha256 = "h2", document_id = "d2")+ store.add_chunks(rag_conn, "kb_a", "d2", [_chunk("alpha")], [[math.nan] * len(VOCAB)])+ ranked = store.search_dense(rag_conn, "kb_a", embed("alpha"), 10)+ assert [cid for cid, _ in ranked] == ["d1:0"]++def test_dense_knn_binds_k_rather_than_limit(rag_conn):"""vec0's KNN bound must arrive as ``k = ?``, not a bare ``LIMIT ?``, which SQLite forwardsto a virtual table's planner only from 3.41 on. CI's SQLite accepts both and returns the