Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
EmbeddingGemma 2 support and embedding improvements (#12865)
Daniel Han9d901362ea5c
diff --git a/tests/python/test_sentence_transformer_config_kwargs.py b/tests/python/test_sentence_transformer_config_kwargs.pynew file mode 100644index 00000000000..d910400a458--- /dev/null+++ b/tests/python/test_sentence_transformer_config_kwargs.py@@ -0,0 +1,167 @@+"""FastSentenceTransformer: config_kwargs reach the config on the training path, and the+FORCE_FLOAT32 family check used by for_inference on GPUs without bfloat16, and float16 pooling."""++import json++import pytest+from transformers import AutoConfig, BertConfig, LlamaConfig, Qwen3MoeConfig++import unsloth # noqa: F401+from unsloth.models import sentence_transformer as st_mod+from unsloth.models.sentence_transformer import FastSentenceTransformer, _is_force_float32_config+++class _Captured(Exception):+ pass+++def _tiny_decoder(tmp_path):+ config = LlamaConfig(+ vocab_size = 128,+ hidden_size = 32,+ intermediate_size = 64,+ num_hidden_layers = 4,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ )+ config.save_pretrained(tmp_path)+ return str(tmp_path)+++def test_config_kwargs_go_to_the_config_not_the_model(tmp_path, monkeypatch):+ path = _tiny_decoder(tmp_path)+ captured = {}++ def fake_from_pretrained(*args, **kwargs):+ captured.update(kwargs)+ raise _Captured++ monkeypatch.setattr(st_mod.FastModel, "from_pretrained", fake_from_pretrained)+ with pytest.raises(_Captured):+ FastSentenceTransformer.from_pretrained(+ path, config_kwargs = {"num_hidden_layers": 1}, local_files_only = True+ )+ assert "config_kwargs" not in captured+ assert captured["config"].num_hidden_layers == 1+++def test_explicit_config_wins_over_config_kwargs(tmp_path, monkeypatch):+ path = _tiny_decoder(tmp_path)+ explicit = AutoConfig.from_pretrained(path)+ captured = {}++ def fake_from_pretrained(*args, **kwargs):+ captured.update(kwargs)+ raise _Captured++ monkeypatch.setattr(st_mod.FastModel, "from_pretrained", fake_from_pretrained)+ with pytest.raises(_Captured):+ FastSentenceTransformer.from_pretrained(+ path,+ config = explicit,+ config_kwargs = {"num_hidden_layers": 1},+ local_files_only = True,+ )+ assert captured["config"] is explicit+++def test_without_config_kwargs_nothing_is_injected(tmp_path, monkeypatch):+ path = _tiny_decoder(tmp_path)+ captured = {}++ def fake_from_pretrained(*args, **kwargs):+ captured.update(kwargs)+ raise _Captured++ monkeypatch.setattr(st_mod.FastModel, "from_pretrained", fake_from_pretrained)+ with pytest.raises(_Captured):+ FastSentenceTransformer.from_pretrained(path, local_files_only = True)+ assert "config" not in captured+ assert "config_kwargs" not in captured+++def test_hub_keys_inside_config_kwargs_do_not_collide(tmp_path, monkeypatch):+ path = _tiny_decoder(tmp_path)+ captured = {}++ def fake_from_pretrained(*args, **kwargs):+ captured.update(kwargs)+ raise _Captured++ monkeypatch.setattr(st_mod.FastModel, "from_pretrained", fake_from_pretrained)+ with pytest.raises(_Captured):+ FastSentenceTransformer.from_pretrained(+ path,+ config_kwargs = {"trust_remote_code": False, "token": None, "num_hidden_layers": 2},+ local_files_only = True,+ )+ assert captured["config"].num_hidden_layers == 2+++def test_force_float32_family_entries_are_normalised(monkeypatch):+ # An entry spelled with "-" or "_" must still match the normalised model type.+ import unsloth.models.loader as loader_mod+ from transformers import PretrainedConfig++ config = PretrainedConfig()+ config.model_type = "foobar"+ monkeypatch.setattr(loader_mod, "FORCE_FLOAT32", ["foo-bar"])+ assert _is_force_float32_config(config)+++def test_force_float32_family_check():+ assert _is_force_float32_config(Qwen3MoeConfig())+ assert not _is_force_float32_config(BertConfig())+ assert not _is_force_float32_config(None)+++def test_float16_mean_pooling_does_not_overflow():+ import torch+ from sentence_transformers.models import Pooling++ pooling = Pooling(word_embedding_dimension = 8, pooling_mode = "mean")+ # 2000 tokens at 4000: the float16 sum is 8e6, far past float16's 65504.+ tokens = torch.full((1, 2000, 8), 4000.0, dtype = torch.float16)+ features = {"token_embeddings": tokens, "attention_mask": torch.ones(1, 2000, dtype = torch.long)}+ out = pooling(features)+ assert torch.isfinite(out["sentence_embedding"]).all()+ assert torch.allclose(out["sentence_embedding"].float(), torch.full((1, 8), 4000.0))+ # The caller's token embeddings are handed back untouched.+ assert out["token_embeddings"] is tokens+++def test_float16_pooling_with_batch_encoding_features():+ # SentenceTransformer.encode passes a BatchEncoding (UserDict), not a dict.+ import torch+ from transformers import BatchEncoding+ from sentence_transformers.models import Pooling++ pooling = Pooling(word_embedding_dimension = 8, pooling_mode = "mean")+ tokens = torch.full((1, 2000, 8), 4000.0, dtype = torch.float16)+ features = BatchEncoding(+ {"token_embeddings": tokens, "attention_mask": torch.ones(1, 2000, dtype = torch.long)}+ )+ out = pooling(features)+ assert torch.isfinite(out["sentence_embedding"]).all()+++def test_bfloat16_pooling_is_unchanged():+ import torch+ from sentence_transformers.models import Pooling++ pooling = Pooling(word_embedding_dimension = 8, pooling_mode = "mean")+ tokens = torch.randn(2, 5, 8, dtype = torch.bfloat16)+ out = pooling(+ {"token_embeddings": tokens, "attention_mask": torch.ones(2, 5, dtype = torch.long)}+ )+ assert out["sentence_embedding"].dtype == torch.bfloat16+++def test_embedding_gemma2_text_only_is_force_float32():+ try:+ from transformers import EmbeddingGemma2Config+ except ImportError:+ pytest.skip("transformers has no EmbeddingGemma2")+ # Dropping the towers removes the gemma4 sub-configs that used to match by accident.+ config = EmbeddingGemma2Config(vision_config = None, audio_config = None)+ assert _is_force_float32_config(config)diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 2cd6f8df97e..8a4c01fc8ec 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -133,6 +133,7 @@ from ._utils import (# Source of truth is unsloth_zoo.model_lists, re-exported for callers importing FORCE_FLOAT32 from here. The fallback list is unioned in so a newer unsloth still forces float32 for these archs against an older zoo._FORCE_FLOAT32_FALLBACK = [+ "embedding_gemma2", # EmbeddingGemma 2: text-only loads have no gemma4 sub-config to match"gemma3,","gemma3text", # Gemma3TextModel (EmbeddingGemma, standalone text-only Gemma3)"gemma3n",diff --git a/unsloth/models/sentence_transformer.py b/unsloth/models/sentence_transformer.pyindex 7d94728d82a..6d37cc32b17 100644--- a/unsloth/models/sentence_transformer.py+++ b/unsloth/models/sentence_transformer.py@@ -100,6 +100,59 @@ def _ensure_sentence_attention_masks(model):return False+def _is_force_float32_config(config):+ """Mirror the loader's FORCE_FLOAT32 match (exact normalised type or substring of the joined types), for paths that skip FastModel."""+ if config is None:+ return False+ from .loader import FORCE_FLOAT32++ try:+ from unsloth_zoo.compiler import get_transformers_model_type+ model_types = list(get_transformers_model_type(config))+ except Exception:+ model_types = [getattr(config, "model_type", "") or ""]+ model_types_all = ",".join(model_types).lower() + ","+ normalised = {t.lower().replace("-", "").replace("_", "") for t in model_types}+ return any(+ name.lower().replace("-", "").replace("_", "") in normalised+ or name.lower() in model_types_all+ for name in FORCE_FLOAT32+ )+++def _maybe_upcast_force_float32_inference(st_model):+ inner = getattr(st_model[0], "auto_model", None)+ if inner is None or not _is_force_float32_config(getattr(inner, "config", None)):+ return False+ params = list(st_model.parameters())+ if SUPPORTS_BFLOAT16:+ if any(p.dtype == torch.float16 for p in params):+ print(f"Unsloth: {inner.config.model_type} does not support float16. Using bfloat16.")+ st_model.to(torch.bfloat16)+ return True+ return False+ low = sum(p.numel() for p in params if p.dtype in (torch.bfloat16, torch.float16))+ if low == 0:+ return False+ device = params[0].device+ if device.type == "cuda":+ free, _ = torch.cuda.mem_get_info(device)+ # Each 16-bit weight grows by 2 bytes; keep half the free memory for activations.+ if low * 2 > 0.5 * free:+ print(+ f"Unsloth: {inner.config.model_type} does not support float16 and this GPU has no "+ "bfloat16, but float32 weights do not fit. Keeping bfloat16 (emulated, slower). "+ "Pass dtype = torch.float32 to force it."+ )+ return False+ print(+ f"Unsloth: {inner.config.model_type} does not support float16 and this GPU has no "+ "bfloat16. Using float32."+ )+ st_model.to(torch.float32)+ return True++def _normalize_save_method(save_method):"""Fold "MERGED_16BIT" and "merged 16bit" onto "merged_16bit". unsloth_save_model (save.py) normalizes case and spaces before validating, so the same spelling has to mean the same thing here, else a keyword call that worked before starts raising."""if isinstance(save_method, str):@@ -2351,6 +2404,9 @@ class FastSentenceTransformer(FastModel):if _validated:st_kwargs["revision"] = _validatedst_model = SentenceTransformer(model_name, **st_kwargs)+ # "auto" keeps a bfloat16 checkpoint in bfloat16 on GPUs without it (T4, V100): emulated and slow. An explicit float16 request is what these families cannot run at all (non-finite activations). Either way move to bfloat16 where supported, else float32 when the copy fits.+ if (dtype is None and not SUPPORTS_BFLOAT16) or dtype == torch.float16:+ _maybe_upcast_force_float32_inference(st_model)if _ensure_sentence_attention_masks(getattr(st_model[0], "auto_model", None)) and hasattr(st_model[0], "unpad_inputs"):@@ -2363,15 +2419,32 @@ class FastSentenceTransformer(FastModel):kwargs["auto_model"] = AutoModeltransformers4 = Version(transformers.__version__).major < 5+ # SentenceTransformer's config_kwargs (e.g. {"vision_config": None} to skip a multimodal model's vision tower) must reach the config, not the model constructor.+ config_kwargs = kwargs.pop("config_kwargs", None) or {}model_type = ""- config = None- try:- config = AutoConfig.from_pretrained(- model_name, token = token, trust_remote_code = trust_remote_code- )+ config = kwargs.get("config", None)+ if config is not None:model_type = getattr(config, "model_type", "")- except:- pass+ else:+ # One dict, caller's config_kwargs last, so {"trust_remote_code": True} overrides instead of duplicating a keyword (as SentenceTransformer merges them). Same revision and cache as the weight load, since this config builds the model when config_kwargs is set.+ _config_load_kwargs = {+ "token": token,+ "trust_remote_code": trust_remote_code,+ "revision": revision,+ "cache_dir": kwargs.get("cache_dir")+ or kwargs.get("cache_folder")+ or os.environ.get("SENTENCE_TRANSFORMERS_HOME"),+ "local_files_only": kwargs.get("local_files_only", False),+ **config_kwargs,+ }+ try:+ config = AutoConfig.from_pretrained(model_name, **_config_load_kwargs)+ model_type = getattr(config, "model_type", "")+ except:+ if config_kwargs:+ raise+ if config_kwargs and config is not None:+ kwargs["config"] = config# Fast encoder path: native torch.compile for encoder models (6x speedup), bypassing Unsloth's auto-compiler, whose @torch.compiler.disable decorators error for encoders on torch 2.9+. Set UNSLOTH_COMPILE_DISABLE=1 for the old path.is_encoder_model = model_type.lower() in FastSentenceTransformer.ENCODER_MODEL_TYPES@@ -2462,6 +2535,7 @@ class FastSentenceTransformer(FastModel):revision = _validated or revision,model_kwargs = model_kwargs,cache_folder = kwargs.get("cache_dir") or kwargs.get("cache_folder"),+ **({"config_kwargs": config_kwargs} if config_kwargs else {}),)st_model._unsloth_fast_encoder = True@@ -2983,6 +3057,14 @@ class FastSentenceTransformer(FastModel):transformer_module = model[0]inner_model = transformer_module.auto_model+ # Multimodal embedding models carry vision / audio towers that text batches never run: an explicit leaf list would also adapt them, adding dead adapters and DDP unused-parameter errors. Default to the text model; gated on the towers so text-only models keep their old routing.+ if any(+ getattr(inner_model, name, None) is not None+ for name in ("vision_tower", "audio_tower")+ ):+ kwargs.setdefault("finetune_vision_layers", False)+ kwargs.setdefault("finetune_audio_layers", False)+peft_model = FastModel.get_peft_model(model = inner_model,r = r,@@ -3243,5 +3325,40 @@ def _patch_st_trainer_load_from_checkpoint():SentenceTransformerTrainer._unsloth_load_from_checkpoint_patched = True+def _patch_pooling_float16_accumulation():+ """Pool float16 token embeddings in float32.++ Pooling sums token embeddings in their own dtype, so float16 (T4 / V100, or Unsloth's+ float32-forced mode, which keeps float16 weights) overflows to inf once a long input's+ sum passes 65504, and Normalize turns that into NaN. Models whose token embeddings reach a+ few thousand hit this on inputs of a few thousand tokens. bfloat16 / float32 are left alone;+ the pooled vector stays float32 so Normalize and the loss don't overflow either."""+ try:+ from sentence_transformers.models import Pooling+ except Exception:+ return+ if getattr(Pooling, "_unsloth_float16_pooling", False):+ return+ _original_forward = Pooling.forward++ def forward(self, features, *args, **kwargs):+ # encode() hands over a BatchEncoding (a UserDict, not a dict), so test for the mapping API.+ token_embeddings = (+ features.get("token_embeddings", None) if hasattr(features, "get") else None+ )+ if not (torch.is_tensor(token_embeddings) and token_embeddings.dtype == torch.float16):+ return _original_forward(self, features, *args, **kwargs)+ features["token_embeddings"] = token_embeddings.float()+ try:+ return _original_forward(self, features, *args, **kwargs)+ finally:+ features["token_embeddings"] = token_embeddings++ forward.__wrapped__ = _original_forward+ Pooling.forward = forward+ Pooling._unsloth_float16_pooling = True++_patch_sentence_transformer_trainer()_patch_st_trainer_load_from_checkpoint()+_patch_pooling_float16_accumulation()