Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
SentenceTransformer: preserve masks for patched Gemma3 attention (#11866)
Etherl8d229136de44
diff --git a/tests/python/test_sentence_transformer_gemma_masks.py b/tests/python/test_sentence_transformer_gemma_masks.pynew file mode 100644index 00000000000..3ecc124ad37--- /dev/null+++ b/tests/python/test_sentence_transformer_gemma_masks.py@@ -0,0 +1,124 @@+# SPDX-License-Identifier: Apache-2.0+"""Gemma's patched SDPA attention must receive masks before ST can flatten inputs."""++import inspect++import pytest+import torch++from real_accelerator import has_real_accelerator++[email protected]("loader_backend", ["sdpa", "flash_attention_2"])+def test_patched_gemma_loader_preserves_sequence_isolation(tmp_path, loader_backend):+ if not has_real_accelerator() or not torch.cuda.is_available():+ pytest.skip("real Gemma loader requires CUDA")+ pytest.importorskip("sentence_transformers")+ from transformers.utils import is_flash_attn_2_available++ if loader_backend == "flash_attention_2" and not is_flash_attn_2_available():+ pytest.skip("regression exercises an installed FA2 loader before the SDPA safety fallback")+ from unsloth import FastSentenceTransformer+ from transformers import Gemma3TextConfig, Gemma3TextModel, PreTrainedTokenizerFast+ from sentence_transformers import SentenceTransformer+ from sentence_transformers.models import Pooling, Transformer+ from tokenizers import Tokenizer+ from tokenizers.models import WordLevel+ from tokenizers.pre_tokenizers import Whitespace++ config = Gemma3TextConfig(+ vocab_size = 64,+ hidden_size = 32,+ intermediate_size = 48,+ num_hidden_layers = 2,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ head_dim = 16,+ max_position_embeddings = 32,+ sliding_window = 4,+ layer_types = ["sliding_attention", "full_attention"],+ use_bidirectional_attention = True,+ use_cache = False,+ pad_token_id = 0,+ )+ config._attn_implementation = "sdpa"+ torch.manual_seed(4460)+ checkpoint = tmp_path / "base"+ Gemma3TextModel(config).save_pretrained(checkpoint)+ vocab = {f"word{i}": i for i in range(64)}+ del vocab["word0"], vocab["word1"]+ vocab.update({"[PAD]": 0, "[UNK]": 1})+ tokenizer = Tokenizer(WordLevel(vocab, unk_token = "[UNK]"))+ tokenizer.pre_tokenizer = Whitespace()+ PreTrainedTokenizerFast(+ tokenizer_object = tokenizer,+ pad_token = "[PAD]",+ unk_token = "[UNK]",+ model_max_length = 32,+ ).save_pretrained(checkpoint)+ options = (+ "model_kwargs"+ if "model_kwargs" in inspect.signature(Transformer).parameters+ else "model_args"+ )+ transformer = Transformer(str(checkpoint), **{options: {"attn_implementation": "sdpa"}})+ SentenceTransformer(modules = [transformer, Pooling(32)], device = "cpu").save_pretrained(+ tmp_path / "sentence"+ )+ model = FastSentenceTransformer.from_pretrained(+ str(tmp_path / "sentence"),+ dtype = torch.bfloat16,+ load_in_4bit = False,+ full_finetuning = True,+ max_seq_length = 32,+ use_gradient_checkpointing = False,+ attn_implementation = loader_backend,+ )+ base = model[0].auto_model+ if not base.layers[0].self_attn.forward.__module__.startswith(+ "unsloth_zoo.temporary_patches.gemma"+ ):+ pytest.skip("installed Zoo does not use the affected Gemma attention implementation")+ if loader_backend == "sdpa":+ from unsloth.models.sentence_transformer import _ensure_sentence_attention_masks++ # Exercise the backend repair without requiring an FA extension to load.+ base.config._attn_implementation = "flash_attention_2"+ assert _ensure_sentence_attention_masks(base)+ if hasattr(model[0], "unpad_inputs"):+ model[0].unpad_inputs = model[0].unpad_inputs+ assert base.config._attn_implementation == "sdpa"+ assert base.config.use_bidirectional_attention is True+ assert not getattr(model[0], "can_flatten_inputs", False)+ base.config.use_cache = False+ masks = []+ handle = base.layers[0].self_attn.register_forward_pre_hook(+ lambda module, args, kwargs: masks.append(kwargs["attention_mask"].detach()),+ with_kwargs = True,+ )+ features = {+ "input_ids": torch.tensor([[5, 6, 7, 0, 0, 0], [8, 9, 10, 11, 12, 13]], device = "cuda"),+ "attention_mask": torch.tensor([[1, 1, 1, 0, 0, 0], [1, 1, 1, 1, 1, 1]], device = "cuda"),+ }+ for training in (False, True):+ model.train(training)+ altered = {key: value.clone() for key, value in features.items()}+ altered["input_ids"][1] = 14+ with torch.autocast("cuda", dtype = torch.bfloat16):+ expected = model({key: value.clone() for key, value in features.items()})[+ "sentence_embedding"+ ]+ actual = model(altered)["sentence_embedding"]+ torch.testing.assert_close(actual[0], expected[0], rtol = 0, atol = 0)+ assert not torch.equal(actual[1], expected[1])+ model.zero_grad(set_to_none = True)+ actual[0].float().square().sum().backward()+ # Token 14 appears only in the unrelated row.+ assert torch.count_nonzero(base.get_input_embeddings().weight.grad[14]) == 0+ assert all(mask.ndim == 4 for mask in masks)+ mask = masks[0]+ allowed = mask if mask.dtype == torch.bool else mask == 0+ assert allowed[0, 0, 0, 1] # Future token is visible in bidirectional attention.+ assert not allowed[0, 0, 0, 3] # Padding is never visible.+ assert not allowed[1, 0, 0, 5] # Out of the sliding window.+ handle.remove()diff --git a/unsloth/models/sentence_transformer.py b/unsloth/models/sentence_transformer.pyindex f63b855c02f..ea45ada465a 100644--- a/unsloth/models/sentence_transformer.py+++ b/unsloth/models/sentence_transformer.py@@ -71,6 +71,34 @@ import shutil_CREATE_TRANSFORMER_MODULE_LOCK = threading.RLock()+def _ensure_sentence_attention_masks(model):+ """Match Gemma3's advertised backend to its patched SDPA implementation."""+ config = getattr(model, "config", None)+ if getattr(config, "model_type", None) != "gemma3_text" or "flash" not in str(+ getattr(config, "_attn_implementation", "")+ ):+ return False+ # Zoo's Gemma3 attention preserves FP32 Q/K but consumes SDPA masks. With+ # a Flash backend, Transformers omits those masks and ST can flatten rows;+ # that implementation does not consume the resulting sequence boundaries.+ patched = any(+ getattr(module.forward, "__module__", "").startswith("unsloth_zoo.temporary_patches.gemma")+ for module in model.modules()+ if type(module).__name__ == "Gemma3Attention"+ )+ if patched:+ if hasattr(model, "set_attn_implementation"):+ model.set_attn_implementation("sdpa")+ else:+ config._attn_implementation = "sdpa"+ logging.warning(+ "Unsloth: Using SDPA for patched Gemma3 sentence attention to preserve "+ "padding, sequence boundaries and bidirectional window masks."+ )+ return True+ return False++def _normalize_save_method(save_method):"""Fold "MERGED_16BIT" and "merged 16bit" onto "merged_16bit". unsloth_save_model (save.py) normalizes case and spaces before validating, so the same spelling has to mean the same thing here, else a keyword call that worked before starts raising."""if isinstance(save_method, str):@@ -1524,6 +1552,11 @@ class FastSentenceTransformer(FastModel):st_kwargs["cache_folder"] = _st_cachest_model = SentenceTransformer(model_name, **st_kwargs)+ if _ensure_sentence_attention_masks(+ getattr(st_model[0], "auto_model", None)+ ) and hasattr(st_model[0], "unpad_inputs"):+ # Refresh ST's cached capability decision after changing the backend.+ st_model[0].unpad_inputs = st_model[0].unpad_inputsreturn st_modelif "auto_model" not in kwargs:@@ -1796,6 +1829,8 @@ class FastSentenceTransformer(FastModel):finally:os.environ["UNSLOTH_WARN_UNINITIALIZED"] = old_environ+ _ensure_sentence_attention_masks(model)+from sentence_transformers import SentenceTransformermodules, no_modules = FastSentenceTransformer._load_modules(