Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

SentenceTransformer: preserve masks for patched Gemma3 attention (#11866)

Etherl8d229136de44
diff --git a/tests/python/test_sentence_transformer_gemma_masks.py b/tests/python/test_sentence_transformer_gemma_masks.py
new file mode 100644
index 00000000000..3ecc124ad37
--- /dev/null
+++ b/tests/python/test_sentence_transformer_gemma_masks.py
@@ -0,0 +1,124 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Gemma's patched SDPA attention must receive masks before ST can flatten inputs."""
+
+import inspect
+
+import pytest
+import torch
+
+from real_accelerator import has_real_accelerator
+
+
[email protected]("loader_backend", ["sdpa", "flash_attention_2"])
+def test_patched_gemma_loader_preserves_sequence_isolation(tmp_path, loader_backend):
+ if not has_real_accelerator() or not torch.cuda.is_available():
+ pytest.skip("real Gemma loader requires CUDA")
+ pytest.importorskip("sentence_transformers")
+ from transformers.utils import is_flash_attn_2_available
+
+ if loader_backend == "flash_attention_2" and not is_flash_attn_2_available():
+ pytest.skip("regression exercises an installed FA2 loader before the SDPA safety fallback")
+ from unsloth import FastSentenceTransformer
+ from transformers import Gemma3TextConfig, Gemma3TextModel, PreTrainedTokenizerFast
+ from sentence_transformers import SentenceTransformer
+ from sentence_transformers.models import Pooling, Transformer
+ from tokenizers import Tokenizer
+ from tokenizers.models import WordLevel
+ from tokenizers.pre_tokenizers import Whitespace
+
+ config = Gemma3TextConfig(
+ vocab_size = 64,
+ hidden_size = 32,
+ intermediate_size = 48,
+ num_hidden_layers = 2,
+ num_attention_heads = 2,
+ num_key_value_heads = 1,
+ head_dim = 16,
+ max_position_embeddings = 32,
+ sliding_window = 4,
+ layer_types = ["sliding_attention", "full_attention"],
+ use_bidirectional_attention = True,
+ use_cache = False,
+ pad_token_id = 0,
+ )
+ config._attn_implementation = "sdpa"
+ torch.manual_seed(4460)
+ checkpoint = tmp_path / "base"
+ Gemma3TextModel(config).save_pretrained(checkpoint)
+ vocab = {f"word{i}": i for i in range(64)}
+ del vocab["word0"], vocab["word1"]
+ vocab.update({"[PAD]": 0, "[UNK]": 1})
+ tokenizer = Tokenizer(WordLevel(vocab, unk_token = "[UNK]"))
+ tokenizer.pre_tokenizer = Whitespace()
+ PreTrainedTokenizerFast(
+ tokenizer_object = tokenizer,
+ pad_token = "[PAD]",
+ unk_token = "[UNK]",
+ model_max_length = 32,
+ ).save_pretrained(checkpoint)
+ options = (
+ "model_kwargs"
+ if "model_kwargs" in inspect.signature(Transformer).parameters
+ else "model_args"
+ )
+ transformer = Transformer(str(checkpoint), **{options: {"attn_implementation": "sdpa"}})
+ SentenceTransformer(modules = [transformer, Pooling(32)], device = "cpu").save_pretrained(
+ tmp_path / "sentence"
+ )
+ model = FastSentenceTransformer.from_pretrained(
+ str(tmp_path / "sentence"),
+ dtype = torch.bfloat16,
+ load_in_4bit = False,
+ full_finetuning = True,
+ max_seq_length = 32,
+ use_gradient_checkpointing = False,
+ attn_implementation = loader_backend,
+ )
+ base = model[0].auto_model
+ if not base.layers[0].self_attn.forward.__module__.startswith(
+ "unsloth_zoo.temporary_patches.gemma"
+ ):
+ pytest.skip("installed Zoo does not use the affected Gemma attention implementation")
+ if loader_backend == "sdpa":
+ from unsloth.models.sentence_transformer import _ensure_sentence_attention_masks
+
+ # Exercise the backend repair without requiring an FA extension to load.
+ base.config._attn_implementation = "flash_attention_2"
+ assert _ensure_sentence_attention_masks(base)
+ if hasattr(model[0], "unpad_inputs"):
+ model[0].unpad_inputs = model[0].unpad_inputs
+ assert base.config._attn_implementation == "sdpa"
+ assert base.config.use_bidirectional_attention is True
+ assert not getattr(model[0], "can_flatten_inputs", False)
+ base.config.use_cache = False
+ masks = []
+ handle = base.layers[0].self_attn.register_forward_pre_hook(
+ lambda module, args, kwargs: masks.append(kwargs["attention_mask"].detach()),
+ with_kwargs = True,
+ )
+ features = {
+ "input_ids": torch.tensor([[5, 6, 7, 0, 0, 0], [8, 9, 10, 11, 12, 13]], device = "cuda"),
+ "attention_mask": torch.tensor([[1, 1, 1, 0, 0, 0], [1, 1, 1, 1, 1, 1]], device = "cuda"),
+ }
+ for training in (False, True):
+ model.train(training)
+ altered = {key: value.clone() for key, value in features.items()}
+ altered["input_ids"][1] = 14
+ with torch.autocast("cuda", dtype = torch.bfloat16):
+ expected = model({key: value.clone() for key, value in features.items()})[
+ "sentence_embedding"
+ ]
+ actual = model(altered)["sentence_embedding"]
+ torch.testing.assert_close(actual[0], expected[0], rtol = 0, atol = 0)
+ assert not torch.equal(actual[1], expected[1])
+ model.zero_grad(set_to_none = True)
+ actual[0].float().square().sum().backward()
+ # Token 14 appears only in the unrelated row.
+ assert torch.count_nonzero(base.get_input_embeddings().weight.grad[14]) == 0
+ assert all(mask.ndim == 4 for mask in masks)
+ mask = masks[0]
+ allowed = mask if mask.dtype == torch.bool else mask == 0
+ assert allowed[0, 0, 0, 1] # Future token is visible in bidirectional attention.
+ assert not allowed[0, 0, 0, 3] # Padding is never visible.
+ assert not allowed[1, 0, 0, 5] # Out of the sliding window.
+ handle.remove()
diff --git a/unsloth/models/sentence_transformer.py b/unsloth/models/sentence_transformer.py
index f63b855c02f..ea45ada465a 100644
--- a/unsloth/models/sentence_transformer.py
+++ b/unsloth/models/sentence_transformer.py
@@ -71,6 +71,34 @@ import shutil
_CREATE_TRANSFORMER_MODULE_LOCK = threading.RLock()
+def _ensure_sentence_attention_masks(model):
+ """Match Gemma3's advertised backend to its patched SDPA implementation."""
+ config = getattr(model, "config", None)
+ if getattr(config, "model_type", None) != "gemma3_text" or "flash" not in str(
+ getattr(config, "_attn_implementation", "")
+ ):
+ return False
+ # Zoo's Gemma3 attention preserves FP32 Q/K but consumes SDPA masks. With
+ # a Flash backend, Transformers omits those masks and ST can flatten rows;
+ # that implementation does not consume the resulting sequence boundaries.
+ patched = any(
+ getattr(module.forward, "__module__", "").startswith("unsloth_zoo.temporary_patches.gemma")
+ for module in model.modules()
+ if type(module).__name__ == "Gemma3Attention"
+ )
+ if patched:
+ if hasattr(model, "set_attn_implementation"):
+ model.set_attn_implementation("sdpa")
+ else:
+ config._attn_implementation = "sdpa"
+ logging.warning(
+ "Unsloth: Using SDPA for patched Gemma3 sentence attention to preserve "
+ "padding, sequence boundaries and bidirectional window masks."
+ )
+ return True
+ return False
+
+
def _normalize_save_method(save_method):
"""Fold "MERGED_16BIT" and "merged 16bit" onto "merged_16bit". unsloth_save_model (save.py) normalizes case and spaces before validating, so the same spelling has to mean the same thing here, else a keyword call that worked before starts raising."""
if isinstance(save_method, str):
@@ -1524,6 +1552,11 @@ class FastSentenceTransformer(FastModel):
st_kwargs["cache_folder"] = _st_cache
st_model = SentenceTransformer(model_name, **st_kwargs)
+ if _ensure_sentence_attention_masks(
+ getattr(st_model[0], "auto_model", None)
+ ) and hasattr(st_model[0], "unpad_inputs"):
+ # Refresh ST's cached capability decision after changing the backend.
+ st_model[0].unpad_inputs = st_model[0].unpad_inputs
return st_model
if "auto_model" not in kwargs:
@@ -1796,6 +1829,8 @@ class FastSentenceTransformer(FastModel):
finally:
os.environ["UNSLOTH_WARN_UNINITIALIZED"] = old_environ
+ _ensure_sentence_attention_masks(model)
+
from sentence_transformers import SentenceTransformer
modules, no_modules = FastSentenceTransformer._load_modules(