Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
SentenceTransformer: opt-in encoder unpadding via shared attention dispatch (#4460)
Etherl615eed1413a7
diff --git a/tests/python/test_sentence_transformer_unpadding.py b/tests/python/test_sentence_transformer_unpadding.pynew file mode 100644index 00000000000..51eed7ec6a1--- /dev/null+++ b/tests/python/test_sentence_transformer_unpadding.py@@ -0,0 +1,285 @@+# SPDX-License-Identifier: AGPL-3.0-only+"""Packed vs padded parity for SentenceTransformer encoder unpadding on tiny BERT / RoBERTa.++"simulated" replaces flash_attn_varlen_func with per-segment SDPA (integration + gradients,+not FlashAttention numerics); "xformers" runs the real BlockDiagonalMask kernel.+"""++import copy+import importlib.util+import inspect+import subprocess+import sys++import pytest+import torch++from real_accelerator import has_real_accelerator++pytestmark = [+ pytest.mark.skipif(+ importlib.util.find_spec("sentence_transformers") is None,+ reason = "needs the optional sentence-transformers package",+ ),+ pytest.mark.skipif(+ int(__import__("transformers").__version__.split(".")[0]) < 5,+ reason = "encoder attention interface needs Transformers 5",+ ),+]++LENGTHS = [3, 7, 12]+++def _build(+ family,+ root,+ pooling = "mean",+):+ from sentence_transformers import SentenceTransformer+ from sentence_transformers.models import Pooling, Transformer+ from tokenizers import Tokenizer+ from tokenizers.models import WordLevel+ from tokenizers.pre_tokenizers import Whitespace+ from transformers import BertConfig, BertModel, PreTrainedTokenizerFast+ from transformers import RobertaConfig, RobertaModel++ pad = 0 if family == "bert" else 1+ config_class, model_class = {+ "bert": (BertConfig, BertModel),+ "roberta": (RobertaConfig, RobertaModel),+ }[family]+ config = config_class(+ vocab_size = 64,+ hidden_size = 16,+ num_hidden_layers = 2,+ num_attention_heads = 2,+ intermediate_size = 24,+ max_position_embeddings = 16,+ pad_token_id = pad,+ hidden_dropout_prob = 0.0,+ attention_probs_dropout_prob = 0.0,+ )+ torch.manual_seed(4460)+ checkpoint = root / family+ model_class(config).save_pretrained(checkpoint)+ vocab = {f"word{i}": i for i in range(64) if i not in (pad, 2)}+ vocab.update({"[PAD]": pad, "[UNK]": 2})+ tokenizer = Tokenizer(WordLevel(vocab, unk_token = "[UNK]"))+ tokenizer.pre_tokenizer = Whitespace()+ PreTrainedTokenizerFast(+ tokenizer_object = tokenizer, pad_token = "[PAD]", unk_token = "[UNK]"+ ).save_pretrained(checkpoint)+ key = (+ "model_kwargs"+ if "model_kwargs" in inspect.signature(Transformer).parameters+ else "model_args"+ )+ transformer = Transformer(str(checkpoint), **{key: {"attn_implementation": "sdpa"}})+ return SentenceTransformer(+ modules = [transformer, Pooling(16, pooling_mode = pooling)], device = "cpu"+ )+++def _features(+ model,+ device = "cuda",+ lengths = LENGTHS,+):+ width = max(lengths)+ ids = torch.full((len(lengths), width), model[0].auto_model.config.pad_token_id)+ mask = torch.zeros_like(ids)+ for row, length in enumerate(lengths):+ ids[row, :length] = torch.arange(5 + row, 5 + row + length)+ mask[row, :length] = 1+ return {"input_ids": ids.to(device), "attention_mask": mask.to(device)}++[email protected](params = ["bert", "roberta"])+def tiny(request, tmp_path):+ return _build(request.param, tmp_path)++[email protected](params = ["simulated", "xformers"])+def kernel(request, monkeypatch):+ """Forces the backend and records the packed token count of every attention call."""+ from unsloth.utils import attention_dispatch as ad+ from unsloth.utils import packing++ if not has_real_accelerator() or not torch.cuda.is_available():+ pytest.skip("packed path needs CUDA")+ calls = []+ if request.param == "xformers":+ if not ad.HAS_XFORMERS or packing._XFormersBidirectionalMask is None:+ pytest.skip("xFormers with BlockDiagonalMask is not installed")+ original = ad.xformers_attention++ def counted(query, key, value, **kwargs):+ assert type(kwargs["attn_bias"]) is packing._XFormersBidirectionalMask+ calls.append(query.shape[1])+ return original(query, key, value, **kwargs)++ monkeypatch.setattr(ad, "xformers_attention", counted)+ monkeypatch.setattr(ad, "select_attention_backend", lambda **_: ad.XFORMERS)+ return calls++ def simulated(query, key, value, cu_q, cu_k, max_q, max_k, **kwargs):+ assert kwargs["causal"] is False and torch.equal(cu_q, cu_k)+ calls.append(query.shape[0])+ bounds = cu_q.tolist()+ return torch.cat(+ [+ torch.nn.functional.scaled_dot_product_attention(+ *(t[a:b].transpose(0, 1) for t in (query, key, value)),+ dropout_p = kwargs.get("dropout_p", 0.0),+ scale = kwargs.get("softmax_scale"),+ ).transpose(0, 1)+ for a, b in zip(bounds, bounds[1:])+ ]+ )++ monkeypatch.setattr(ad, "flash_attn_varlen_func", simulated, raising = False)+ monkeypatch.setattr(ad, "select_attention_backend", lambda **_: ad.FLASH_VARLEN)+ return calls+++def _enable(model, auto = False):+ from unsloth.models._sentence_transformer_unpadding import enable_sentence_transformer_unpadding+ assert enable_sentence_transformer_unpadding(model, auto = auto)+ return model+++def _relative_error(actual, expected):+ actual, expected = actual.float(), expected.float()+ return (+ torch.linalg.vector_norm(actual - expected)+ / torch.linalg.vector_norm(expected).clamp_min(1e-6)+ ).item()++[email protected]("dtype", [torch.float16, torch.bfloat16])+def test_packed_training_step_matches_padded(tiny, kernel, dtype):+ reference = copy.deepcopy(tiny).cuda().to(dtype).train()+ packed = _enable(tiny).cuda().to(dtype).train()+ outputs = []+ for model in (reference, packed):+ out = model(_features(model))+ loss = out["sentence_embedding"].float().square().mean()+ loss.backward()+ outputs.append((out, loss))+ assert kernel == [sum(LENGTHS)] * 2+ (ref_out, ref_loss), (out, loss) = outputs+ tol = 0.02 if dtype == torch.float16 else 0.05+ assert _relative_error(out["sentence_embedding"], ref_out["sentence_embedding"]) < tol+ assert abs(loss.item() - ref_loss.item()) <= tol * abs(ref_loss.item())+ mask = out["attention_mask"].bool()+ assert torch.equal(+ out["token_embeddings"][~mask], torch.zeros_like(out["token_embeddings"][~mask])+ )+ ref_params = dict(reference.named_parameters())+ for name, param in packed.named_parameters():+ expected = ref_params[name].grad+ if expected is not None and expected.float().norm() > 1e-4:+ assert _relative_error(param.grad, expected) < 3 * tol, name+++def test_packed_sentences_do_not_share_attention(tiny, kernel):+ model = _enable(tiny).cuda().half().train()+ features = _features(model)+ before = model(dict(features))["sentence_embedding"]+ features["input_ids"][1:, :3] = 40+ after = model(dict(features))["sentence_embedding"]+ assert torch.equal(before[0], after[0])+ assert not torch.equal(before[1:], after[1:])++[email protected]("case", ["eval", "auto_small_batch", "no_padding", "fp32_no_autocast"])+def test_ineligible_batches_stay_padded(tiny, kernel, case):+ reference = copy.deepcopy(tiny).cuda()+ model = _enable(tiny, auto = case == "auto_small_batch").cuda()+ lengths = [12, 12, 12] if case == "no_padding" else LENGTHS+ for m in (reference, model):+ if case != "fp32_no_autocast":+ m.half()+ m.train(case != "eval")+ expected = reference(_features(reference, lengths = lengths))["sentence_embedding"]+ actual = model(_features(model, lengths = lengths))["sentence_embedding"]+ assert kernel == []+ torch.testing.assert_close(actual, expected, rtol = 0, atol = 0)+++def test_autocast_fp32_weights_are_packed(tiny, kernel):+ model = _enable(tiny).cuda().train()+ with torch.autocast("cuda", dtype = torch.bfloat16):+ model(_features(model))["sentence_embedding"].float().sum().backward()+ assert kernel == [sum(LENGTHS)] * 2+++def test_installs_through_unsloth_zoo_bf16_autocast_wrapper(tiny, kernel):+ from unsloth_zoo.training_utils import _wrap_forward_in_bf16_autocast++ base = tiny[0].auto_model+ _wrap_forward_in_bf16_autocast(base, torch.bfloat16)+ assert type(base) is not type(base)._unsloth_autocast_base+ model = _enable(tiny).cuda().train()+ model(_features(model))["sentence_embedding"].float().sum().backward()+ assert kernel == [sum(LENGTHS)] * 2++[email protected]("pooling", ["cls", "max", "mean_sqrt_len_tokens"])+def test_non_mean_pooling_is_not_installed(tmp_path, kernel, pooling):+ from unsloth.models._sentence_transformer_unpadding import enable_sentence_transformer_unpadding++ model = _build("bert", tmp_path, pooling = pooling)+ forward = model.forward+ assert not enable_sentence_transformer_unpadding(model)+ assert model.forward == forward+ assert model[0].auto_model.config._attn_implementation == "sdpa"+++def test_disable_restores_stock_forwards(tiny, kernel):+ from unsloth.models._sentence_transformer_unpadding import (+ disable_sentence_transformer_unpadding,+ )++ encoder = tiny[0].auto_model.encoder+ forward, encoder_forward = tiny.forward, encoder.forward+ _enable(tiny)+ assert disable_sentence_transformer_unpadding(tiny)+ assert tiny.forward == forward and encoder.forward == encoder_forward+ assert tiny[0].auto_model.config._attn_implementation == "sdpa"+ tiny.cuda().half().train()+ tiny(_features(tiny))+ assert kernel == []+++def test_invalid_policy_rejected_before_loading():+ from unsloth import FastSentenceTransformer+ for invalid in ("AUTO", "true", 1, None):+ with pytest.raises(ValueError, match = "use_unpadding"):+ FastSentenceTransformer.from_pretrained("does-not-exist", use_unpadding = invalid)+++def test_saved_model_reloads_with_stock_sentence_transformers(kernel, tmp_path):+ model = _enable(_build("bert", tmp_path)).cuda().half().train()+ model(_features(model))["sentence_embedding"].float().square().mean().backward()+ torch.optim.SGD(model.parameters(), lr = 0.1).step()+ assert kernel+ model.save_pretrained(str(tmp_path / "saved"))+ model.eval().cpu().float()+ features = _features(model, "cpu")+ with torch.no_grad():+ expected = model(dict(features))["sentence_embedding"]+ torch.save({"features": features, "expected": expected}, tmp_path / "expected.pt")+ script = """import sys, torch+from sentence_transformers import SentenceTransformer+data = torch.load(sys.argv[1] + '/expected.pt', weights_only=True)+model = SentenceTransformer(sys.argv[1] + '/saved', device='cpu').float().eval()+with torch.no_grad():+ actual = model(data['features'])['sentence_embedding']+torch.testing.assert_close(actual, data['expected'], atol=1e-5, rtol=1e-4)+assert 'unsloth' not in sys.modules+"""+ result = subprocess.run(+ [sys.executable, "-c", script, str(tmp_path)], capture_output = True, text = True, timeout = 300+ )+ assert result.returncode == 0, result.stdout + result.stderrdiff --git a/tests/utils/test_attention_dispatch_bidirectional.py b/tests/utils/test_attention_dispatch_bidirectional.pynew file mode 100644index 00000000000..e8f8b4d9e8d--- /dev/null+++ b/tests/utils/test_attention_dispatch_bidirectional.py@@ -0,0 +1,166 @@+# SPDX-License-Identifier: AGPL-3.0-only+"""Bidirectional packed attention must preserve sentence boundaries in every fallback."""++from dataclasses import replace++import pytest+import torch+from torch.nn.functional import scaled_dot_product_attention++import unsloth # noqa: F401+from unsloth.utils import attention_dispatch as ad+from unsloth.utils import packing+++def _context(lengths = (2, 3), **kwargs):+ lengths = torch.tensor(lengths, dtype = torch.int32)+ total = int(lengths.sum())+ cu = torch.cat((torch.zeros(1, dtype = torch.int32), lengths.cumsum(0).int()))+ return ad.AttentionContext(+ bsz = 1,+ q_len = total,+ kv_seq_len = total,+ n_heads = 2,+ head_dim = 4,+ requires_grad = True,+ seq_info = (lengths, cu, int(lengths.max())),+ attention_mask = None,+ causal_mask = None,+ is_causal = False,+ **kwargs,+ )+++def _qkv():+ generator = torch.Generator().manual_seed(4460)+ return tuple(torch.randn(1, 2, 5, 4, generator = generator).requires_grad_() for _ in range(3))+++def _run(+ context,+ qkv,+ backend = ad.SDPA,+ **kwargs,+):+ config = ad.AttentionConfig(backend = backend, n_kv_heads = 2, n_groups = 1, **kwargs)+ return ad.run_attention(config = config, context = context, Q = qkv[0], K = qkv[1], V = qkv[2])++[email protected]("backend", [ad.SDPA, ad.FLASH_VARLEN, ad.XFORMERS])+def test_packed_bidirectional_outputs_and_gradients_match_independent_rows(monkeypatch, backend):+ # Flash / xFormers: force the int32-overflow SDPA fallback so CPU SDPA runs for real.+ if backend != ad.SDPA:+ monkeypatch.setattr(ad, "_VARLEN_INT32_GUARD_DISABLED", False)+ monkeypatch.setattr(ad, "_varlen_backward_overflows_int32", lambda *args: True)+ monkeypatch.setattr(ad, "_warn_varlen_int32_overflow_once", lambda *args: None)++ def unexpected_kernel(*args, **kwargs):+ pytest.fail("overflow must bypass the accelerator kernel")++ monkeypatch.setattr(ad, "flash_attn_varlen_func", unexpected_kernel)+ monkeypatch.setattr(ad, "xformers_attention", unexpected_kernel, raising = False)++ qkv = _qkv()+ output = _run(_context(), qkv, backend)+ reference = torch.cat(+ [+ scaled_dot_product_attention(*(tensor[:, :, start:end] for tensor in qkv))+ for start, end in ((0, 2), (2, 5))+ ],+ dim = 2,+ ).transpose(1, 2)+ torch.testing.assert_close(output, reference)+ actual_grads = torch.autograd.grad(output.square().sum(), qkv, retain_graph = True)+ reference_grads = torch.autograd.grad(reference.square().sum(), qkv)+ for actual, expected in zip(actual_grads, reference_grads):+ torch.testing.assert_close(actual, expected)+++def test_future_tokens_affect_their_sentence_but_not_other_sentences():+ qkv = tuple(torch.zeros_like(tensor) for tensor in _qkv())+ output = _run(_context(), qkv)+ changed = list(qkv)+ changed[2] = changed[2].clone()+ changed[2][:, :, 1] = 6+ modified = _run(_context(), changed)+ torch.testing.assert_close(modified[:, 0], torch.full_like(modified[:, 0], 3))+ torch.testing.assert_close(modified[:, 2:], output[:, 2:])+++def test_dense_bidirectional_sdpa_matches_reference():+ qkv = _qkv()+ output = _run(replace(_context(), seq_info = None), qkv)+ torch.testing.assert_close(output, scaled_dot_product_attention(*qkv).transpose(1, 2))+++def test_default_context_preserves_causal_sdpa():+ qkv = _qkv()+ context = replace(_context(), seq_info = None, is_causal = True)+ torch.testing.assert_close(+ _run(context, qkv),+ scaled_dot_product_attention(*qkv, is_causal = True).transpose(1, 2),+ )+ assert ad.AttentionContext.__dataclass_fields__["is_causal"].default is True++[email protected]("backend", [ad.FLASH_DENSE, ad.FLASH_VARLEN])+def test_flash_receives_noncausal_without_mutating_config(monkeypatch, backend):+ captured = []++ def fake_kernel(query, key, value, *args, **kwargs):+ captured.append(kwargs["causal"])+ return torch.zeros_like(query)++ monkeypatch.setattr(ad, "flash_attn_func", fake_kernel)+ monkeypatch.setattr(ad, "flash_attn_varlen_func", fake_kernel)+ kwargs = {"causal": True}+ _run(_context(), _qkv(), backend, flash_dense_kwargs = kwargs, flash_varlen_kwargs = kwargs)+ assert captured == [False]+ assert kwargs == {"causal": True}+++def test_sdpa_mask_cache_distinguishes_direction():+ seq_info = _context().seq_info+ options = dict(dtype = torch.float32, device = torch.device("cpu"))+ causal = packing.build_sdpa_packed_attention_mask(seq_info, **options)+ bidirectional = packing.build_sdpa_packed_attention_mask(seq_info, is_causal = False, **options)+ assert torch.isneginf(causal[0, 0, 0, 1])+ assert bidirectional[0, 0, 0, 1] == 0+ assert torch.isneginf(bidirectional[0, 0, 0, 2])+ torch.testing.assert_close(+ packing.build_sdpa_packed_attention_mask(seq_info, **options), causal+ )+++def test_xformers_uses_bidirectional_blocks_and_separate_cache(monkeypatch):+ class Block:+ @classmethod+ def from_seqlens(cls, lengths):+ return cls()++ class CausalBlock(Block):+ pass++ monkeypatch.setattr(packing, "_XFormersBlockMask", CausalBlock)+ monkeypatch.setattr(packing, "_XFormersBidirectionalMask", Block)+ monkeypatch.setattr(packing, "_XFORMERS_MASK_CACHE", packing.OrderedDict())+ monkeypatch.setattr(packing, "_XFORMERS_BLOCK_MASK_CACHE", {})+ seq_info = _context().seq_info+ causal = packing.build_xformers_block_causal_mask(seq_info)+ bidirectional = packing.build_xformers_block_causal_mask(seq_info, is_causal = False)+ assert type(causal) is CausalBlock+ assert type(bidirectional) is Block+ assert packing.build_xformers_block_causal_mask(seq_info) is causal++ def fake_kernel(+ query,+ key,+ value,+ attn_bias = None,+ **kwargs,+ ):+ assert type(attn_bias) is Block+ return torch.zeros_like(query)++ monkeypatch.setattr(ad, "xformers_attention", fake_kernel, raising = False)+ _run(_context(), _qkv(), ad.XFORMERS)diff --git a/unsloth/models/_sentence_transformer_unpadding.py b/unsloth/models/_sentence_transformer_unpadding.pynew file mode 100644index 00000000000..d41f3340c06--- /dev/null+++ b/unsloth/models/_sentence_transformer_unpadding.py@@ -0,0 +1,249 @@+# SPDX-License-Identifier: AGPL-3.0-only++"""Training-only encoder unpadding for mean-pooled BERT / RoBERTa SentenceTransformers.++Embeddings and pooling stay stock; only valid tokens enter the encoder, whose output is+scattered back to the padded layout before pooling. Checkpoints are unchanged.+"""++from types import MethodType++import torch++_ATTENTION = "unsloth_sentence_varlen"+_MASK = "_unsloth_sentence_padding_mask"+_SEQUENCES = "_unsloth_sentence_sequences"+# Below this many padded slots MiniLM training was CPU-launch-bound (RTX 5090).+_MIN_AUTO_TOKENS = 8192+++def _unpadding_backend():+ from ..utils import attention_dispatch as ad+ from ..utils.packing import _XFormersBidirectionalMask++ backend = ad.select_attention_backend(use_varlen = True)+ if backend == ad.FLASH_VARLEN:+ return backend+ if backend == ad.XFORMERS and _XFormersBidirectionalMask is not None:+ return backend+ return None+++def _sentence_attention(module, query, key, value, attention_mask, **kwargs):+ from transformers.integrations.sdpa_attention import sdpa_attention_forward+ from ..utils.attention_dispatch import AttentionConfig, AttentionContext, run_attention++ packed = kwargs.pop(_SEQUENCES, None)+ if packed is None:+ return sdpa_attention_forward(module, query, key, value, attention_mask, **kwargs)+ seq_info, backend = packed+ heads, tokens, head_dim = query.shape[1:]+ dropout, scale = kwargs.get("dropout", 0.0), kwargs.get("scaling")+ config = AttentionConfig(+ backend = backend,+ n_kv_heads = key.shape[1],+ n_groups = heads // key.shape[1],+ flash_varlen_kwargs = {"dropout_p": dropout, "softmax_scale": scale},+ xformers_kwargs = {"p": dropout, "scale": scale},+ sdpa_kwargs = {"dropout_p": dropout, "scale": scale},+ )+ context = AttentionContext(+ bsz = 1,+ q_len = tokens,+ kv_seq_len = tokens,+ n_heads = heads,+ head_dim = head_dim,+ requires_grad = module.training,+ seq_info = seq_info,+ attention_mask = None,+ causal_mask = None,+ is_causal = False,+ )+ return run_attention(config = config, context = context, Q = query, K = key, V = value), None+++def _mean_pooling_pipeline(model):+ from sentence_transformers.models import Dense, Normalize, Pooling, Transformer++ modules = list(model.children())+ if len(modules) < 2 or type(modules[0]) is not Transformer or type(modules[1]) is not Pooling:+ return False+ pooling = modules[1]+ modes = getattr(pooling, "pooling_mode", None)+ if modes is not None:+ if modes not in ("mean", ["mean"]):+ return False+ elif pooling.get_pooling_mode_str() != "mean":+ return False+ return all(type(module) in (Dense, Normalize) for module in modules[2:])+++def _encoder_forward(+ encoder,+ hidden_states,+ attention_mask = None,+ *args,+ **kwargs,+):+ mask = kwargs.pop(_MASK, None)+ original_forward = encoder._unsloth_original_forward+ if args or mask is None:+ return original_forward(hidden_states, attention_mask, *args, **kwargs)+ # FP32 hidden states are only packed under autocast, where Q/K/V come out half precision.+ if (+ torch.compiler.is_compiling()+ or not encoder.training+ or not hidden_states.is_cuda+ or hidden_states.shape[:2] != mask.shape+ or mask.numel() < encoder._unsloth_min_tokens+ or (hidden_states.dtype == torch.float32 and not torch.is_autocast_enabled())+ or kwargs.get("encoder_hidden_states") is not None+ or kwargs.get("past_key_values") is not None+ or (backend := _unpadding_backend()) is None+ ):+ return original_forward(hidden_states, attention_mask = attention_mask, **kwargs)++ keep = mask == 1+ lengths = keep.sum(dim = 1, dtype = torch.int32)+ summary = torch.cat((lengths, ((mask == 0) | keep).all().to(torch.int32).view(1))).tolist()+ row_lengths, binary = summary[:-1], summary[-1]+ batch, width, channels = hidden_states.shape+ if not binary or min(row_lengths) == 0 or sum(row_lengths) == batch * width:+ return original_forward(hidden_states, attention_mask = attention_mask, **kwargs)++ indices = keep.view(-1).nonzero().flatten()+ cu_seqlens = torch.nn.functional.pad(lengths.cumsum(0, dtype = torch.int32), (1, 0))+ seq_info = (lengths, cu_seqlens, max(row_lengths))+ packed = hidden_states.reshape(-1, channels).index_select(0, indices).unsqueeze(0)+ output = original_forward(+ packed, attention_mask = None, **kwargs, **{_SEQUENCES: (seq_info, backend)}+ )+ restored = output.last_hidden_state.new_zeros((batch * width, channels))+ output.last_hidden_state = restored.index_copy(+ 0, indices, output.last_hidden_state.squeeze(0)+ ).view(batch, width, channels)+ return output+++def _sentence_forward(model, input, **kwargs):+ original_forward = model._unsloth_original_forward+ if (+ torch.compiler.is_compiling()+ or not model.training+ # A routed keyword mask overrides the one in features.+ or "attention_mask" in kwargs+ or not _mean_pooling_pipeline(model)+ ):+ return original_forward(input, **kwargs)+ mask = input.get("attention_mask")+ base = model[0].auto_model+ config = base.config+ flags = ("output_hidden_states", "output_attentions")+ if (+ hasattr(base, "_orig_mod")+ or config._attn_implementation != _ATTENTION+ or not isinstance(mask, torch.Tensor)+ or mask.ndim != 2+ or mask.numel() == 0+ or input.get("modality", "text") != "text"+ or any(getattr(config, f) or kwargs.get(f) or input.get(f) for f in flags)+ ):+ return original_forward(input, **kwargs)+ result = original_forward({**input, _MASK: mask}, **kwargs)+ result.pop(_MASK, None)+ input.update(result)+ return result+++def disable_sentence_transformer_unpadding(model):+ """Restore stock forwards, e.g. before torch.compile; caller-replaced forwards are kept."""+ if not getattr(model, "_unsloth_unpadding_installed", False):+ return False+ transformer = model[0]+ base = transformer.auto_model+ base = getattr(base, "_orig_mod", base)+ if hasattr(base, "get_base_model"):+ base = base.get_base_model()+ encoder = base.encoder+ if (+ getattr(model.forward, "__func__", None) is not _sentence_forward+ or getattr(encoder.forward, "__func__", None) is not _encoder_forward+ ):+ return False+ model.forward = model._unsloth_original_forward+ encoder.forward = encoder._unsloth_original_forward+ if base.config._attn_implementation == _ATTENTION:+ base.config._attn_implementation = "sdpa"+ if transformer.model_forward_params is not None:+ transformer.model_forward_params = set(transformer.model_forward_params) - {_MASK}+ del model._unsloth_original_forward, model._unsloth_unpadding_installed+ del encoder._unsloth_original_forward, encoder._unsloth_min_tokens+ return True+++def enable_sentence_transformer_unpadding(model, auto = False):+ """Pack valid tokens through FlashAttention varlen / xFormers during training.++ Returns False (model untouched) unless: Transformers 5, stock BertModel / RobertaModel+ with absolute positions and SDPA, a mean-only pooling pipeline, and a bidirectional+ varlen backend. ``auto`` skips batches under _MIN_AUTO_TOKENS padded slots.+ """+ import transformers++ if getattr(model, "_unsloth_unpadding_installed", False):+ return True+ if int(transformers.__version__.split(".")[0]) < 5 or not _mean_pooling_pipeline(model):+ return False+ backend = _unpadding_backend()+ if backend is None:+ return False++ from transformers.models.bert.modeling_bert import BertModel+ from transformers.models.roberta.modeling_roberta import RobertaModel++ transformer = model[0]+ base = transformer.auto_model+ # Exact types (seen through unsloth_zoo's bf16 autocast subclass): PEFT-wrapped,+ # remote-code and compiled models keep the stock path.+ base_type = getattr(type(base), "_unsloth_autocast_base", type(base))+ if base_type not in (BertModel, RobertaModel):+ return False+ config = base.config+ head_dim = config.hidden_size // config.num_attention_heads+ if (+ config.is_decoder+ or config.add_cross_attention+ or head_dim > 256+ or (backend == "xformers" and head_dim % 8 != 0)+ or getattr(config, "position_embedding_type", "absolute") != "absolute"+ or getattr(config, "chunk_size_feed_forward", 0)+ or config._attn_implementation != "sdpa"+ or not hasattr(transformer, "model_forward_params")+ ):+ return False+ modalities = getattr(transformer, "modality_config", None)+ if modalities is not None and modalities.get("text") != {+ "method": "forward",+ "method_output_name": "last_hidden_state",+ }:+ return False++ from transformers import AttentionInterface, AttentionMaskInterface+ from transformers.masking_utils import sdpa_mask++ AttentionInterface.register(_ATTENTION, _sentence_attention)+ # Without a registered mask builder the padded fallback would silently lose its padding mask.+ AttentionMaskInterface.register(_ATTENTION, sdpa_mask)+ config._attn_implementation = _ATTENTION+ # None means ST forwards every feature key unfiltered.+ if transformer.model_forward_params is not None:+ transformer.model_forward_params = set(transformer.model_forward_params) | {_MASK}+ encoder = base.encoder+ # Bound methods (not closures) so deepcopy rebinds to the copy.+ encoder._unsloth_original_forward = encoder.forward+ encoder._unsloth_min_tokens = _MIN_AUTO_TOKENS if auto else 0+ encoder.forward = MethodType(_encoder_forward, encoder)+ model._unsloth_original_forward = model.forward+ model.forward = MethodType(_sentence_forward, model)+ model._unsloth_unpadding_installed = True+ return Truediff --git a/unsloth/models/sentence_transformer.py b/unsloth/models/sentence_transformer.pyindex f382a5ab343..fb4a0ccedc6 100644--- a/unsloth/models/sentence_transformer.py+++ b/unsloth/models/sentence_transformer.py@@ -2167,9 +2167,23 @@ class FastSentenceTransformer(FastModel):# Lower bound to avoid compiling on extremely short runs.return int(max(20, final_threshold))+ @staticmethod+ def _enable_unpadding(st_model, use_unpadding):+ if not use_unpadding:+ return+ from ._sentence_transformer_unpadding import enable_sentence_transformer_unpadding+ if not enable_sentence_transformer_unpadding(st_model, auto = use_unpadding == "auto"):+ print(+ "Unsloth: use_unpadding needs Transformers 5, a BERT / RoBERTa encoder, "+ "mean pooling and flash-attn or xformers; training stays padded."+ )+@staticmethoddef _apply_torch_compile(model, mode = "default"):"""Apply torch.compile to a SentenceTransformer model (with an accelerate unwrap_model bug workaround)."""+ if getattr(model, "_unsloth_unpadding_installed", False):+ from ._sentence_transformer_unpadding import disable_sentence_transformer_unpadding+ disable_sentence_transformer_unpadding(model)if hasattr(model, "__getitem__"):inner_model = model[0].auto_modelcompiled = torch.compile(inner_model, mode = mode)@@ -2209,8 +2223,13 @@ class FastSentenceTransformer(FastModel):unsloth_tiled_mlp = False,pooling_mode = "mean",for_inference = False,+ use_unpadding = False,**kwargs,):+ # use_unpadding: "auto" packs eligible training batches with >= 8192 padded slots,+ # True packs every eligible batch (saves memory, can cost speed).+ if use_unpadding not in (False, True, "auto") or type(use_unpadding) not in (bool, str):+ raise ValueError('use_unpadding must be "auto", True, or False')try:from sentence_transformers import SentenceTransformerfrom sentence_transformers.models import Transformer, Pooling, Normalize@@ -2448,6 +2467,7 @@ class FastSentenceTransformer(FastModel):st_model._dtype = dtypest_model._load_in_4bit = load_in_4bitst_model.no_modules = False+ FastSentenceTransformer._enable_unpadding(st_model, use_unpadding)FastSentenceTransformer._patch_transformer_module_save_config(st_model[0], getattr(st_model[0].auto_model, "config", None))@@ -2774,6 +2794,7 @@ class FastSentenceTransformer(FastModel):st_model.push_to_hub_merged = types.MethodType(_push_to_hub_merged, st_model)st_model._unsloth_trust_remote_code = trust_remote_code+ FastSentenceTransformer._enable_unpadding(st_model, use_unpadding)return st_model@staticmethoddiff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.pyindex 5d0edc6a3ec..7f0a60888cf 100644--- a/unsloth/utils/attention_dispatch.py+++ b/unsloth/utils/attention_dispatch.py@@ -201,6 +201,7 @@ class AttentionContext:# PrefixGrouper: non-None routes Q/K/V through the FlexAttention shared-prefix kernel; None leaves# every existing construction and behavior unchanged.prefix_seg_info: Optional[Any] = None+ is_causal: bool = Truedef select_attention_backend(use_varlen: bool = False) -> str:@@ -364,6 +365,12 @@ def run_attention(flash_varlen_kwargs = config.flash_varlen_kwargs or {}sdpa_kwargs = config.sdpa_kwargs or {}xformers_kwargs = config.xformers_kwargs or {}+ # Mask builders only see is_causal when False, so causal callers and their patches are unchanged.+ direction = {}+ if not context.is_causal:+ flash_dense_kwargs = {**flash_dense_kwargs, "causal": False}+ flash_varlen_kwargs = {**flash_varlen_kwargs, "causal": False}+ direction = {"is_causal": False}bsz = context.bszn_heads = context.n_heads@@ -428,6 +435,7 @@ def run_attention(# Only CausalLM_fast_forward supplies the mask; a direct decoder call (Liger, TRL's get_decoder paths) would attend bidirectionally.if (base_mask is None+ and context.is_causaland xformers is not Noneand context.seq_info is Noneand q_len == kv_seq_len@@ -438,6 +446,7 @@ def run_attention(sliding_window = sliding_window,base_mask = base_mask,total_tokens = K.shape[-2],+ **direction,)attn_bias = move_xformers_attention_bias(attn_bias, Q.device)@@ -503,6 +512,7 @@ def run_attention(device = Q.device,sliding_window = sliding_window,total_tokens = K.shape[-2],+ **direction,)else:q_len_local = Q.shape[-2]@@ -523,7 +533,13 @@ def run_attention(q_pos = torch.arange(past_len, past_len + q_len_local, device = Q.device)k_pos = torch.arange(k_len_local, device = Q.device)- causal_keep = k_pos[None, :] <= q_pos[:, None] # True = allowed (SDPA)+ causal_keep = (+ k_pos[None, :] <= q_pos[:, None]+ if context.is_causal+ else torch.ones(+ (q_len_local, k_len_local), dtype = torch.bool, device = Q.device+ )+ ) # True = allowed (SDPA)if sliding_window is not None:causal_keep &= k_pos[None, :] >= (q_pos[:, None] - (sliding_window - 1))@@ -553,7 +569,9 @@ def run_attention(q_len_local, k_len_local, sliding_window, Q.device)- is_causal_local = local_mask is None and q_len_local == k_len_local+ is_causal_local = (+ context.is_causal and local_mask is None and q_len_local == k_len_local+ )kwargs = dict(sdpa_kwargs)kwargs.setdefault("attn_mask", local_mask)diff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.pyindex 419a863ae0c..7e23b4b57b5 100644--- a/unsloth/utils/packing.py+++ b/unsloth/utils/packing.py@@ -34,8 +34,18 @@ except Exception:except Exception:_XFormersBlockMask = None+try:+ from xformers.ops.fmha.attn_bias import BlockDiagonalMask as _XFormersBidirectionalMask+except Exception:+ try:+ from xformers.attn_bias import BlockDiagonalMask as _XFormersBidirectionalMask+ except Exception:+ _XFormersBidirectionalMask = None+_XFORMERS_MASK_CACHE_MAXSIZE = 32-_XFORMERS_MASK_CACHE: OrderedDict[Tuple[torch.device, Tuple[int, ...], int], Any] = OrderedDict()+_XFORMERS_MASK_CACHE: OrderedDict[Tuple[torch.device, Tuple[int, ...], int, bool], Any] = (+ OrderedDict()+)# Cache per device for get_packed_info_from_kwargs to avoid repeated D2H sync across layers_PACKED_INFO_CACHE: dict = {}@@ -103,20 +113,24 @@ def move_xformers_attention_bias(attn_bias: Any, device: torch.device):def _get_cached_block_mask(- lengths: Tuple[int, ...], sliding_window: Optional[int], device: torch.device+ lengths: Tuple[int, ...],+ sliding_window: Optional[int],+ device: torch.device,+ is_causal: bool = True,):- if _XFormersBlockMask is None:+ mask_class = _XFormersBlockMask if is_causal else _XFormersBidirectionalMask+ if mask_class is None:return Nonedevice = torch.device(device)window_key = _window_cache_key(sliding_window)- cache_key = (device, lengths, window_key)+ cache_key = (device, lengths, window_key, is_causal)cached = _XFORMERS_MASK_CACHE.get(cache_key)if cached is not None:_XFORMERS_MASK_CACHE.move_to_end(cache_key)return cached- mask = _XFormersBlockMask.from_seqlens(list(lengths))+ mask = mask_class.from_seqlens(list(lengths))if window_key and mask is not None and hasattr(mask, "make_local_attention"):mask = mask.make_local_attention(window_size = window_key)mask = move_xformers_attention_bias(mask, device)@@ -636,14 +650,16 @@ def build_xformers_block_causal_mask(sliding_window: Optional[int] = None,base_mask: Optional[Any] = None,total_tokens: Optional[int] = None,+ is_causal: bool = True,):- if _XFormersBlockMask is None:+ mask_class = _XFormersBlockMask if is_causal else _XFormersBidirectionalMask+ if mask_class is None:return Noneif seq_info is not None:seq_lengths, _, _ = seq_info# Cache the mask to avoid repeated D2H sync across layersdevice = seq_lengths.device- params = (sliding_window, total_tokens)+ params = (sliding_window, total_tokens, is_causal)entry = _XFORMERS_BLOCK_MASK_CACHE.get(device)if entry is not None and entry["seq_lengths"] is seq_lengths and entry["params"] == params:return entry["mask"]@@ -653,7 +669,7 @@ def build_xformers_block_causal_mask(return Nonelengths = tuple(int(x) for x in lengths_tensor.tolist())lengths = _with_padding_segment(lengths, total_tokens)- mask = _get_cached_block_mask(lengths, sliding_window, device)+ mask = _get_cached_block_mask(lengths, sliding_window, device, is_causal = is_causal)_XFORMERS_BLOCK_MASK_CACHE[device] = {"seq_lengths": seq_lengths,@@ -680,10 +696,11 @@ def build_sdpa_packed_attention_mask(device: torch.device,sliding_window: Optional[int] = None,total_tokens: Optional[int] = None,+ is_causal: bool = True,) -> torch.Tensor:seq_lengths, _, _ = seq_info- params = (dtype, sliding_window, total_tokens)+ params = (dtype, sliding_window, total_tokens, is_causal)entry = _SDPA_MASK_CACHE.get(device)if entry is not None and entry["seq_lengths"] is seq_lengths and entry["params"] == params:return entry["mask"]@@ -704,8 +721,9 @@ def build_sdpa_packed_attention_mask(if length <= 0:continueblock = torch.zeros((length, length), dtype = dtype, device = device)- upper = torch.triu(torch.ones((length, length), device = device), diagonal = 1).bool()- block = block.masked_fill(upper, float("-inf"))+ if is_causal:+ upper = torch.triu(torch.ones((length, length), device = device), diagonal = 1).bool()+ block = block.masked_fill(upper, float("-inf"))if sliding_window is not None and sliding_window > 0 and length > sliding_window:idx = torch.arange(length, device = device)dist = idx.unsqueeze(1) - idx.unsqueeze(0)