Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

SentenceTransformer: opt-in encoder unpadding via shared attention dispatch (#4460)

Etherl615eed1413a7
diff --git a/tests/python/test_sentence_transformer_unpadding.py b/tests/python/test_sentence_transformer_unpadding.py
new file mode 100644
index 00000000000..51eed7ec6a1
--- /dev/null
+++ b/tests/python/test_sentence_transformer_unpadding.py
@@ -0,0 +1,285 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""Packed vs padded parity for SentenceTransformer encoder unpadding on tiny BERT / RoBERTa.
+
+"simulated" replaces flash_attn_varlen_func with per-segment SDPA (integration + gradients,
+not FlashAttention numerics); "xformers" runs the real BlockDiagonalMask kernel.
+"""
+
+import copy
+import importlib.util
+import inspect
+import subprocess
+import sys
+
+import pytest
+import torch
+
+from real_accelerator import has_real_accelerator
+
+pytestmark = [
+ pytest.mark.skipif(
+ importlib.util.find_spec("sentence_transformers") is None,
+ reason = "needs the optional sentence-transformers package",
+ ),
+ pytest.mark.skipif(
+ int(__import__("transformers").__version__.split(".")[0]) < 5,
+ reason = "encoder attention interface needs Transformers 5",
+ ),
+]
+
+LENGTHS = [3, 7, 12]
+
+
+def _build(
+ family,
+ root,
+ pooling = "mean",
+):
+ from sentence_transformers import SentenceTransformer
+ from sentence_transformers.models import Pooling, Transformer
+ from tokenizers import Tokenizer
+ from tokenizers.models import WordLevel
+ from tokenizers.pre_tokenizers import Whitespace
+ from transformers import BertConfig, BertModel, PreTrainedTokenizerFast
+ from transformers import RobertaConfig, RobertaModel
+
+ pad = 0 if family == "bert" else 1
+ config_class, model_class = {
+ "bert": (BertConfig, BertModel),
+ "roberta": (RobertaConfig, RobertaModel),
+ }[family]
+ config = config_class(
+ vocab_size = 64,
+ hidden_size = 16,
+ num_hidden_layers = 2,
+ num_attention_heads = 2,
+ intermediate_size = 24,
+ max_position_embeddings = 16,
+ pad_token_id = pad,
+ hidden_dropout_prob = 0.0,
+ attention_probs_dropout_prob = 0.0,
+ )
+ torch.manual_seed(4460)
+ checkpoint = root / family
+ model_class(config).save_pretrained(checkpoint)
+ vocab = {f"word{i}": i for i in range(64) if i not in (pad, 2)}
+ vocab.update({"[PAD]": pad, "[UNK]": 2})
+ tokenizer = Tokenizer(WordLevel(vocab, unk_token = "[UNK]"))
+ tokenizer.pre_tokenizer = Whitespace()
+ PreTrainedTokenizerFast(
+ tokenizer_object = tokenizer, pad_token = "[PAD]", unk_token = "[UNK]"
+ ).save_pretrained(checkpoint)
+ key = (
+ "model_kwargs"
+ if "model_kwargs" in inspect.signature(Transformer).parameters
+ else "model_args"
+ )
+ transformer = Transformer(str(checkpoint), **{key: {"attn_implementation": "sdpa"}})
+ return SentenceTransformer(
+ modules = [transformer, Pooling(16, pooling_mode = pooling)], device = "cpu"
+ )
+
+
+def _features(
+ model,
+ device = "cuda",
+ lengths = LENGTHS,
+):
+ width = max(lengths)
+ ids = torch.full((len(lengths), width), model[0].auto_model.config.pad_token_id)
+ mask = torch.zeros_like(ids)
+ for row, length in enumerate(lengths):
+ ids[row, :length] = torch.arange(5 + row, 5 + row + length)
+ mask[row, :length] = 1
+ return {"input_ids": ids.to(device), "attention_mask": mask.to(device)}
+
+
[email protected](params = ["bert", "roberta"])
+def tiny(request, tmp_path):
+ return _build(request.param, tmp_path)
+
+
[email protected](params = ["simulated", "xformers"])
+def kernel(request, monkeypatch):
+ """Forces the backend and records the packed token count of every attention call."""
+ from unsloth.utils import attention_dispatch as ad
+ from unsloth.utils import packing
+
+ if not has_real_accelerator() or not torch.cuda.is_available():
+ pytest.skip("packed path needs CUDA")
+ calls = []
+ if request.param == "xformers":
+ if not ad.HAS_XFORMERS or packing._XFormersBidirectionalMask is None:
+ pytest.skip("xFormers with BlockDiagonalMask is not installed")
+ original = ad.xformers_attention
+
+ def counted(query, key, value, **kwargs):
+ assert type(kwargs["attn_bias"]) is packing._XFormersBidirectionalMask
+ calls.append(query.shape[1])
+ return original(query, key, value, **kwargs)
+
+ monkeypatch.setattr(ad, "xformers_attention", counted)
+ monkeypatch.setattr(ad, "select_attention_backend", lambda **_: ad.XFORMERS)
+ return calls
+
+ def simulated(query, key, value, cu_q, cu_k, max_q, max_k, **kwargs):
+ assert kwargs["causal"] is False and torch.equal(cu_q, cu_k)
+ calls.append(query.shape[0])
+ bounds = cu_q.tolist()
+ return torch.cat(
+ [
+ torch.nn.functional.scaled_dot_product_attention(
+ *(t[a:b].transpose(0, 1) for t in (query, key, value)),
+ dropout_p = kwargs.get("dropout_p", 0.0),
+ scale = kwargs.get("softmax_scale"),
+ ).transpose(0, 1)
+ for a, b in zip(bounds, bounds[1:])
+ ]
+ )
+
+ monkeypatch.setattr(ad, "flash_attn_varlen_func", simulated, raising = False)
+ monkeypatch.setattr(ad, "select_attention_backend", lambda **_: ad.FLASH_VARLEN)
+ return calls
+
+
+def _enable(model, auto = False):
+ from unsloth.models._sentence_transformer_unpadding import enable_sentence_transformer_unpadding
+ assert enable_sentence_transformer_unpadding(model, auto = auto)
+ return model
+
+
+def _relative_error(actual, expected):
+ actual, expected = actual.float(), expected.float()
+ return (
+ torch.linalg.vector_norm(actual - expected)
+ / torch.linalg.vector_norm(expected).clamp_min(1e-6)
+ ).item()
+
+
[email protected]("dtype", [torch.float16, torch.bfloat16])
+def test_packed_training_step_matches_padded(tiny, kernel, dtype):
+ reference = copy.deepcopy(tiny).cuda().to(dtype).train()
+ packed = _enable(tiny).cuda().to(dtype).train()
+ outputs = []
+ for model in (reference, packed):
+ out = model(_features(model))
+ loss = out["sentence_embedding"].float().square().mean()
+ loss.backward()
+ outputs.append((out, loss))
+ assert kernel == [sum(LENGTHS)] * 2
+ (ref_out, ref_loss), (out, loss) = outputs
+ tol = 0.02 if dtype == torch.float16 else 0.05
+ assert _relative_error(out["sentence_embedding"], ref_out["sentence_embedding"]) < tol
+ assert abs(loss.item() - ref_loss.item()) <= tol * abs(ref_loss.item())
+ mask = out["attention_mask"].bool()
+ assert torch.equal(
+ out["token_embeddings"][~mask], torch.zeros_like(out["token_embeddings"][~mask])
+ )
+ ref_params = dict(reference.named_parameters())
+ for name, param in packed.named_parameters():
+ expected = ref_params[name].grad
+ if expected is not None and expected.float().norm() > 1e-4:
+ assert _relative_error(param.grad, expected) < 3 * tol, name
+
+
+def test_packed_sentences_do_not_share_attention(tiny, kernel):
+ model = _enable(tiny).cuda().half().train()
+ features = _features(model)
+ before = model(dict(features))["sentence_embedding"]
+ features["input_ids"][1:, :3] = 40
+ after = model(dict(features))["sentence_embedding"]
+ assert torch.equal(before[0], after[0])
+ assert not torch.equal(before[1:], after[1:])
+
+
[email protected]("case", ["eval", "auto_small_batch", "no_padding", "fp32_no_autocast"])
+def test_ineligible_batches_stay_padded(tiny, kernel, case):
+ reference = copy.deepcopy(tiny).cuda()
+ model = _enable(tiny, auto = case == "auto_small_batch").cuda()
+ lengths = [12, 12, 12] if case == "no_padding" else LENGTHS
+ for m in (reference, model):
+ if case != "fp32_no_autocast":
+ m.half()
+ m.train(case != "eval")
+ expected = reference(_features(reference, lengths = lengths))["sentence_embedding"]
+ actual = model(_features(model, lengths = lengths))["sentence_embedding"]
+ assert kernel == []
+ torch.testing.assert_close(actual, expected, rtol = 0, atol = 0)
+
+
+def test_autocast_fp32_weights_are_packed(tiny, kernel):
+ model = _enable(tiny).cuda().train()
+ with torch.autocast("cuda", dtype = torch.bfloat16):
+ model(_features(model))["sentence_embedding"].float().sum().backward()
+ assert kernel == [sum(LENGTHS)] * 2
+
+
+def test_installs_through_unsloth_zoo_bf16_autocast_wrapper(tiny, kernel):
+ from unsloth_zoo.training_utils import _wrap_forward_in_bf16_autocast
+
+ base = tiny[0].auto_model
+ _wrap_forward_in_bf16_autocast(base, torch.bfloat16)
+ assert type(base) is not type(base)._unsloth_autocast_base
+ model = _enable(tiny).cuda().train()
+ model(_features(model))["sentence_embedding"].float().sum().backward()
+ assert kernel == [sum(LENGTHS)] * 2
+
+
[email protected]("pooling", ["cls", "max", "mean_sqrt_len_tokens"])
+def test_non_mean_pooling_is_not_installed(tmp_path, kernel, pooling):
+ from unsloth.models._sentence_transformer_unpadding import enable_sentence_transformer_unpadding
+
+ model = _build("bert", tmp_path, pooling = pooling)
+ forward = model.forward
+ assert not enable_sentence_transformer_unpadding(model)
+ assert model.forward == forward
+ assert model[0].auto_model.config._attn_implementation == "sdpa"
+
+
+def test_disable_restores_stock_forwards(tiny, kernel):
+ from unsloth.models._sentence_transformer_unpadding import (
+ disable_sentence_transformer_unpadding,
+ )
+
+ encoder = tiny[0].auto_model.encoder
+ forward, encoder_forward = tiny.forward, encoder.forward
+ _enable(tiny)
+ assert disable_sentence_transformer_unpadding(tiny)
+ assert tiny.forward == forward and encoder.forward == encoder_forward
+ assert tiny[0].auto_model.config._attn_implementation == "sdpa"
+ tiny.cuda().half().train()
+ tiny(_features(tiny))
+ assert kernel == []
+
+
+def test_invalid_policy_rejected_before_loading():
+ from unsloth import FastSentenceTransformer
+ for invalid in ("AUTO", "true", 1, None):
+ with pytest.raises(ValueError, match = "use_unpadding"):
+ FastSentenceTransformer.from_pretrained("does-not-exist", use_unpadding = invalid)
+
+
+def test_saved_model_reloads_with_stock_sentence_transformers(kernel, tmp_path):
+ model = _enable(_build("bert", tmp_path)).cuda().half().train()
+ model(_features(model))["sentence_embedding"].float().square().mean().backward()
+ torch.optim.SGD(model.parameters(), lr = 0.1).step()
+ assert kernel
+ model.save_pretrained(str(tmp_path / "saved"))
+ model.eval().cpu().float()
+ features = _features(model, "cpu")
+ with torch.no_grad():
+ expected = model(dict(features))["sentence_embedding"]
+ torch.save({"features": features, "expected": expected}, tmp_path / "expected.pt")
+ script = """import sys, torch
+from sentence_transformers import SentenceTransformer
+data = torch.load(sys.argv[1] + '/expected.pt', weights_only=True)
+model = SentenceTransformer(sys.argv[1] + '/saved', device='cpu').float().eval()
+with torch.no_grad():
+ actual = model(data['features'])['sentence_embedding']
+torch.testing.assert_close(actual, data['expected'], atol=1e-5, rtol=1e-4)
+assert 'unsloth' not in sys.modules
+"""
+ result = subprocess.run(
+ [sys.executable, "-c", script, str(tmp_path)], capture_output = True, text = True, timeout = 300
+ )
+ assert result.returncode == 0, result.stdout + result.stderr
diff --git a/tests/utils/test_attention_dispatch_bidirectional.py b/tests/utils/test_attention_dispatch_bidirectional.py
new file mode 100644
index 00000000000..e8f8b4d9e8d
--- /dev/null
+++ b/tests/utils/test_attention_dispatch_bidirectional.py
@@ -0,0 +1,166 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""Bidirectional packed attention must preserve sentence boundaries in every fallback."""
+
+from dataclasses import replace
+
+import pytest
+import torch
+from torch.nn.functional import scaled_dot_product_attention
+
+import unsloth # noqa: F401
+from unsloth.utils import attention_dispatch as ad
+from unsloth.utils import packing
+
+
+def _context(lengths = (2, 3), **kwargs):
+ lengths = torch.tensor(lengths, dtype = torch.int32)
+ total = int(lengths.sum())
+ cu = torch.cat((torch.zeros(1, dtype = torch.int32), lengths.cumsum(0).int()))
+ return ad.AttentionContext(
+ bsz = 1,
+ q_len = total,
+ kv_seq_len = total,
+ n_heads = 2,
+ head_dim = 4,
+ requires_grad = True,
+ seq_info = (lengths, cu, int(lengths.max())),
+ attention_mask = None,
+ causal_mask = None,
+ is_causal = False,
+ **kwargs,
+ )
+
+
+def _qkv():
+ generator = torch.Generator().manual_seed(4460)
+ return tuple(torch.randn(1, 2, 5, 4, generator = generator).requires_grad_() for _ in range(3))
+
+
+def _run(
+ context,
+ qkv,
+ backend = ad.SDPA,
+ **kwargs,
+):
+ config = ad.AttentionConfig(backend = backend, n_kv_heads = 2, n_groups = 1, **kwargs)
+ return ad.run_attention(config = config, context = context, Q = qkv[0], K = qkv[1], V = qkv[2])
+
+
[email protected]("backend", [ad.SDPA, ad.FLASH_VARLEN, ad.XFORMERS])
+def test_packed_bidirectional_outputs_and_gradients_match_independent_rows(monkeypatch, backend):
+ # Flash / xFormers: force the int32-overflow SDPA fallback so CPU SDPA runs for real.
+ if backend != ad.SDPA:
+ monkeypatch.setattr(ad, "_VARLEN_INT32_GUARD_DISABLED", False)
+ monkeypatch.setattr(ad, "_varlen_backward_overflows_int32", lambda *args: True)
+ monkeypatch.setattr(ad, "_warn_varlen_int32_overflow_once", lambda *args: None)
+
+ def unexpected_kernel(*args, **kwargs):
+ pytest.fail("overflow must bypass the accelerator kernel")
+
+ monkeypatch.setattr(ad, "flash_attn_varlen_func", unexpected_kernel)
+ monkeypatch.setattr(ad, "xformers_attention", unexpected_kernel, raising = False)
+
+ qkv = _qkv()
+ output = _run(_context(), qkv, backend)
+ reference = torch.cat(
+ [
+ scaled_dot_product_attention(*(tensor[:, :, start:end] for tensor in qkv))
+ for start, end in ((0, 2), (2, 5))
+ ],
+ dim = 2,
+ ).transpose(1, 2)
+ torch.testing.assert_close(output, reference)
+ actual_grads = torch.autograd.grad(output.square().sum(), qkv, retain_graph = True)
+ reference_grads = torch.autograd.grad(reference.square().sum(), qkv)
+ for actual, expected in zip(actual_grads, reference_grads):
+ torch.testing.assert_close(actual, expected)
+
+
+def test_future_tokens_affect_their_sentence_but_not_other_sentences():
+ qkv = tuple(torch.zeros_like(tensor) for tensor in _qkv())
+ output = _run(_context(), qkv)
+ changed = list(qkv)
+ changed[2] = changed[2].clone()
+ changed[2][:, :, 1] = 6
+ modified = _run(_context(), changed)
+ torch.testing.assert_close(modified[:, 0], torch.full_like(modified[:, 0], 3))
+ torch.testing.assert_close(modified[:, 2:], output[:, 2:])
+
+
+def test_dense_bidirectional_sdpa_matches_reference():
+ qkv = _qkv()
+ output = _run(replace(_context(), seq_info = None), qkv)
+ torch.testing.assert_close(output, scaled_dot_product_attention(*qkv).transpose(1, 2))
+
+
+def test_default_context_preserves_causal_sdpa():
+ qkv = _qkv()
+ context = replace(_context(), seq_info = None, is_causal = True)
+ torch.testing.assert_close(
+ _run(context, qkv),
+ scaled_dot_product_attention(*qkv, is_causal = True).transpose(1, 2),
+ )
+ assert ad.AttentionContext.__dataclass_fields__["is_causal"].default is True
+
+
[email protected]("backend", [ad.FLASH_DENSE, ad.FLASH_VARLEN])
+def test_flash_receives_noncausal_without_mutating_config(monkeypatch, backend):
+ captured = []
+
+ def fake_kernel(query, key, value, *args, **kwargs):
+ captured.append(kwargs["causal"])
+ return torch.zeros_like(query)
+
+ monkeypatch.setattr(ad, "flash_attn_func", fake_kernel)
+ monkeypatch.setattr(ad, "flash_attn_varlen_func", fake_kernel)
+ kwargs = {"causal": True}
+ _run(_context(), _qkv(), backend, flash_dense_kwargs = kwargs, flash_varlen_kwargs = kwargs)
+ assert captured == [False]
+ assert kwargs == {"causal": True}
+
+
+def test_sdpa_mask_cache_distinguishes_direction():
+ seq_info = _context().seq_info
+ options = dict(dtype = torch.float32, device = torch.device("cpu"))
+ causal = packing.build_sdpa_packed_attention_mask(seq_info, **options)
+ bidirectional = packing.build_sdpa_packed_attention_mask(seq_info, is_causal = False, **options)
+ assert torch.isneginf(causal[0, 0, 0, 1])
+ assert bidirectional[0, 0, 0, 1] == 0
+ assert torch.isneginf(bidirectional[0, 0, 0, 2])
+ torch.testing.assert_close(
+ packing.build_sdpa_packed_attention_mask(seq_info, **options), causal
+ )
+
+
+def test_xformers_uses_bidirectional_blocks_and_separate_cache(monkeypatch):
+ class Block:
+ @classmethod
+ def from_seqlens(cls, lengths):
+ return cls()
+
+ class CausalBlock(Block):
+ pass
+
+ monkeypatch.setattr(packing, "_XFormersBlockMask", CausalBlock)
+ monkeypatch.setattr(packing, "_XFormersBidirectionalMask", Block)
+ monkeypatch.setattr(packing, "_XFORMERS_MASK_CACHE", packing.OrderedDict())
+ monkeypatch.setattr(packing, "_XFORMERS_BLOCK_MASK_CACHE", {})
+ seq_info = _context().seq_info
+ causal = packing.build_xformers_block_causal_mask(seq_info)
+ bidirectional = packing.build_xformers_block_causal_mask(seq_info, is_causal = False)
+ assert type(causal) is CausalBlock
+ assert type(bidirectional) is Block
+ assert packing.build_xformers_block_causal_mask(seq_info) is causal
+
+ def fake_kernel(
+ query,
+ key,
+ value,
+ attn_bias = None,
+ **kwargs,
+ ):
+ assert type(attn_bias) is Block
+ return torch.zeros_like(query)
+
+ monkeypatch.setattr(ad, "xformers_attention", fake_kernel, raising = False)
+ _run(_context(), _qkv(), ad.XFORMERS)
diff --git a/unsloth/models/_sentence_transformer_unpadding.py b/unsloth/models/_sentence_transformer_unpadding.py
new file mode 100644
index 00000000000..d41f3340c06
--- /dev/null
+++ b/unsloth/models/_sentence_transformer_unpadding.py
@@ -0,0 +1,249 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+
+"""Training-only encoder unpadding for mean-pooled BERT / RoBERTa SentenceTransformers.
+
+Embeddings and pooling stay stock; only valid tokens enter the encoder, whose output is
+scattered back to the padded layout before pooling. Checkpoints are unchanged.
+"""
+
+from types import MethodType
+
+import torch
+
+_ATTENTION = "unsloth_sentence_varlen"
+_MASK = "_unsloth_sentence_padding_mask"
+_SEQUENCES = "_unsloth_sentence_sequences"
+# Below this many padded slots MiniLM training was CPU-launch-bound (RTX 5090).
+_MIN_AUTO_TOKENS = 8192
+
+
+def _unpadding_backend():
+ from ..utils import attention_dispatch as ad
+ from ..utils.packing import _XFormersBidirectionalMask
+
+ backend = ad.select_attention_backend(use_varlen = True)
+ if backend == ad.FLASH_VARLEN:
+ return backend
+ if backend == ad.XFORMERS and _XFormersBidirectionalMask is not None:
+ return backend
+ return None
+
+
+def _sentence_attention(module, query, key, value, attention_mask, **kwargs):
+ from transformers.integrations.sdpa_attention import sdpa_attention_forward
+ from ..utils.attention_dispatch import AttentionConfig, AttentionContext, run_attention
+
+ packed = kwargs.pop(_SEQUENCES, None)
+ if packed is None:
+ return sdpa_attention_forward(module, query, key, value, attention_mask, **kwargs)
+ seq_info, backend = packed
+ heads, tokens, head_dim = query.shape[1:]
+ dropout, scale = kwargs.get("dropout", 0.0), kwargs.get("scaling")
+ config = AttentionConfig(
+ backend = backend,
+ n_kv_heads = key.shape[1],
+ n_groups = heads // key.shape[1],
+ flash_varlen_kwargs = {"dropout_p": dropout, "softmax_scale": scale},
+ xformers_kwargs = {"p": dropout, "scale": scale},
+ sdpa_kwargs = {"dropout_p": dropout, "scale": scale},
+ )
+ context = AttentionContext(
+ bsz = 1,
+ q_len = tokens,
+ kv_seq_len = tokens,
+ n_heads = heads,
+ head_dim = head_dim,
+ requires_grad = module.training,
+ seq_info = seq_info,
+ attention_mask = None,
+ causal_mask = None,
+ is_causal = False,
+ )
+ return run_attention(config = config, context = context, Q = query, K = key, V = value), None
+
+
+def _mean_pooling_pipeline(model):
+ from sentence_transformers.models import Dense, Normalize, Pooling, Transformer
+
+ modules = list(model.children())
+ if len(modules) < 2 or type(modules[0]) is not Transformer or type(modules[1]) is not Pooling:
+ return False
+ pooling = modules[1]
+ modes = getattr(pooling, "pooling_mode", None)
+ if modes is not None:
+ if modes not in ("mean", ["mean"]):
+ return False
+ elif pooling.get_pooling_mode_str() != "mean":
+ return False
+ return all(type(module) in (Dense, Normalize) for module in modules[2:])
+
+
+def _encoder_forward(
+ encoder,
+ hidden_states,
+ attention_mask = None,
+ *args,
+ **kwargs,
+):
+ mask = kwargs.pop(_MASK, None)
+ original_forward = encoder._unsloth_original_forward
+ if args or mask is None:
+ return original_forward(hidden_states, attention_mask, *args, **kwargs)
+ # FP32 hidden states are only packed under autocast, where Q/K/V come out half precision.
+ if (
+ torch.compiler.is_compiling()
+ or not encoder.training
+ or not hidden_states.is_cuda
+ or hidden_states.shape[:2] != mask.shape
+ or mask.numel() < encoder._unsloth_min_tokens
+ or (hidden_states.dtype == torch.float32 and not torch.is_autocast_enabled())
+ or kwargs.get("encoder_hidden_states") is not None
+ or kwargs.get("past_key_values") is not None
+ or (backend := _unpadding_backend()) is None
+ ):
+ return original_forward(hidden_states, attention_mask = attention_mask, **kwargs)
+
+ keep = mask == 1
+ lengths = keep.sum(dim = 1, dtype = torch.int32)
+ summary = torch.cat((lengths, ((mask == 0) | keep).all().to(torch.int32).view(1))).tolist()
+ row_lengths, binary = summary[:-1], summary[-1]
+ batch, width, channels = hidden_states.shape
+ if not binary or min(row_lengths) == 0 or sum(row_lengths) == batch * width:
+ return original_forward(hidden_states, attention_mask = attention_mask, **kwargs)
+
+ indices = keep.view(-1).nonzero().flatten()
+ cu_seqlens = torch.nn.functional.pad(lengths.cumsum(0, dtype = torch.int32), (1, 0))
+ seq_info = (lengths, cu_seqlens, max(row_lengths))
+ packed = hidden_states.reshape(-1, channels).index_select(0, indices).unsqueeze(0)
+ output = original_forward(
+ packed, attention_mask = None, **kwargs, **{_SEQUENCES: (seq_info, backend)}
+ )
+ restored = output.last_hidden_state.new_zeros((batch * width, channels))
+ output.last_hidden_state = restored.index_copy(
+ 0, indices, output.last_hidden_state.squeeze(0)
+ ).view(batch, width, channels)
+ return output
+
+
+def _sentence_forward(model, input, **kwargs):
+ original_forward = model._unsloth_original_forward
+ if (
+ torch.compiler.is_compiling()
+ or not model.training
+ # A routed keyword mask overrides the one in features.
+ or "attention_mask" in kwargs
+ or not _mean_pooling_pipeline(model)
+ ):
+ return original_forward(input, **kwargs)
+ mask = input.get("attention_mask")
+ base = model[0].auto_model
+ config = base.config
+ flags = ("output_hidden_states", "output_attentions")
+ if (
+ hasattr(base, "_orig_mod")
+ or config._attn_implementation != _ATTENTION
+ or not isinstance(mask, torch.Tensor)
+ or mask.ndim != 2
+ or mask.numel() == 0
+ or input.get("modality", "text") != "text"
+ or any(getattr(config, f) or kwargs.get(f) or input.get(f) for f in flags)
+ ):
+ return original_forward(input, **kwargs)
+ result = original_forward({**input, _MASK: mask}, **kwargs)
+ result.pop(_MASK, None)
+ input.update(result)
+ return result
+
+
+def disable_sentence_transformer_unpadding(model):
+ """Restore stock forwards, e.g. before torch.compile; caller-replaced forwards are kept."""
+ if not getattr(model, "_unsloth_unpadding_installed", False):
+ return False
+ transformer = model[0]
+ base = transformer.auto_model
+ base = getattr(base, "_orig_mod", base)
+ if hasattr(base, "get_base_model"):
+ base = base.get_base_model()
+ encoder = base.encoder
+ if (
+ getattr(model.forward, "__func__", None) is not _sentence_forward
+ or getattr(encoder.forward, "__func__", None) is not _encoder_forward
+ ):
+ return False
+ model.forward = model._unsloth_original_forward
+ encoder.forward = encoder._unsloth_original_forward
+ if base.config._attn_implementation == _ATTENTION:
+ base.config._attn_implementation = "sdpa"
+ if transformer.model_forward_params is not None:
+ transformer.model_forward_params = set(transformer.model_forward_params) - {_MASK}
+ del model._unsloth_original_forward, model._unsloth_unpadding_installed
+ del encoder._unsloth_original_forward, encoder._unsloth_min_tokens
+ return True
+
+
+def enable_sentence_transformer_unpadding(model, auto = False):
+ """Pack valid tokens through FlashAttention varlen / xFormers during training.
+
+ Returns False (model untouched) unless: Transformers 5, stock BertModel / RobertaModel
+ with absolute positions and SDPA, a mean-only pooling pipeline, and a bidirectional
+ varlen backend. ``auto`` skips batches under _MIN_AUTO_TOKENS padded slots.
+ """
+ import transformers
+
+ if getattr(model, "_unsloth_unpadding_installed", False):
+ return True
+ if int(transformers.__version__.split(".")[0]) < 5 or not _mean_pooling_pipeline(model):
+ return False
+ backend = _unpadding_backend()
+ if backend is None:
+ return False
+
+ from transformers.models.bert.modeling_bert import BertModel
+ from transformers.models.roberta.modeling_roberta import RobertaModel
+
+ transformer = model[0]
+ base = transformer.auto_model
+ # Exact types (seen through unsloth_zoo's bf16 autocast subclass): PEFT-wrapped,
+ # remote-code and compiled models keep the stock path.
+ base_type = getattr(type(base), "_unsloth_autocast_base", type(base))
+ if base_type not in (BertModel, RobertaModel):
+ return False
+ config = base.config
+ head_dim = config.hidden_size // config.num_attention_heads
+ if (
+ config.is_decoder
+ or config.add_cross_attention
+ or head_dim > 256
+ or (backend == "xformers" and head_dim % 8 != 0)
+ or getattr(config, "position_embedding_type", "absolute") != "absolute"
+ or getattr(config, "chunk_size_feed_forward", 0)
+ or config._attn_implementation != "sdpa"
+ or not hasattr(transformer, "model_forward_params")
+ ):
+ return False
+ modalities = getattr(transformer, "modality_config", None)
+ if modalities is not None and modalities.get("text") != {
+ "method": "forward",
+ "method_output_name": "last_hidden_state",
+ }:
+ return False
+
+ from transformers import AttentionInterface, AttentionMaskInterface
+ from transformers.masking_utils import sdpa_mask
+
+ AttentionInterface.register(_ATTENTION, _sentence_attention)
+ # Without a registered mask builder the padded fallback would silently lose its padding mask.
+ AttentionMaskInterface.register(_ATTENTION, sdpa_mask)
+ config._attn_implementation = _ATTENTION
+ # None means ST forwards every feature key unfiltered.
+ if transformer.model_forward_params is not None:
+ transformer.model_forward_params = set(transformer.model_forward_params) | {_MASK}
+ encoder = base.encoder
+ # Bound methods (not closures) so deepcopy rebinds to the copy.
+ encoder._unsloth_original_forward = encoder.forward
+ encoder._unsloth_min_tokens = _MIN_AUTO_TOKENS if auto else 0
+ encoder.forward = MethodType(_encoder_forward, encoder)
+ model._unsloth_original_forward = model.forward
+ model.forward = MethodType(_sentence_forward, model)
+ model._unsloth_unpadding_installed = True
+ return True
diff --git a/unsloth/models/sentence_transformer.py b/unsloth/models/sentence_transformer.py
index f382a5ab343..fb4a0ccedc6 100644
--- a/unsloth/models/sentence_transformer.py
+++ b/unsloth/models/sentence_transformer.py
@@ -2167,9 +2167,23 @@ class FastSentenceTransformer(FastModel):
# Lower bound to avoid compiling on extremely short runs.
return int(max(20, final_threshold))
+ @staticmethod
+ def _enable_unpadding(st_model, use_unpadding):
+ if not use_unpadding:
+ return
+ from ._sentence_transformer_unpadding import enable_sentence_transformer_unpadding
+ if not enable_sentence_transformer_unpadding(st_model, auto = use_unpadding == "auto"):
+ print(
+ "Unsloth: use_unpadding needs Transformers 5, a BERT / RoBERTa encoder, "
+ "mean pooling and flash-attn or xformers; training stays padded."
+ )
+
@staticmethod
def _apply_torch_compile(model, mode = "default"):
"""Apply torch.compile to a SentenceTransformer model (with an accelerate unwrap_model bug workaround)."""
+ if getattr(model, "_unsloth_unpadding_installed", False):
+ from ._sentence_transformer_unpadding import disable_sentence_transformer_unpadding
+ disable_sentence_transformer_unpadding(model)
if hasattr(model, "__getitem__"):
inner_model = model[0].auto_model
compiled = torch.compile(inner_model, mode = mode)
@@ -2209,8 +2223,13 @@ class FastSentenceTransformer(FastModel):
unsloth_tiled_mlp = False,
pooling_mode = "mean",
for_inference = False,
+ use_unpadding = False,
**kwargs,
):
+ # use_unpadding: "auto" packs eligible training batches with >= 8192 padded slots,
+ # True packs every eligible batch (saves memory, can cost speed).
+ if use_unpadding not in (False, True, "auto") or type(use_unpadding) not in (bool, str):
+ raise ValueError('use_unpadding must be "auto", True, or False')
try:
from sentence_transformers import SentenceTransformer
from sentence_transformers.models import Transformer, Pooling, Normalize
@@ -2448,6 +2467,7 @@ class FastSentenceTransformer(FastModel):
st_model._dtype = dtype
st_model._load_in_4bit = load_in_4bit
st_model.no_modules = False
+ FastSentenceTransformer._enable_unpadding(st_model, use_unpadding)
FastSentenceTransformer._patch_transformer_module_save_config(
st_model[0], getattr(st_model[0].auto_model, "config", None)
)
@@ -2774,6 +2794,7 @@ class FastSentenceTransformer(FastModel):
st_model.push_to_hub_merged = types.MethodType(_push_to_hub_merged, st_model)
st_model._unsloth_trust_remote_code = trust_remote_code
+ FastSentenceTransformer._enable_unpadding(st_model, use_unpadding)
return st_model
@staticmethod
diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.py
index 5d0edc6a3ec..7f0a60888cf 100644
--- a/unsloth/utils/attention_dispatch.py
+++ b/unsloth/utils/attention_dispatch.py
@@ -201,6 +201,7 @@ class AttentionContext:
# PrefixGrouper: non-None routes Q/K/V through the FlexAttention shared-prefix kernel; None leaves
# every existing construction and behavior unchanged.
prefix_seg_info: Optional[Any] = None
+ is_causal: bool = True
def select_attention_backend(use_varlen: bool = False) -> str:
@@ -364,6 +365,12 @@ def run_attention(
flash_varlen_kwargs = config.flash_varlen_kwargs or {}
sdpa_kwargs = config.sdpa_kwargs or {}
xformers_kwargs = config.xformers_kwargs or {}
+ # Mask builders only see is_causal when False, so causal callers and their patches are unchanged.
+ direction = {}
+ if not context.is_causal:
+ flash_dense_kwargs = {**flash_dense_kwargs, "causal": False}
+ flash_varlen_kwargs = {**flash_varlen_kwargs, "causal": False}
+ direction = {"is_causal": False}
bsz = context.bsz
n_heads = context.n_heads
@@ -428,6 +435,7 @@ def run_attention(
# Only CausalLM_fast_forward supplies the mask; a direct decoder call (Liger, TRL's get_decoder paths) would attend bidirectionally.
if (
base_mask is None
+ and context.is_causal
and xformers is not None
and context.seq_info is None
and q_len == kv_seq_len
@@ -438,6 +446,7 @@ def run_attention(
sliding_window = sliding_window,
base_mask = base_mask,
total_tokens = K.shape[-2],
+ **direction,
)
attn_bias = move_xformers_attention_bias(attn_bias, Q.device)
@@ -503,6 +512,7 @@ def run_attention(
device = Q.device,
sliding_window = sliding_window,
total_tokens = K.shape[-2],
+ **direction,
)
else:
q_len_local = Q.shape[-2]
@@ -523,7 +533,13 @@ def run_attention(
q_pos = torch.arange(past_len, past_len + q_len_local, device = Q.device)
k_pos = torch.arange(k_len_local, device = Q.device)
- causal_keep = k_pos[None, :] <= q_pos[:, None] # True = allowed (SDPA)
+ causal_keep = (
+ k_pos[None, :] <= q_pos[:, None]
+ if context.is_causal
+ else torch.ones(
+ (q_len_local, k_len_local), dtype = torch.bool, device = Q.device
+ )
+ ) # True = allowed (SDPA)
if sliding_window is not None:
causal_keep &= k_pos[None, :] >= (q_pos[:, None] - (sliding_window - 1))
@@ -553,7 +569,9 @@ def run_attention(
q_len_local, k_len_local, sliding_window, Q.device
)
- is_causal_local = local_mask is None and q_len_local == k_len_local
+ is_causal_local = (
+ context.is_causal and local_mask is None and q_len_local == k_len_local
+ )
kwargs = dict(sdpa_kwargs)
kwargs.setdefault("attn_mask", local_mask)
diff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.py
index 419a863ae0c..7e23b4b57b5 100644
--- a/unsloth/utils/packing.py
+++ b/unsloth/utils/packing.py
@@ -34,8 +34,18 @@ except Exception:
except Exception:
_XFormersBlockMask = None
+try:
+ from xformers.ops.fmha.attn_bias import BlockDiagonalMask as _XFormersBidirectionalMask
+except Exception:
+ try:
+ from xformers.attn_bias import BlockDiagonalMask as _XFormersBidirectionalMask
+ except Exception:
+ _XFormersBidirectionalMask = None
+
_XFORMERS_MASK_CACHE_MAXSIZE = 32
-_XFORMERS_MASK_CACHE: OrderedDict[Tuple[torch.device, Tuple[int, ...], int], Any] = OrderedDict()
+_XFORMERS_MASK_CACHE: OrderedDict[Tuple[torch.device, Tuple[int, ...], int, bool], Any] = (
+ OrderedDict()
+)
# Cache per device for get_packed_info_from_kwargs to avoid repeated D2H sync across layers
_PACKED_INFO_CACHE: dict = {}
@@ -103,20 +113,24 @@ def move_xformers_attention_bias(attn_bias: Any, device: torch.device):
def _get_cached_block_mask(
- lengths: Tuple[int, ...], sliding_window: Optional[int], device: torch.device
+ lengths: Tuple[int, ...],
+ sliding_window: Optional[int],
+ device: torch.device,
+ is_causal: bool = True,
):
- if _XFormersBlockMask is None:
+ mask_class = _XFormersBlockMask if is_causal else _XFormersBidirectionalMask
+ if mask_class is None:
return None
device = torch.device(device)
window_key = _window_cache_key(sliding_window)
- cache_key = (device, lengths, window_key)
+ cache_key = (device, lengths, window_key, is_causal)
cached = _XFORMERS_MASK_CACHE.get(cache_key)
if cached is not None:
_XFORMERS_MASK_CACHE.move_to_end(cache_key)
return cached
- mask = _XFormersBlockMask.from_seqlens(list(lengths))
+ mask = mask_class.from_seqlens(list(lengths))
if window_key and mask is not None and hasattr(mask, "make_local_attention"):
mask = mask.make_local_attention(window_size = window_key)
mask = move_xformers_attention_bias(mask, device)
@@ -636,14 +650,16 @@ def build_xformers_block_causal_mask(
sliding_window: Optional[int] = None,
base_mask: Optional[Any] = None,
total_tokens: Optional[int] = None,
+ is_causal: bool = True,
):
- if _XFormersBlockMask is None:
+ mask_class = _XFormersBlockMask if is_causal else _XFormersBidirectionalMask
+ if mask_class is None:
return None
if seq_info is not None:
seq_lengths, _, _ = seq_info
# Cache the mask to avoid repeated D2H sync across layers
device = seq_lengths.device
- params = (sliding_window, total_tokens)
+ params = (sliding_window, total_tokens, is_causal)
entry = _XFORMERS_BLOCK_MASK_CACHE.get(device)
if entry is not None and entry["seq_lengths"] is seq_lengths and entry["params"] == params:
return entry["mask"]
@@ -653,7 +669,7 @@ def build_xformers_block_causal_mask(
return None
lengths = tuple(int(x) for x in lengths_tensor.tolist())
lengths = _with_padding_segment(lengths, total_tokens)
- mask = _get_cached_block_mask(lengths, sliding_window, device)
+ mask = _get_cached_block_mask(lengths, sliding_window, device, is_causal = is_causal)
_XFORMERS_BLOCK_MASK_CACHE[device] = {
"seq_lengths": seq_lengths,
@@ -680,10 +696,11 @@ def build_sdpa_packed_attention_mask(
device: torch.device,
sliding_window: Optional[int] = None,
total_tokens: Optional[int] = None,
+ is_causal: bool = True,
) -> torch.Tensor:
seq_lengths, _, _ = seq_info
- params = (dtype, sliding_window, total_tokens)
+ params = (dtype, sliding_window, total_tokens, is_causal)
entry = _SDPA_MASK_CACHE.get(device)
if entry is not None and entry["seq_lengths"] is seq_lengths and entry["params"] == params:
return entry["mask"]
@@ -704,8 +721,9 @@ def build_sdpa_packed_attention_mask(
if length <= 0:
continue
block = torch.zeros((length, length), dtype = dtype, device = device)
- upper = torch.triu(torch.ones((length, length), device = device), diagonal = 1).bool()
- block = block.masked_fill(upper, float("-inf"))
+ if is_causal:
+ upper = torch.triu(torch.ones((length, length), device = device), diagonal = 1).bool()
+ block = block.masked_fill(upper, float("-inf"))
if sliding_window is not None and sliding_window > 0 and length > sliding_window:
idx = torch.arange(length, device = device)
dist = idx.unsqueeze(1) - idx.unsqueeze(0)