Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep padding-free off for attention that mixes packed documents (DeepSeek V4, Zaya, RecurrentGemma) (#16119)

Daniel Hana4242baa9629
diff --git a/tests/test_padding_free_boundary_leak.py b/tests/test_padding_free_boundary_leak.py
new file mode 100644
index 00000000000..bbd09b48675
--- /dev/null
+++ b/tests/test_padding_free_boundary_leak.py
@@ -0,0 +1,164 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+from types import SimpleNamespace
+
+import pytest
+
+torch = pytest.importorskip("torch")
+transformers = pytest.importorskip("transformers")
+
+try:
+ import unsloth.trainer as trainer_module
+except ImportError:
+ import unsloth
+ if getattr(unsloth, "DEVICE_TYPE", None) != "mlx":
+ raise
+ pytest.skip("unsloth.trainer is the MLX shim here", allow_module_level = True)
+
+
[email protected](autouse = True)
+def _return_logits_unset(monkeypatch):
+ monkeypatch.delenv("UNSLOTH_RETURN_LOGITS", raising = False)
+ monkeypatch.delenv("UNSLOTH_PADDING_CHOICE", raising = False)
+
+
+def _tiny(model_type):
+ config = transformers.AutoConfig.for_model(model_type)
+ common = dict(
+ hidden_size = 64,
+ intermediate_size = 128,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ head_dim = 16,
+ vocab_size = 256,
+ max_position_embeddings = 512,
+ )
+ if model_type == "deepseek_v4":
+ common.update(
+ num_key_value_heads = 1,
+ head_dim = 32,
+ moe_intermediate_size = 64,
+ n_routed_experts = 4,
+ num_experts_per_tok = 2,
+ q_lora_rank = 32,
+ o_lora_rank = 32,
+ o_groups = 2,
+ qk_rope_head_dim = 16,
+ index_n_heads = 2,
+ index_head_dim = 16,
+ index_topk = 4096,
+ sliding_window = 16,
+ compress_rates = {"compressed_sparse_attention": 4, "heavily_compressed_attention": 8},
+ num_hidden_layers = 3,
+ layer_types = [
+ "sliding_attention",
+ "compressed_sparse_attention",
+ "heavily_compressed_attention",
+ ],
+ mlp_layer_types = ["moe", "moe", "moe"],
+ )
+ else:
+ common.update(num_hidden_layers = 2)
+ for key, value in common.items():
+ setattr(config, key, value)
+ config._attn_implementation = "eager"
+ torch.manual_seed(0)
+ return transformers.AutoModelForCausalLM.from_config(config, torch_dtype = torch.float32).eval()
+
+
+def _tiny_or_skip(model_type):
+ try:
+ return _tiny(model_type)
+ except (KeyError, ValueError) as exc:
+ pytest.skip(f"transformers {transformers.__version__} cannot build {model_type}: {exc}")
+
+
+def _second_document_error(model):
+ g = torch.Generator().manual_seed(1)
+ a = torch.randint(3, 250, (1, 48), generator = g)
+ b = torch.randint(3, 250, (1, 40), generator = g)
+ with torch.no_grad():
+ alone = model(input_ids = b, use_cache = False).logits
+ position_ids = torch.cat([torch.arange(a.shape[1]), torch.arange(b.shape[1])])[None]
+ packed = model(
+ input_ids = torch.cat([a, b], 1), position_ids = position_ids, use_cache = False
+ ).logits[:, a.shape[1] :]
+ return ((packed - alone).norm() / alone.norm()).item()
+
+
+class _StubTrainer:
+ def __init__(
+ self,
+ model = None,
+ args = None,
+ **kwargs,
+ ):
+ self.model = model
+ self.args = args
+
+
+def _sft_args(**overrides):
+ args = SimpleNamespace(
+ packing = False,
+ padding_free = None,
+ max_length = 64,
+ eval_packing = None,
+ )
+ for key, value in overrides.items():
+ setattr(args, key, value)
+ return args
+
+
+def _build_sft(monkeypatch, model, args):
+ for name in ("enable_padding_free_metadata", "enable_sample_packing"):
+ monkeypatch.setattr(trainer_module, name, lambda model, trainer: None)
+ module = SimpleNamespace(SFTTrainer = type("SFTTrainer", (_StubTrainer,), {}))
+ trainer_module._patch_sft_trainer_auto_packing(module)
+ module.SFTTrainer(model = model, args = args)
+ return args
+
+
+def test_deepseek_v4_packed_row_leaks_and_dense_does_not():
+ v4 = _tiny_or_skip("deepseek_v4")
+ assert _second_document_error(v4) > 1e-2
+ assert _second_document_error(_tiny("llama")) < 1e-5
+
+
+def test_sft_keeps_padding_free_off_for_deepseek_v4(monkeypatch, capsys):
+ v4 = _tiny_or_skip("deepseek_v4")
+ args = _build_sft(monkeypatch, v4, _sft_args())
+ assert args.padding_free is False and args.packing is False
+ assert "mixes tokens across packed documents" in capsys.readouterr().out
+ args = _build_sft(monkeypatch, v4, _sft_args(padding_free = True, eval_packing = True))
+ assert args.padding_free is False and args.eval_packing is False
+ args = _build_sft(monkeypatch, v4, _sft_args(packing = True))
+ assert args.packing is False and args.padding_free is False
+
+
[email protected]("model_type", ["llama", "qwen3"])
+def test_sft_still_auto_enables_padding_free_for_dense(monkeypatch, capsys, model_type):
+ model = _tiny_or_skip(model_type)
+ args = _build_sft(monkeypatch, model, _sft_args())
+ assert args.padding_free is True
+ assert "mixes tokens across packed documents" not in capsys.readouterr().out
+
+
+ "name",
+ [
+ "DeepseekV4CSACompressor",
+ "DeepseekV4HCACompressor",
+ "ZayaCCAProjection",
+ "RecurrentGemmaRglru",
+ ],
+)
+def test_boundary_blocker_matches_modules_and_string_model_types(name):
+ from unsloth.utils.packing import packing_boundary_blocker
+
+ inner = type(name, (torch.nn.Module,), {})()
+ model = torch.nn.Sequential(torch.nn.Linear(2, 2), torch.nn.Sequential(inner))
+ assert packing_boundary_blocker(model) == name
+ assert packing_boundary_blocker(torch.nn.Linear(2, 2)) is None
+ assert packing_boundary_blocker("org/repo", ["deepseek_v4"]) == "deepseek_v4"
+ assert packing_boundary_blocker("org/repo", ["deepseek_v3"]) is None
diff --git a/unsloth/trainer.py b/unsloth/trainer.py
index 0411aff9b5d..da7e41edaa7 100644
--- a/unsloth/trainer.py
+++ b/unsloth/trainer.py
@@ -40,7 +40,11 @@ from unsloth.utils import (
enable_padding_free_metadata,
enable_sample_packing,
)
-from unsloth.utils.packing import _stateful_mixer_kind, patch_hybrid_linear_attention_varlen
+from unsloth.utils.packing import (
+ _stateful_mixer_kind,
+ packing_boundary_blocker,
+ patch_hybrid_linear_attention_varlen,
+)
from unsloth.utils.token_budget import install_token_budget
from unsloth_zoo.training_utils import (
unsloth_train as _unsloth_train,
@@ -1829,6 +1833,7 @@ def _patch_sft_trainer_auto_packing(trl_module):
model = args[0] if len(args) >= 1 else kwargs.get("model")
model_config = None
+ model_types = ()
is_vlm = False
is_unsupported_model = False
is_hybrid = False
@@ -1905,6 +1910,7 @@ def _patch_sft_trainer_auto_packing(trl_module):
is_ring_context_parallel = (getattr(config_arg, "context_parallel_size", 1) or 1) > 1 and (
getattr(config_arg, "context_parallel_backend", None) or "ring"
) == "ring"
+ boundary_blocker = packing_boundary_blocker(model, model_types)
# Read before __init__: the compiled trainer ends it with model.for_inference(), which sets the flag.
eval_logits_wanted = os.environ.get(
"UNSLOTH_RETURN_LOGITS", "0"
@@ -1922,13 +1928,29 @@ def _patch_sft_trainer_auto_packing(trl_module):
or forward_rejects_packing
)
requested_pack = bool(getattr(config_arg, "packing", False))
+ if boundary_blocker is not None:
+ if (
+ not blocked
+ and not requested_pack
+ and (
+ getattr(config_arg, "padding_free", None) is True
+ or _should_auto_padding_free(config_arg)
+ )
+ ):
+ print(
+ f"Unsloth: Padding-free batching off: {boundary_blocker} mixes tokens across "
+ "packed documents."
+ )
+ blocked = True
if blocked:
if hasattr(config_arg, "packing"):
setattr(config_arg, "packing", False)
if hasattr(config_arg, "padding_free"):
setattr(config_arg, "padding_free", False)
# TRL resolves eval_packing on its own; packed eval batches would hit ring attention's varlen refusal.
- if is_ring_context_parallel and getattr(config_arg, "eval_packing", None):
+ if (is_ring_context_parallel or boundary_blocker is not None) and getattr(
+ config_arg, "eval_packing", None
+ ):
setattr(config_arg, "eval_packing", False)
if blocked and requested_pack:
@@ -1947,6 +1969,8 @@ def _patch_sft_trainer_auto_packing(trl_module):
reason = f"unsupported model type(s): {', '.join(model_types)}"
elif is_ring_context_parallel:
reason = "ring context parallelism"
+ elif boundary_blocker is not None:
+ reason = f"{boundary_blocker} mixes tokens across packed documents"
elif forward_rejects_packing:
# Name the real blocker (for a string `model=`, the resolved class).
blocker = (
diff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.py
index 75065fa4ea3..4e6e7c88c87 100644
--- a/unsloth/utils/packing.py
+++ b/unsloth/utils/packing.py
@@ -473,6 +473,29 @@ def _stateful_mixer_kind(module) -> Optional[str]:
return None
+# Pool / recur over the whole row (V4 compressed entries numbered row-wide), ignoring position_ids.
+_PACKING_BOUNDARY_BLIND_MODULES = (
+ "DeepseekV4CSACompressor",
+ "DeepseekV4HCACompressor",
+ "ZayaCCAProjection",
+ "RecurrentGemmaRglru",
+)
+_PACKING_BOUNDARY_BLIND_MODEL_TYPES = ("deepseek_v4", "zaya", "recurrent_gemma")
+
+
+def packing_boundary_blocker(model, model_types = ()) -> Optional[str]:
+ modules = getattr(model, "modules", None)
+ if callable(modules) and not isinstance(model, str):
+ for module in modules():
+ name = type(module).__name__
+ if name in _PACKING_BOUNDARY_BLIND_MODULES:
+ return name
+ for model_type in model_types or ():
+ if model_type in _PACKING_BOUNDARY_BLIND_MODEL_TYPES:
+ return model_type
+ return None
+
+
def _iter_stateful_mixers(model):
"""(module, kind) for every stateful mixer, outermost first; submodules of a gated-delta
mixer (its own short convs) are owned by that mixer's shim."""
@@ -2146,4 +2169,5 @@ __all__ = [
"mask_packed_sequence_boundaries",
"mask_packed_boundary_labels",
"clear_packed_caches",
+ "packing_boundary_blocker",
]