Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep padding-free off for attention that mixes packed documents (DeepSeek V4, Zaya, RecurrentGemma) (#16119)
Daniel Hana4242baa9629
diff --git a/tests/test_padding_free_boundary_leak.py b/tests/test_padding_free_boundary_leak.pynew file mode 100644index 00000000000..bbd09b48675--- /dev/null+++ b/tests/test_padding_free_boundary_leak.py@@ -0,0 +1,164 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++from types import SimpleNamespace++import pytest++torch = pytest.importorskip("torch")+transformers = pytest.importorskip("transformers")++try:+ import unsloth.trainer as trainer_module+except ImportError:+ import unsloth+ if getattr(unsloth, "DEVICE_TYPE", None) != "mlx":+ raise+ pytest.skip("unsloth.trainer is the MLX shim here", allow_module_level = True)++[email protected](autouse = True)+def _return_logits_unset(monkeypatch):+ monkeypatch.delenv("UNSLOTH_RETURN_LOGITS", raising = False)+ monkeypatch.delenv("UNSLOTH_PADDING_CHOICE", raising = False)+++def _tiny(model_type):+ config = transformers.AutoConfig.for_model(model_type)+ common = dict(+ hidden_size = 64,+ intermediate_size = 128,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ head_dim = 16,+ vocab_size = 256,+ max_position_embeddings = 512,+ )+ if model_type == "deepseek_v4":+ common.update(+ num_key_value_heads = 1,+ head_dim = 32,+ moe_intermediate_size = 64,+ n_routed_experts = 4,+ num_experts_per_tok = 2,+ q_lora_rank = 32,+ o_lora_rank = 32,+ o_groups = 2,+ qk_rope_head_dim = 16,+ index_n_heads = 2,+ index_head_dim = 16,+ index_topk = 4096,+ sliding_window = 16,+ compress_rates = {"compressed_sparse_attention": 4, "heavily_compressed_attention": 8},+ num_hidden_layers = 3,+ layer_types = [+ "sliding_attention",+ "compressed_sparse_attention",+ "heavily_compressed_attention",+ ],+ mlp_layer_types = ["moe", "moe", "moe"],+ )+ else:+ common.update(num_hidden_layers = 2)+ for key, value in common.items():+ setattr(config, key, value)+ config._attn_implementation = "eager"+ torch.manual_seed(0)+ return transformers.AutoModelForCausalLM.from_config(config, torch_dtype = torch.float32).eval()+++def _tiny_or_skip(model_type):+ try:+ return _tiny(model_type)+ except (KeyError, ValueError) as exc:+ pytest.skip(f"transformers {transformers.__version__} cannot build {model_type}: {exc}")+++def _second_document_error(model):+ g = torch.Generator().manual_seed(1)+ a = torch.randint(3, 250, (1, 48), generator = g)+ b = torch.randint(3, 250, (1, 40), generator = g)+ with torch.no_grad():+ alone = model(input_ids = b, use_cache = False).logits+ position_ids = torch.cat([torch.arange(a.shape[1]), torch.arange(b.shape[1])])[None]+ packed = model(+ input_ids = torch.cat([a, b], 1), position_ids = position_ids, use_cache = False+ ).logits[:, a.shape[1] :]+ return ((packed - alone).norm() / alone.norm()).item()+++class _StubTrainer:+ def __init__(+ self,+ model = None,+ args = None,+ **kwargs,+ ):+ self.model = model+ self.args = args+++def _sft_args(**overrides):+ args = SimpleNamespace(+ packing = False,+ padding_free = None,+ max_length = 64,+ eval_packing = None,+ )+ for key, value in overrides.items():+ setattr(args, key, value)+ return args+++def _build_sft(monkeypatch, model, args):+ for name in ("enable_padding_free_metadata", "enable_sample_packing"):+ monkeypatch.setattr(trainer_module, name, lambda model, trainer: None)+ module = SimpleNamespace(SFTTrainer = type("SFTTrainer", (_StubTrainer,), {}))+ trainer_module._patch_sft_trainer_auto_packing(module)+ module.SFTTrainer(model = model, args = args)+ return args+++def test_deepseek_v4_packed_row_leaks_and_dense_does_not():+ v4 = _tiny_or_skip("deepseek_v4")+ assert _second_document_error(v4) > 1e-2+ assert _second_document_error(_tiny("llama")) < 1e-5+++def test_sft_keeps_padding_free_off_for_deepseek_v4(monkeypatch, capsys):+ v4 = _tiny_or_skip("deepseek_v4")+ args = _build_sft(monkeypatch, v4, _sft_args())+ assert args.padding_free is False and args.packing is False+ assert "mixes tokens across packed documents" in capsys.readouterr().out+ args = _build_sft(monkeypatch, v4, _sft_args(padding_free = True, eval_packing = True))+ assert args.padding_free is False and args.eval_packing is False+ args = _build_sft(monkeypatch, v4, _sft_args(packing = True))+ assert args.packing is False and args.padding_free is False++[email protected]("model_type", ["llama", "qwen3"])+def test_sft_still_auto_enables_padding_free_for_dense(monkeypatch, capsys, model_type):+ model = _tiny_or_skip(model_type)+ args = _build_sft(monkeypatch, model, _sft_args())+ assert args.padding_free is True+ assert "mixes tokens across packed documents" not in capsys.readouterr().out+++ "name",+ [+ "DeepseekV4CSACompressor",+ "DeepseekV4HCACompressor",+ "ZayaCCAProjection",+ "RecurrentGemmaRglru",+ ],+)+def test_boundary_blocker_matches_modules_and_string_model_types(name):+ from unsloth.utils.packing import packing_boundary_blocker++ inner = type(name, (torch.nn.Module,), {})()+ model = torch.nn.Sequential(torch.nn.Linear(2, 2), torch.nn.Sequential(inner))+ assert packing_boundary_blocker(model) == name+ assert packing_boundary_blocker(torch.nn.Linear(2, 2)) is None+ assert packing_boundary_blocker("org/repo", ["deepseek_v4"]) == "deepseek_v4"+ assert packing_boundary_blocker("org/repo", ["deepseek_v3"]) is Nonediff --git a/unsloth/trainer.py b/unsloth/trainer.pyindex 0411aff9b5d..da7e41edaa7 100644--- a/unsloth/trainer.py+++ b/unsloth/trainer.py@@ -40,7 +40,11 @@ from unsloth.utils import (enable_padding_free_metadata,enable_sample_packing,)-from unsloth.utils.packing import _stateful_mixer_kind, patch_hybrid_linear_attention_varlen+from unsloth.utils.packing import (+ _stateful_mixer_kind,+ packing_boundary_blocker,+ patch_hybrid_linear_attention_varlen,+)from unsloth.utils.token_budget import install_token_budgetfrom unsloth_zoo.training_utils import (unsloth_train as _unsloth_train,@@ -1829,6 +1833,7 @@ def _patch_sft_trainer_auto_packing(trl_module):model = args[0] if len(args) >= 1 else kwargs.get("model")model_config = None+ model_types = ()is_vlm = Falseis_unsupported_model = Falseis_hybrid = False@@ -1905,6 +1910,7 @@ def _patch_sft_trainer_auto_packing(trl_module):is_ring_context_parallel = (getattr(config_arg, "context_parallel_size", 1) or 1) > 1 and (getattr(config_arg, "context_parallel_backend", None) or "ring") == "ring"+ boundary_blocker = packing_boundary_blocker(model, model_types)# Read before __init__: the compiled trainer ends it with model.for_inference(), which sets the flag.eval_logits_wanted = os.environ.get("UNSLOTH_RETURN_LOGITS", "0"@@ -1922,13 +1928,29 @@ def _patch_sft_trainer_auto_packing(trl_module):or forward_rejects_packing)requested_pack = bool(getattr(config_arg, "packing", False))+ if boundary_blocker is not None:+ if (+ not blocked+ and not requested_pack+ and (+ getattr(config_arg, "padding_free", None) is True+ or _should_auto_padding_free(config_arg)+ )+ ):+ print(+ f"Unsloth: Padding-free batching off: {boundary_blocker} mixes tokens across "+ "packed documents."+ )+ blocked = Trueif blocked:if hasattr(config_arg, "packing"):setattr(config_arg, "packing", False)if hasattr(config_arg, "padding_free"):setattr(config_arg, "padding_free", False)# TRL resolves eval_packing on its own; packed eval batches would hit ring attention's varlen refusal.- if is_ring_context_parallel and getattr(config_arg, "eval_packing", None):+ if (is_ring_context_parallel or boundary_blocker is not None) and getattr(+ config_arg, "eval_packing", None+ ):setattr(config_arg, "eval_packing", False)if blocked and requested_pack:@@ -1947,6 +1969,8 @@ def _patch_sft_trainer_auto_packing(trl_module):reason = f"unsupported model type(s): {', '.join(model_types)}"elif is_ring_context_parallel:reason = "ring context parallelism"+ elif boundary_blocker is not None:+ reason = f"{boundary_blocker} mixes tokens across packed documents"elif forward_rejects_packing:# Name the real blocker (for a string `model=`, the resolved class).blocker = (diff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.pyindex 75065fa4ea3..4e6e7c88c87 100644--- a/unsloth/utils/packing.py+++ b/unsloth/utils/packing.py@@ -473,6 +473,29 @@ def _stateful_mixer_kind(module) -> Optional[str]:return None+# Pool / recur over the whole row (V4 compressed entries numbered row-wide), ignoring position_ids.+_PACKING_BOUNDARY_BLIND_MODULES = (+ "DeepseekV4CSACompressor",+ "DeepseekV4HCACompressor",+ "ZayaCCAProjection",+ "RecurrentGemmaRglru",+)+_PACKING_BOUNDARY_BLIND_MODEL_TYPES = ("deepseek_v4", "zaya", "recurrent_gemma")+++def packing_boundary_blocker(model, model_types = ()) -> Optional[str]:+ modules = getattr(model, "modules", None)+ if callable(modules) and not isinstance(model, str):+ for module in modules():+ name = type(module).__name__+ if name in _PACKING_BOUNDARY_BLIND_MODULES:+ return name+ for model_type in model_types or ():+ if model_type in _PACKING_BOUNDARY_BLIND_MODEL_TYPES:+ return model_type+ return None++def _iter_stateful_mixers(model):"""(module, kind) for every stateful mixer, outermost first; submodules of a gated-deltamixer (its own short convs) are owned by that mixer's shim."""@@ -2146,4 +2169,5 @@ __all__ = ["mask_packed_sequence_boundaries","mask_packed_boundary_labels","clear_packed_caches",+ "packing_boundary_blocker",]