Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Never train with a KV cache, so padding-free rows keep their document boundaries (#13177)
Daniel Han323932b27266
diff --git a/tests/test_padding_free_train_use_cache.py b/tests/test_padding_free_train_use_cache.pynew file mode 100644index 00000000000..ac901c2869c--- /dev/null+++ b/tests/test_padding_free_train_use_cache.py@@ -0,0 +1,134 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.+"""Training under auto padding-free must never build a KV cache: transformers then drops its packed-sequence+mask and packed documents attend across each other. Trainer __init__ ends in for_inference (restoring+use_cache), so for_training has to disable it again even without gradient checkpointing."""++from __future__ import annotations++from types import SimpleNamespace++import pytest++torch = pytest.importorskip("torch")+training_utils = pytest.importorskip("unsloth_zoo.training_utils")+if not hasattr(training_utils, "disable_use_cache"):+ pytest.skip("unsloth_zoo predates disable_use_cache", allow_module_level = True)+++def _tiny_llama():+ from transformers import LlamaConfig, LlamaForCausalLM+ config = LlamaConfig(+ vocab_size = 64,+ hidden_size = 16,+ intermediate_size = 32,+ num_hidden_layers = 1,+ num_attention_heads = 2,+ num_key_value_heads = 2,+ use_cache = True,+ )+ return LlamaForCausalLM(config)+++def _fast_models():+ from unsloth.models.llama import FastLlamaModel+ from unsloth.models.vision import FastBaseModel+ return [FastLlamaModel, FastBaseModel]++[email protected]("use_gradient_checkpointing", [False, True, "unsloth"])[email protected]("fast_model", range(2))+def test_for_training_disables_the_cache_with_or_without_checkpointing(+ fast_model, use_gradient_checkpointing+):+ cls = _fast_models()[fast_model]+ model = _tiny_llama()+ # As in an Unsloth trainer: prepared, then for_inference at the end of __init__.+ training_utils.disable_use_cache(model)+ training_utils.restore_use_cache(model)+ assert model.config.use_cache is True+ try:+ cls.for_training(model, use_gradient_checkpointing = use_gradient_checkpointing)+ except Exception as error: # for_training arms checkpointing machinery some hosts lack+ if model.config.use_cache is True:+ pytest.skip(f"{cls.__name__}.for_training cannot run here: {error}")+ assert model.config.use_cache is False++[email protected]("fast_model", range(2))+def test_a_model_never_prepared_still_trains_without_a_cache_and_infers_with_one(fast_model):+ cls = _fast_models()[fast_model]+ model = _tiny_llama()+ assert getattr(model, "_unsloth_use_cache_originals", None) is None+ try:+ cls.for_training(model, use_gradient_checkpointing = False)+ except Exception as error:+ if model.config.use_cache is True:+ pytest.skip(f"{cls.__name__}.for_training cannot run here: {error}")+ assert model.config.use_cache is False+ training_utils.restore_use_cache(model)+ assert model.config.use_cache is True+++class _Trainer:+ args = SimpleNamespace(gradient_accumulation_steps = 1)+ model_accepts_loss_kwargs = True++ def __init__(self):+ self.seen = []++ def _old_compute_loss(self, model, inputs, *args, **kwargs):+ self.seen.append(dict(inputs))+ return torch.tensor(0.0)+++def _compute_loss(inputs, model = None):+ from unsloth.models._utils import _unsloth_pre_compute_loss++ trainer = _Trainer()+ if model is None:+ model = SimpleNamespace(config = SimpleNamespace(model_type = "llama"), training = True)+ _unsloth_pre_compute_loss(trainer, model, inputs, num_items_in_batch = torch.tensor(4))+ return trainer.seen[0]+++def test_a_packed_training_batch_runs_without_a_cache():+ inputs = {+ "input_ids": torch.zeros(1, 4, dtype = torch.long),+ "packed_seq_lengths": torch.tensor([2, 2]),+ }+ assert _compute_loss(inputs)["use_cache"] is False+ assert "use_cache" not in inputs, "the collated batch was mutated"+++def test_a_padded_training_batch_is_left_alone():+ assert "use_cache" not in _compute_loss({"input_ids": torch.zeros(2, 4, dtype = torch.long)})+++def test_an_explicit_use_cache_in_the_batch_wins():+ inputs = {+ "input_ids": torch.zeros(1, 4, dtype = torch.long),+ "packed_seq_lengths": torch.tensor([4]),+ "use_cache": True,+ }+ assert _compute_loss(inputs)["use_cache"] is True+++def test_a_forward_without_use_cache_is_not_handed_one():+ # Remote code may name packed_seq_lengths but neither use_cache nor **kwargs: the extra key would raise.+ class _Explicit(torch.nn.Module):+ config = SimpleNamespace(model_type = "llama")++ def forward(+ self,+ input_ids = None,+ labels = None,+ packed_seq_lengths = None,+ ):+ return input_ids++ inputs = {+ "input_ids": torch.zeros(1, 4, dtype = torch.long),+ "packed_seq_lengths": torch.tensor([2, 2]),+ }+ assert "use_cache" not in _compute_loss(inputs, _Explicit())diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex cda7c8a2316..ad32446e7d2 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -4794,6 +4794,19 @@ def test_mask_creation():assert torch.all(correct_mask == our_mask)+def _forward_takes_use_cache(model) -> bool:+ """False for a forward naming packed_seq_lengths without use_cache / **kwargs (the key would raise)."""+ takes = getattr(model, "_unsloth_forward_takes_use_cache", None)+ if takes is None:+ from unsloth.trainer import _forward_accepts_packing_kwargs+ takes = _forward_accepts_packing_kwargs(model, "use_cache")+ try:+ model._unsloth_forward_takes_use_cache = takes+ except Exception:+ pass+ return takes++def _unsloth_pre_compute_loss(self, model, inputs, *args, **kwargs):num_items_in_batch = None@@ -4804,6 +4817,14 @@ def _unsloth_pre_compute_loss(self, model, inputs, *args, **kwargs):elif "num_items_in_batch" not in inputs:inputs["num_items_in_batch"] = num_items_in_batch+ # Training twin of the #3470 prediction_step guard: a KV cache drops the packed-sequence mask.+ if (+ "packed_seq_lengths" in inputs+ and "use_cache" not in inputs+ and _forward_takes_use_cache(model)+ ):+ inputs = {**inputs, "use_cache": False}+if (num_items_in_batch is Noneand getattr(getattr(self, "args", self), "gradient_accumulation_steps", 1) != 1diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 74e927b9e99..c56751907e6 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -4741,17 +4741,12 @@ class FastLlamaModel:if hasattr(embeddings, "training"):embeddings.training = True- # Re-disable use_cache if prepare_model_for_training had disabled it and for_inference restored- # it; the record only exists after a disable.- if (- use_gradient_checkpointing- and getattr(model, "_unsloth_use_cache_originals", None) is not None- ):- try:- from unsloth_zoo.training_utils import disable_use_cache- disable_use_cache(model)- except ImportError:- pass+ # Not only under GC: with a KV cache transformers drops the packed-sequence mask (padding-free rows leak).+ try:+ from unsloth_zoo.training_utils import disable_use_cache+ disable_use_cache(model)+ except ImportError:+ passreturn modeldiff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex b50c877b438..88133f5dae8 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -285,7 +285,9 @@ def PatchRL(FastLanguageModel):# for_inference restores use_cache, and transformers drops its packed-sequence mask once a cache# exists (masking_utils._preprocess_mask_arguments), so packed rows would attend across (#3470).if "packed_seq_lengths" in inputs and "use_cache" not in inputs:- inputs = {**inputs, "use_cache": False}+ from unsloth.models._utils import _forward_takes_use_cache+ if _forward_takes_use_cache(model):+ inputs = {**inputs, "use_cache": False}if ignore_keys is None:if hasattr(self.model, "config"):ignore_keys = getattr(self.model.config, "keys_to_ignore_at_inference", [])diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 756607997aa..c085f5acf48 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -5080,17 +5080,12 @@ class FastBaseModel:embeddings = _embeddings_or_none(model, _getter)if hasattr(embeddings, "training"):embeddings.training = True- # Re-disable use_cache if prepare_model_for_training had disabled it and for_inference restored it; the record only exists after a disable.- if (- use_gradient_checkpointing- and getattr(model, "_unsloth_use_cache_originals", None) is not None- ):- # Auto-enable grouped-GEMM MoE (transformers<5 ModuleList experts); see llama.py.- try:- from unsloth_zoo.training_utils import disable_use_cache- disable_use_cache(model)- except ImportError:- pass+ # Not only under GC: with a KV cache transformers drops the packed-sequence mask (padding-free rows leak).+ try:+ from unsloth_zoo.training_utils import disable_use_cache+ disable_use_cache(model)+ except ImportError:+ passos.environ["UNSLOTH_RETURN_LOGITS"] = "0"if torch_compiler_set_stance is not None:diff --git a/unsloth/trainer.py b/unsloth/trainer.pyindex 6fa5e4e927d..a8f19947c0e 100644--- a/unsloth/trainer.py+++ b/unsloth/trainer.py@@ -347,7 +347,7 @@ def _attribute_path(obj, path):return obj-def _forward_accepts_packing_kwargs(model) -> bool:+def _forward_accepts_packing_kwargs(model, name = "packed_seq_lengths") -> bool:"""Can this model's forward be handed the packing metadata at all?Padding-free passes `packed_seq_lengths` down, and a forward naming neither it@@ -406,10 +406,10 @@ def _forward_accepts_packing_kwargs(model) -> bool:breaktarget = unwrapped- return _forward_signature_accepts_packing(getattr(target, "forward", None))+ return _forward_signature_accepts_packing(getattr(target, "forward", None), name)-def _forward_signature_accepts_packing(forward) -> bool:+def _forward_signature_accepts_packing(forward, name = "packed_seq_lengths") -> bool:"""The signature question alone, so a class can be asked it as well as an instance.Shared, not duplicated: a disagreement would turn padding-free on for one@@ -423,7 +423,7 @@ def _forward_signature_accepts_packing(forward) -> bool:return Trueif any(p.kind is inspect.Parameter.VAR_KEYWORD for p in parameters.values()):return True- return "packed_seq_lengths" in parameters+ return name in parametersdef _resolve_string_model_class(model_name, model_config, config_arg):