Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Never train with a KV cache, so padding-free rows keep their document boundaries (#13177)

Daniel Han323932b27266
diff --git a/tests/test_padding_free_train_use_cache.py b/tests/test_padding_free_train_use_cache.py
new file mode 100644
index 00000000000..ac901c2869c
--- /dev/null
+++ b/tests/test_padding_free_train_use_cache.py
@@ -0,0 +1,134 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+"""Training under auto padding-free must never build a KV cache: transformers then drops its packed-sequence
+mask and packed documents attend across each other. Trainer __init__ ends in for_inference (restoring
+use_cache), so for_training has to disable it again even without gradient checkpointing."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+torch = pytest.importorskip("torch")
+training_utils = pytest.importorskip("unsloth_zoo.training_utils")
+if not hasattr(training_utils, "disable_use_cache"):
+ pytest.skip("unsloth_zoo predates disable_use_cache", allow_module_level = True)
+
+
+def _tiny_llama():
+ from transformers import LlamaConfig, LlamaForCausalLM
+ config = LlamaConfig(
+ vocab_size = 64,
+ hidden_size = 16,
+ intermediate_size = 32,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ num_key_value_heads = 2,
+ use_cache = True,
+ )
+ return LlamaForCausalLM(config)
+
+
+def _fast_models():
+ from unsloth.models.llama import FastLlamaModel
+ from unsloth.models.vision import FastBaseModel
+ return [FastLlamaModel, FastBaseModel]
+
+
[email protected]("use_gradient_checkpointing", [False, True, "unsloth"])
[email protected]("fast_model", range(2))
+def test_for_training_disables_the_cache_with_or_without_checkpointing(
+ fast_model, use_gradient_checkpointing
+):
+ cls = _fast_models()[fast_model]
+ model = _tiny_llama()
+ # As in an Unsloth trainer: prepared, then for_inference at the end of __init__.
+ training_utils.disable_use_cache(model)
+ training_utils.restore_use_cache(model)
+ assert model.config.use_cache is True
+ try:
+ cls.for_training(model, use_gradient_checkpointing = use_gradient_checkpointing)
+ except Exception as error: # for_training arms checkpointing machinery some hosts lack
+ if model.config.use_cache is True:
+ pytest.skip(f"{cls.__name__}.for_training cannot run here: {error}")
+ assert model.config.use_cache is False
+
+
[email protected]("fast_model", range(2))
+def test_a_model_never_prepared_still_trains_without_a_cache_and_infers_with_one(fast_model):
+ cls = _fast_models()[fast_model]
+ model = _tiny_llama()
+ assert getattr(model, "_unsloth_use_cache_originals", None) is None
+ try:
+ cls.for_training(model, use_gradient_checkpointing = False)
+ except Exception as error:
+ if model.config.use_cache is True:
+ pytest.skip(f"{cls.__name__}.for_training cannot run here: {error}")
+ assert model.config.use_cache is False
+ training_utils.restore_use_cache(model)
+ assert model.config.use_cache is True
+
+
+class _Trainer:
+ args = SimpleNamespace(gradient_accumulation_steps = 1)
+ model_accepts_loss_kwargs = True
+
+ def __init__(self):
+ self.seen = []
+
+ def _old_compute_loss(self, model, inputs, *args, **kwargs):
+ self.seen.append(dict(inputs))
+ return torch.tensor(0.0)
+
+
+def _compute_loss(inputs, model = None):
+ from unsloth.models._utils import _unsloth_pre_compute_loss
+
+ trainer = _Trainer()
+ if model is None:
+ model = SimpleNamespace(config = SimpleNamespace(model_type = "llama"), training = True)
+ _unsloth_pre_compute_loss(trainer, model, inputs, num_items_in_batch = torch.tensor(4))
+ return trainer.seen[0]
+
+
+def test_a_packed_training_batch_runs_without_a_cache():
+ inputs = {
+ "input_ids": torch.zeros(1, 4, dtype = torch.long),
+ "packed_seq_lengths": torch.tensor([2, 2]),
+ }
+ assert _compute_loss(inputs)["use_cache"] is False
+ assert "use_cache" not in inputs, "the collated batch was mutated"
+
+
+def test_a_padded_training_batch_is_left_alone():
+ assert "use_cache" not in _compute_loss({"input_ids": torch.zeros(2, 4, dtype = torch.long)})
+
+
+def test_an_explicit_use_cache_in_the_batch_wins():
+ inputs = {
+ "input_ids": torch.zeros(1, 4, dtype = torch.long),
+ "packed_seq_lengths": torch.tensor([4]),
+ "use_cache": True,
+ }
+ assert _compute_loss(inputs)["use_cache"] is True
+
+
+def test_a_forward_without_use_cache_is_not_handed_one():
+ # Remote code may name packed_seq_lengths but neither use_cache nor **kwargs: the extra key would raise.
+ class _Explicit(torch.nn.Module):
+ config = SimpleNamespace(model_type = "llama")
+
+ def forward(
+ self,
+ input_ids = None,
+ labels = None,
+ packed_seq_lengths = None,
+ ):
+ return input_ids
+
+ inputs = {
+ "input_ids": torch.zeros(1, 4, dtype = torch.long),
+ "packed_seq_lengths": torch.tensor([2, 2]),
+ }
+ assert "use_cache" not in _compute_loss(inputs, _Explicit())
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index cda7c8a2316..ad32446e7d2 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -4794,6 +4794,19 @@ def test_mask_creation():
assert torch.all(correct_mask == our_mask)
+def _forward_takes_use_cache(model) -> bool:
+ """False for a forward naming packed_seq_lengths without use_cache / **kwargs (the key would raise)."""
+ takes = getattr(model, "_unsloth_forward_takes_use_cache", None)
+ if takes is None:
+ from unsloth.trainer import _forward_accepts_packing_kwargs
+ takes = _forward_accepts_packing_kwargs(model, "use_cache")
+ try:
+ model._unsloth_forward_takes_use_cache = takes
+ except Exception:
+ pass
+ return takes
+
+
def _unsloth_pre_compute_loss(self, model, inputs, *args, **kwargs):
num_items_in_batch = None
@@ -4804,6 +4817,14 @@ def _unsloth_pre_compute_loss(self, model, inputs, *args, **kwargs):
elif "num_items_in_batch" not in inputs:
inputs["num_items_in_batch"] = num_items_in_batch
+ # Training twin of the #3470 prediction_step guard: a KV cache drops the packed-sequence mask.
+ if (
+ "packed_seq_lengths" in inputs
+ and "use_cache" not in inputs
+ and _forward_takes_use_cache(model)
+ ):
+ inputs = {**inputs, "use_cache": False}
+
if (
num_items_in_batch is None
and getattr(getattr(self, "args", self), "gradient_accumulation_steps", 1) != 1
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 74e927b9e99..c56751907e6 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -4741,17 +4741,12 @@ class FastLlamaModel:
if hasattr(embeddings, "training"):
embeddings.training = True
- # Re-disable use_cache if prepare_model_for_training had disabled it and for_inference restored
- # it; the record only exists after a disable.
- if (
- use_gradient_checkpointing
- and getattr(model, "_unsloth_use_cache_originals", None) is not None
- ):
- try:
- from unsloth_zoo.training_utils import disable_use_cache
- disable_use_cache(model)
- except ImportError:
- pass
+ # Not only under GC: with a KV cache transformers drops the packed-sequence mask (padding-free rows leak).
+ try:
+ from unsloth_zoo.training_utils import disable_use_cache
+ disable_use_cache(model)
+ except ImportError:
+ pass
return model
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index b50c877b438..88133f5dae8 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -285,7 +285,9 @@ def PatchRL(FastLanguageModel):
# for_inference restores use_cache, and transformers drops its packed-sequence mask once a cache
# exists (masking_utils._preprocess_mask_arguments), so packed rows would attend across (#3470).
if "packed_seq_lengths" in inputs and "use_cache" not in inputs:
- inputs = {**inputs, "use_cache": False}
+ from unsloth.models._utils import _forward_takes_use_cache
+ if _forward_takes_use_cache(model):
+ inputs = {**inputs, "use_cache": False}
if ignore_keys is None:
if hasattr(self.model, "config"):
ignore_keys = getattr(self.model.config, "keys_to_ignore_at_inference", [])
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 756607997aa..c085f5acf48 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -5080,17 +5080,12 @@ class FastBaseModel:
embeddings = _embeddings_or_none(model, _getter)
if hasattr(embeddings, "training"):
embeddings.training = True
- # Re-disable use_cache if prepare_model_for_training had disabled it and for_inference restored it; the record only exists after a disable.
- if (
- use_gradient_checkpointing
- and getattr(model, "_unsloth_use_cache_originals", None) is not None
- ):
- # Auto-enable grouped-GEMM MoE (transformers<5 ModuleList experts); see llama.py.
- try:
- from unsloth_zoo.training_utils import disable_use_cache
- disable_use_cache(model)
- except ImportError:
- pass
+ # Not only under GC: with a KV cache transformers drops the packed-sequence mask (padding-free rows leak).
+ try:
+ from unsloth_zoo.training_utils import disable_use_cache
+ disable_use_cache(model)
+ except ImportError:
+ pass
os.environ["UNSLOTH_RETURN_LOGITS"] = "0"
if torch_compiler_set_stance is not None:
diff --git a/unsloth/trainer.py b/unsloth/trainer.py
index 6fa5e4e927d..a8f19947c0e 100644
--- a/unsloth/trainer.py
+++ b/unsloth/trainer.py
@@ -347,7 +347,7 @@ def _attribute_path(obj, path):
return obj
-def _forward_accepts_packing_kwargs(model) -> bool:
+def _forward_accepts_packing_kwargs(model, name = "packed_seq_lengths") -> bool:
"""Can this model's forward be handed the packing metadata at all?
Padding-free passes `packed_seq_lengths` down, and a forward naming neither it
@@ -406,10 +406,10 @@ def _forward_accepts_packing_kwargs(model) -> bool:
break
target = unwrapped
- return _forward_signature_accepts_packing(getattr(target, "forward", None))
+ return _forward_signature_accepts_packing(getattr(target, "forward", None), name)
-def _forward_signature_accepts_packing(forward) -> bool:
+def _forward_signature_accepts_packing(forward, name = "packed_seq_lengths") -> bool:
"""The signature question alone, so a class can be asked it as well as an instance.
Shared, not duplicated: a disagreement would turn padding-free on for one
@@ -423,7 +423,7 @@ def _forward_signature_accepts_packing(forward) -> bool:
return True
if any(p.kind is inspect.Parameter.VAR_KEYWORD for p in parameters.values()):
return True
- return "packed_seq_lengths" in parameters
+ return name in parameters
def _resolve_string_model_class(model_name, model_config, config_arg):