Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Compile the Laya encoder for decision model training (#12778)
Daniel Hancb7829375e90
diff --git a/tests/test_decision_model.py b/tests/test_decision_model.pyindex c8348e664f3..07643d357a9 100644--- a/tests/test_decision_model.py+++ b/tests/test_decision_model.py@@ -436,6 +436,124 @@ def test_train_calibrate_save_and_serve(checkpoint, tmp_path, lora):assert all(torch.equal(weights[k], v.detach().cpu().half()) for k, v in expected.items())+def test_long_runs_compile_the_encoder_layers_and_leave_them_eager(+ checkpoint, tmp_path, monkeypatch+):+ from unsloth.models import _decision_fast++ model, tokenizer = FastDecisionModel.from_pretrained(+ str(checkpoint), use_gradient_checkpointing = False+ )+ model = FastDecisionModel.get_peft_model(model, r = 4, lora_alpha = 8)+ items, _ = FastDecisionModel.build_dataset([_row(i) for i in range(8)], tokenizer, model)+ compiled, static = [], []+ monkeypatch.setattr(torch.nn.Module, "compile", lambda self, **kw: compiled.append((self, kw)))+ monkeypatch.setattr(_decision_fast, "_compile_static", lambda fn: static.append(fn) or fn)+ layers = _decision_fast._encoder_layers(model)+ assert len(layers) == 2+ trainer = DecisionTrainer(model = model, args = _args(tmp_path, max_steps = 1), train_dataset = items)+ # Short runs and CPUs stay eager; UNSLOTH_DECISION_COMPILE=1 forces it, =0 refuses it.+ monkeypatch.delenv("UNSLOTH_DECISION_COMPILE", raising = False)+ trainer.train()+ assert not compiled+ assert _decision_fast._wants_compile(model, 10**6) == next(model.parameters()).is_cuda+ monkeypatch.setattr(_decision_fast, "_on_gpu", lambda model: True)+ assert _decision_fast._wants_compile(model, _decision_fast.COMPILE_MIN_FORWARDS)+ assert not _decision_fast._wants_compile(model, _decision_fast.COMPILE_MIN_FORWARDS - 1)+ monkeypatch.setenv("UNSLOTH_DECISION_COMPILE", "1")+ trainer.train()+ # The test decisions are short, so each layer compiles once for static 64-token buckets.+ assert not compiled and len(static) == len(layers)+ assert all(layer._compiled_call_impl is None for layer in layers)+ monkeypatch.setattr(_decision_fast, "STATIC_MAX_LEN", 0)+ trainer.train()+ assert [m for m, _ in compiled] == layers and all(kw == {"dynamic": True} for _, kw in compiled)+ assert all(layer._compiled_call_impl is None for layer in layers)+ monkeypatch.setenv("UNSLOTH_DECISION_COMPILE", "0")+ assert not _decision_fast._wants_compile(model, 10**6)+++def test_a_failing_compile_trains_eagerly(checkpoint, tmp_path, monkeypatch):+ from unsloth.models import _decision_fast as fast++ model, tokenizer = FastDecisionModel.from_pretrained(+ str(checkpoint), use_gradient_checkpointing = False+ )+ model = FastDecisionModel.get_peft_model(model, r = 4, lora_alpha = 8)+ items, _ = FastDecisionModel.build_dataset([_row(i) for i in range(8)], tokenizer, model)++ def broken(*args, **kwargs):+ raise RuntimeError("inductor cannot serve this platform")++ def compile(self, **kwargs):+ self._compiled_call_impl = broken++ monkeypatch.setattr(torch.nn.Module, "compile", compile)+ monkeypatch.setattr(fast, "_compile_static", lambda fn: broken)+ monkeypatch.setenv("UNSLOTH_DECISION_COMPILE", "1")+ trainer = DecisionTrainer(model = model, args = _args(tmp_path, max_steps = 2), train_dataset = items)+ trainer.train()+ assert trainer.state.global_step == 2+ assert model._unsloth_decision_compiled is False+ assert all(layer._compiled_call_impl is None for layer in fast._encoder_layers(model))+ assert all(p.grad is None or torch.isfinite(p.grad).all() for p in model.parameters())+++def test_compiled_layers_checkpoint_with_torch_and_get_unsloths_back(+ checkpoint, tmp_path, monkeypatch+):+ import functools++ from unsloth.models import _decision_fast as fast++ torch_checkpoint = fast._torch_checkpoint()+ assert torch_checkpoint.__module__ == "torch.utils.checkpoint"++ model, _ = FastDecisionModel.from_pretrained(str(checkpoint), use_gradient_checkpointing = False)+ model = FastDecisionModel.get_peft_model(model, r = 4, lora_alpha = 8)+ layers = fast._encoder_layers(model)++ def unsloth_checkpoint(*args, **kwargs):+ raise AssertionError("compiled layers must not use Unsloth's checkpoint")++ offloaded = functools.partial(unsloth_checkpoint, use_reentrant = True)+ # unsloth_zoo patches torch.utils.checkpoint.checkpoint once a model loads with its checkpointing.+ monkeypatch.setattr(torch.utils.checkpoint, "_old_checkpoint", torch_checkpoint, raising = False)+ monkeypatch.setattr(torch.utils.checkpoint, "checkpoint", unsloth_checkpoint)+ for layer in layers:+ layer._gradient_checkpointing_func = offloaded+ monkeypatch.setattr(torch.nn.Module, "compile", lambda self, **kw: None)+ monkeypatch.setattr(fast, "_wants_compile", lambda model, forwards: True)+ monkeypatch.setattr(fast, "_warm_up", lambda model, amp_dtype: None)+ with fast.compiled_encoder(model, 10**6) as compiled:+ assert compiled+ for layer in layers:+ func = layer._gradient_checkpointing_func+ assert func.func is torch_checkpoint and func.keywords == {"use_reentrant": False}+ assert all(layer._gradient_checkpointing_func is offloaded for layer in layers)++ def broken(model, amp_dtype):+ raise RuntimeError("inductor cannot serve this platform")++ monkeypatch.setattr(fast, "_warm_up", broken)+ with fast.compiled_encoder(model, 10**6) as compiled:+ assert not compiled+ assert all(layer._gradient_checkpointing_func is offloaded for layer in layers)++ # torch.compile itself can refuse (an unsupported Python): the second layer raises here.+ def refuse(self, **kwargs):+ if self is layers[1]:+ raise RuntimeError("Dynamo is not supported on this Python")+ self._compiled_call_impl = lambda *a, **k: None++ monkeypatch.setattr(torch.nn.Module, "compile", refuse)+ monkeypatch.setattr(fast, "_warm_up", lambda model, amp_dtype: None)+ with fast.compiled_encoder(model, 10**6) as compiled:+ assert not compiled+ assert all(layer._compiled_call_impl is None for layer in layers)+ assert all(layer._gradient_checkpointing_func is offloaded for layer in layers)++def test_a_resized_laya_vocabulary_saves_and_reloads(checkpoint, tmp_path):model, tokenizer = FastDecisionModel.from_pretrained(str(checkpoint), use_gradient_checkpointing = False@@ -470,6 +588,44 @@ def test_full_clef_finetuning_never_quantizes_the_backbone(tmp_path, monkeypatchassert seen[1].get("quantization_config") is not None+def test_the_compile_warm_up_leaves_the_rng_where_it_was(checkpoint, monkeypatch):+ from unsloth.models import _decision_fast as fast++ model, _ = FastDecisionModel.from_pretrained(str(checkpoint), use_gradient_checkpointing = False)+ monkeypatch.setattr(torch.nn.Module, "compile", lambda self, **kw: None)+ monkeypatch.setattr(fast, "_wants_compile", lambda model, forwards: True)+ torch.manual_seed(0)+ before = torch.get_rng_state()+ with fast.compiled_encoder(model, 10**6) as compiled:+ assert compiled+ assert torch.equal(torch.get_rng_state(), before)+++def test_the_compile_warm_up_forks_the_rng_of_the_models_own_accelerator(monkeypatch):+ from unsloth.models import _decision_fast as fast++ forked = []++ class Forked(Exception):+ pass++ def fork_rng(**kwargs):+ forked.append(kwargs)+ raise Forked++ model = types.SimpleNamespace(+ training = True,+ parameters = lambda: iter([torch.empty(1, device = "meta")]),+ train = lambda mode = True: None,+ encoder = types.SimpleNamespace(config = types.SimpleNamespace(vocab_size = 100)),+ )+ monkeypatch.setattr(torch.random, "fork_rng", fork_rng)+ with pytest.raises(Forked):+ fast._warm_up(model, None)+ # An XPU / MPS model forks its own RNG, not CUDA's (meta stands in for any accelerator).+ assert forked == [{"devices": [0], "device_type": "meta"}]++def test_clef_backbone_stays_on_one_device_unless_the_caller_places_it(tmp_path, monkeypatch):from unsloth.models import loader, loader_utils@@ -499,6 +655,8 @@ def test_toy_task_beats_the_base_model(checkpoint, tmp_path, monkeypatch):# only after ~100 steps on a GPU (same curve otherwise, any precision), so 80 steps is a threshold# tuned to CPU arithmetic, not a GPU defect.monkeypatch.setattr(decision, "_device", lambda: torch.device("cpu"))+ # Eager too: a compiled encoder draws the same accuracy from a different 45-decision RNG stream.+ monkeypatch.setenv("UNSLOTH_DECISION_COMPILE", "0")model, tokenizer = FastDecisionModel.from_pretrained(str(checkpoint), use_gradient_checkpointing = False)@@ -1287,3 +1445,209 @@ def test_clef_save_keeps_the_source_tokenizer_files_byte_identical(clef_checkpoiassert (out / name).read_bytes() == (clef_checkpoint / name).read_bytes(), name# Without it transformers warns of an "incorrect regex pattern" when loading the tokenizer.assert "transformers_version" in json.loads((out / "config.json").read_text())+++def test_decision_forward_never_picks_cudnn_attention(checkpoint, tmp_path):+ model, tokenizer = FastDecisionModel.from_pretrained(+ str(checkpoint), full_finetuning = True, use_gradient_checkpointing = False+ )+ items, _ = FastDecisionModel.build_dataset([_row(i) for i in range(4)], tokenizer, model)+ cudnn = []+ forward = model.forward++ def spy(*args, **kwargs):+ cudnn.append(torch.backends.cuda.cudnn_sdp_enabled())+ return forward(*args, **kwargs)++ model.forward = spy+ trainer = DecisionTrainer(+ model = model, args = _args(tmp_path), train_dataset = items, processing_class = tokenizer+ )+ device = next(model.parameters()).device+ batch = {+ k: v.to(device) for k, v in DecisionDataCollator(tokenizer.pad_token_id)(items).items()+ }+ trainer.compute_loss(model, batch)+ decision._logits(model, items, tokenizer.pad_token_id)+ assert cudnn and not any(cudnn)+++def test_checkpointed_recompute_never_picks_cudnn_attention(checkpoint, tmp_path):+ # The recompute runs in backward: cuDNN there saved other tensors than the forward (CheckpointError).+ model, tokenizer = FastDecisionModel.from_pretrained(+ str(checkpoint), full_finetuning = True, use_gradient_checkpointing = True+ )+ items, _ = FastDecisionModel.build_dataset([_row(i) for i in range(4)], tokenizer, model)+ layer = model.encoder.layers[0]+ cudnn = []+ forward = layer.forward++ def spy(*args, **kwargs):+ cudnn.append(torch.backends.cuda.cudnn_sdp_enabled())+ return forward(*args, **kwargs)++ layer.forward = spy+ trainer = DecisionTrainer(+ model = model, args = _args(tmp_path), train_dataset = items, processing_class = tokenizer+ )+ device = next(model.parameters()).device+ batch = {+ k: v.to(device) for k, v in DecisionDataCollator(tokenizer.pad_token_id)(items).items()+ }+ trainer.accelerator.backward(trainer.compute_loss(model, batch))+ assert len(cudnn) == 2 and not any(cudnn)+++def test_logits_batch_similar_lengths_and_keep_the_callers_order(checkpoint):+ model, tokenizer = FastDecisionModel.from_pretrained(+ str(checkpoint), full_finetuning = True, use_gradient_checkpointing = False+ )+ items, _ = FastDecisionModel.build_dataset([_row(i) for i in range(4)], tokenizer, model)+ assert len({len(item["input_ids"]) for item in items}) > 1+ widths = []+ forward = model.forward++ def spy(*args, **kwargs):+ widths.append(kwargs["input_ids"].shape[1])+ return forward(*args, **kwargs)++ model.forward = spy+ batched = decision._logits(model, items, tokenizer.pad_token_id, batch_size = 3)+ assert widths == sorted(widths)+ model.forward = forward+ for item, logits in zip(items, batched):+ alone = decision._logits(model, [item], tokenizer.pad_token_id)[0]+ assert logits.shape == (len(item["markers"]),)+ torch.testing.assert_close(logits, alone, rtol = 2e-2, atol = 2e-2)++[email protected]("scaling", [1, 2])+def test_lean_lora_forward_matches_peft(checkpoint, scaling):+ model, _ = FastDecisionModel.from_pretrained(+ str(checkpoint), dtype = torch.float32, use_gradient_checkpointing = False+ )+ model = FastDecisionModel.get_peft_model(model, r = 4, lora_alpha = 4 * scaling)+ layers = [m for m in model.encoder.modules() if hasattr(m, "_unsloth_peft_forward")]+ from peft.tuners.lora.layer import Linear++ if Linear.forward.__name__ == "unsloth_forward":+ # An earlier FastModel load in this process compiled PEFT's forward; the lean one stays out.+ assert not layers+ return+ assert layers and all(m.forward.__func__ is decision._lean_lora_forward for m in layers)+ layer = layers[0]+ torch.manual_seed(0)+ with torch.no_grad():+ layer.lora_B["default"].weight.normal_()+ device = layer.lora_A["default"].weight.device+ autocast = [False] + ([True] if device.type == "cuda" else [])+ for enabled in autocast:+ x = torch.randn(2, 5, layer.in_features, device = device, requires_grad = True)+ outputs = []+ for forward in (layer.forward, layer._unsloth_peft_forward):+ # The dtype the model trains in on this GPU: fp16 on a T4, which cannot run bf16.+ with torch.autocast(device.type, dtype = decision._amp_dtype(device), enabled = enabled):+ out = forward(x)+ grads = torch.autograd.grad(+ out.float().square().sum(), [x, layer.lora_A["default"].weight]+ )+ outputs.append((out, *grads))+ for lean, peft in zip(*outputs):+ assert lean.dtype == peft.dtype and torch.equal(lean, peft)+ # Disabled or merged adapters take PEFT's own path.+ with model.encoder.disable_adapter():+ x = torch.randn(2, 5, layer.in_features, device = device)+ torch.testing.assert_close(layer(x), layer.base_layer(x))+++def test_compiled_layers_use_plain_sdpa_and_put_the_attention_back(checkpoint):+ from transformers.integrations.sdpa_attention import sdpa_attention_forward++ from unsloth.models import _decision_fast++ model, _ = FastDecisionModel.from_pretrained(+ str(checkpoint), full_finetuning = True, use_gradient_checkpointing = False+ )+ config = model.encoder.config+ assert config._attn_implementation == "sdpa"+ with _decision_fast._compilable_attention(model, False):+ assert config._attn_implementation == "sdpa"+ with _decision_fast._compilable_attention(model, True):+ assert config._attn_implementation == "unsloth_decision_sdpa"+ assert config._attn_implementation == "sdpa"+ q, k, v = (torch.randn(2, 4, 6, 8) for _ in range(3))+ mask = torch.ones(2, 1, 6, 6, dtype = torch.bool)+ mask[1, ..., 4:] = False+ module = model.encoder.layers[0].attn+ ours, _ = _decision_fast._encoder_sdpa(module, q, k, v, mask, scaling = 0.5)+ reference, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 0.5)+ torch.testing.assert_close(ours, reference)+++def test_static_length_padding_leaves_the_loss_unchanged(checkpoint, tmp_path):+ from unsloth.models import _decision_fast++ model, tokenizer = FastDecisionModel.from_pretrained(+ str(checkpoint), full_finetuning = True, use_gradient_checkpointing = False+ )+ for module in model.modules():+ if isinstance(module, torch.nn.Dropout):+ module.p = 0.0+ if isinstance(module, torch.nn.MultiheadAttention):+ module.dropout = 0.0+ items, _ = FastDecisionModel.build_dataset([_row(i) for i in range(4)], tokenizer, model)+ trainer = DecisionTrainer(+ model = model, args = _args(tmp_path), train_dataset = items, processing_class = tokenizer+ )+ device = next(model.parameters()).device+ batch = DecisionDataCollator(tokenizer.pad_token_id)(items)+ assert batch["input_ids"].shape[1] % _decision_fast.STATIC_MULTIPLE+ losses = []+ for multiple in (0, _decision_fast.STATIC_MULTIPLE):+ model._unsloth_pad_multiple = multiple+ inputs = {k: v.to(device) for k, v in batch.items()}+ padded = _decision_fast.pad_length(model, dict(inputs))+ assert padded["input_ids"].shape[1] % (multiple or 1) == 0+ with torch.no_grad():+ losses.append(trainer.compute_loss(model, inputs))+ model._unsloth_pad_multiple = 0+ torch.testing.assert_close(losses[0], losses[1])+++def test_static_compiled_layers_run_eagerly_outside_training():+ from unsloth.models import _decision_fast++ calls = []+ layer = torch.nn.Linear(2, 2)+ call = _decision_fast._training_only(+ layer, lambda *a, **k: calls.append("compiled") or layer._call_impl(*a, **k)+ )+ x = torch.randn(1, 2)+ call(x)+ layer.eval()+ call(x)+ assert calls == ["compiled"]++[email protected]("max_length, static", [(634, True), (4096, False), (None, False)])+def test_short_data_compiles_static_buckets_and_long_data_dynamic(+ checkpoint, monkeypatch, max_length, static+):+ from unsloth.models import _decision_fast++ monkeypatch.setenv("UNSLOTH_DECISION_COMPILE", "1")+ monkeypatch.setattr(_decision_fast, "_warm_up", lambda model, amp_dtype: None)+ model, _ = FastDecisionModel.from_pretrained(+ str(checkpoint), full_finetuning = True, use_gradient_checkpointing = False+ )+ layers = _decision_fast._encoder_layers(model)+ limit = torch._dynamo.config.recompile_limit+ with _decision_fast.compiled_encoder(model, 10, None, max_length) as compiled:+ assert compiled+ assert model._unsloth_pad_multiple == (_decision_fast.STATIC_MULTIPLE if static else 0)+ assert all(layer._compiled_call_impl is not None for layer in layers)+ assert all((layer._compiled_call_impl.__name__ == "call") == static for layer in layers)+ assert (torch._dynamo.config.recompile_limit > limit) == (static and limit < 28)+ assert all(layer._compiled_call_impl is None for layer in layers)+ assert model._unsloth_pad_multiple == 0+ assert torch._dynamo.config.recompile_limit == limitdiff --git a/unsloth/models/_decision_fast.py b/unsloth/models/_decision_fast.pynew file mode 100644index 00000000000..776386b09ae--- /dev/null+++ b/unsloth/models/_decision_fast.py@@ -0,0 +1,223 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++# Laya's encoder is launch bound at decision sizes, so long runs compile each layer.+# UNSLOTH_DECISION_COMPILE=0 or 1 turns it off or forces it on.++import contextlib+import functools+import importlib.util+import os+from typing import Optional++import torch+import torch.nn as nn+++__all__ = ["compiled_encoder", "pad_length"]++# Shorter runs spend more on a cold compile than it saves.+COMPILE_MIN_FORWARDS = 4000+# Static shapes launch far cheaper than dynamic ones on short decisions, one graph per 64-token bucket+# (G4: full fine-tune 0.528 -> 0.482 s/step at 2 x 16). Past STATIC_MAX_LEN there are too many buckets+# to compile, so long-context data keeps dynamic shapes.+STATIC_MULTIPLE = 64+STATIC_MAX_LEN = 1024+++def _encoder_sdpa(+ module,+ query,+ key,+ value,+ attention_mask,+ dropout = 0.0,+ scaling = None,+ **kwargs,+):+ # transformers' SDPA attention without unsloth_zoo's wrappers, which carry a __module__ that+ # torch 2.11's dynamo cannot guard, so the compiled layers fell back to eager on Linux.+ out = torch.nn.functional.scaled_dot_product_attention(+ query, key, value, attn_mask = attention_mask, dropout_p = dropout, scale = scaling+ )+ return out.transpose(1, 2).contiguous(), None+++def _compilable_attention(model, enabled: bool):+ config = model.encoder.config+ original = config._attn_implementation+ if enabled and original == "sdpa":+ from transformers import AttentionInterface, AttentionMaskInterface+ from transformers.masking_utils import sdpa_mask++ AttentionInterface.register("unsloth_decision_sdpa", _encoder_sdpa)+ AttentionMaskInterface.register("unsloth_decision_sdpa", sdpa_mask)+ config._attn_implementation = "unsloth_decision_sdpa"+ try:+ yield+ finally:+ config._attn_implementation = original+++def _compile_static(function):+ return torch.compile(function, dynamic = False)+++def _training_only(layer, compiled):+ # Evaluation batches have arbitrary lengths, which static graphs would compile one by one.+ def call(*args, **kwargs):+ return compiled(*args, **kwargs) if layer.training else layer._call_impl(*args, **kwargs)++ return call+++def _recompile_limit(limit: int):+ config = torch._dynamo.config+ name = "recompile_limit" if hasattr(config, "recompile_limit") else "cache_size_limit"+ original = getattr(config, name)+ setattr(config, name, max(original, limit))+ try:+ yield+ finally:+ setattr(config, name, original)+++def _torch_checkpoint():+ # torch's own checkpoint, also once unsloth_zoo has patched torch.utils.checkpoint.checkpoint.+ module = torch.utils.checkpoint+ for func in (module.checkpoint, getattr(module, "_old_checkpoint", None)):+ if getattr(func, "__module__", None) == module.__name__:+ return func+ return None+++def _encoder_layers(model):+ if not isinstance(getattr(model, "head", None), nn.TransformerEncoder):+ return []+ encoder = getattr(model, "encoder", None)+ if encoder is not None and hasattr(encoder, "get_base_model"):+ encoder = encoder.get_base_model()+ layers = getattr(encoder, "layers", None)+ return list(layers) if isinstance(layers, nn.ModuleList) else []+++def _on_gpu(model) -> bool:+ return next(model.parameters()).device.type == "cuda"+++def _wants_compile(model, forwards: int) -> bool:+ choice = os.environ.get("UNSLOTH_DECISION_COMPILE", "auto")+ if choice in ("0", "1"):+ return choice == "1"+ return (+ _on_gpu(model)+ and forwards >= COMPILE_MIN_FORWARDS+ and importlib.util.find_spec("triton") is not None+ )+++def _warm_up(model, amp_dtype) -> None:+ # Fails here, not mid-run, on a platform Inductor cannot serve; RNG and gradients are restored.+ device = next(model.parameters()).device+ vocab = int(getattr(model.encoder.config, "vocab_size", 1000))+ params = [p for p in model.parameters() if p.requires_grad]+ grads = [p.grad for p in params]+ training = model.training+ accelerator = device.type != "cpu"+ devices = [device.index or 0] if accelerator else []+ try:+ model.train()+ with torch.random.fork_rng(+ devices = devices, device_type = device.type if accelerator else "cuda"+ ):+ ids = torch.randint(5, min(vocab, 1000), (2, 64), device = device)+ mask = torch.ones_like(ids)+ mask[1, 48:] = 0+ with torch.autocast(device.type, dtype = amp_dtype, enabled = amp_dtype is not None):+ h = model.encoder(input_ids = ids, attention_mask = mask).last_hidden_state+ if h.requires_grad:+ h.float().sum().backward()+ finally:+ model.train(training)+ for p, g in zip(params, grads):+ p.grad = g+++def compiled_encoder(+ model,+ forwards: int,+ amp_dtype = None,+ max_length: Optional[int] = None,+):+ """Compile each Laya encoder layer in place for one training run, then go back to eager.++ In place keeps parameter names, saving and LoRA merging unchanged.+ """+ layers = _encoder_layers(model)+ if not (layers and _wants_compile(model, forwards)):+ layers = []+ static = bool(layers) and max_length is not None and max_length <= STATIC_MAX_LEN+ # Every bucket compiles once with and once without grad (the KL reference forward), plus slack.+ buckets = -(-max_length // STATIC_MULTIPLE) if static else 0+ # Unsloth's reentrant offloaded checkpoint gives compiled bf16 layers wrong gradients (cosine 0.68+ # to eager, 0.30 full fine-tune), so compiled runs checkpoint with torch's own.+ swapped = {}++ def restore():+ for layer in layers:+ layer._compiled_call_impl = None+ for layer, func in swapped.items():+ layer._gradient_checkpointing_func = func+ model.__dict__["_unsloth_pad_multiple"] = 0++ with _compilable_attention(model, bool(layers)), _recompile_limit(2 * buckets + 8):+ if layers:+ try:+ torch_checkpoint = _torch_checkpoint()+ if torch_checkpoint is None:+ raise RuntimeError("torch's own checkpoint is not reachable")+ for layer in layers:+ func = getattr(layer, "_gradient_checkpointing_func", None)+ if func is not None and getattr(func, "func", func) is not torch_checkpoint:+ swapped[layer] = func+ layer._gradient_checkpointing_func = functools.partial(+ torch_checkpoint, use_reentrant = False+ )+ if static:+ layer._compiled_call_impl = _training_only(+ layer, _compile_static(layer._call_impl)+ )+ else:+ layer.compile(dynamic = True)+ model.__dict__["_unsloth_pad_multiple"] = STATIC_MULTIPLE if static else 0+ _warm_up(model, amp_dtype)+ except Exception as error:+ restore()+ torch._dynamo.reset()+ print(+ f"Unsloth: compiling the Laya encoder failed ({type(error).__name__}), training eagerly."+ )+ layers, swapped = [], {}+ model.__dict__["_unsloth_decision_compiled"] = bool(layers)+ try:+ yield bool(layers)+ finally:+ restore()+++def pad_length(model, inputs: dict) -> dict:+ # Pads a training batch to the static bucket; padded positions are masked keys and never+ # gathered, so the loss is unchanged.+ multiple = model.__dict__.get("_unsloth_pad_multiple", 0)+ extra = -inputs["input_ids"].shape[1] % multiple if multiple and model.training else 0+ if extra:+ pad = model.encoder.config.pad_token_id or 0+ inputs = {+ **inputs,+ "input_ids": torch.nn.functional.pad(inputs["input_ids"], (0, extra), value = pad),+ "attention_mask": torch.nn.functional.pad(inputs["attention_mask"], (0, extra)),+ }+ return inputsdiff --git a/unsloth/models/decision.py b/unsloth/models/decision.pyindex a5fece628b6..ffb02831787 100644--- a/unsloth/models/decision.py+++ b/unsloth/models/decision.py@@ -31,6 +31,7 @@ from ._utils import (is_bfloat16_supported,)from .loader_utils import is_distributed+from ._decision_fast import compiled_encoder, pad_lengthTRAIN_MAX_LEN, TRAIN_HEAD_MAX_LEN = 1024, 256HOLDOUT_MAX = 400@@ -208,6 +209,14 @@ def _amp_dtype(device):return torch.bfloat16 if device.type == "xpu" else None+def _no_cudnn_attention():+ # cuDNN SDPA rebuilds its bf16 plan for every new sequence length, about 100x slower steps on a B200.+ from torch.nn.attention import SDPBackend, sdpa_kernel+ return sdpa_kernel(+ [SDPBackend.FLASH_ATTENTION, SDPBackend.EFFICIENT_ATTENTION, SDPBackend.MATH]+ )++def _gradient_checkpointing(model, use_gradient_checkpointing) -> None:device = next(model.parameters()).device# Unsloth's offloaded checkpointing needs an accelerator, and its re-entrant backward breaks DDP (#3713).@@ -226,6 +235,45 @@ def _gradient_checkpointing(model, use_gradient_checkpointing) -> None:model.encoder.gradient_checkpointing_disable()+def _lean_lora_forward(self, x, *args, **kwargs):+ adapter = self._unsloth_adapter+ if (+ self.disable_adapters+ or self.merged+ or args+ or kwargs+ or (+ not torch.is_autocast_enabled(x.device.type)+ and x.dtype != self.lora_A[adapter].weight.dtype+ )+ ):+ return self._unsloth_peft_forward(x, *args, **kwargs)+ # PEFT's maths without its per-call checks, its round trip of x through the fp32 adapter+ # dtype, and the multiply when scaling is 1.+ lora = self.lora_B[adapter](self.lora_A[adapter](x))+ scaling = self.scaling[adapter]+ return self.base_layer(x) + (lora if scaling == 1 else lora * scaling)+++def _lean_lora(encoder) -> None:+ # Plain LoRA (one adapter, no dropout, DoRA or other variant) skips PEFT's per-call checks and casts.+ from peft.tuners.lora.layer import Linear++ # Unsloth's compiler (a FastModel load earlier in this process) already gave PEFT a compiled forward.+ if Linear.forward.__name__ == "unsloth_forward":+ return+ for module in encoder.modules():+ if type(module) is not Linear or len(module.lora_A) != 1 or module.lora_variant:+ continue+ adapter = next(iter(module.lora_A))+ if not isinstance(module.lora_dropout[adapter], torch.nn.Identity):+ continue+ # Bound methods, so the deepcopy that merges for saving rebinds them to the copy.+ module._unsloth_adapter = adapter+ module._unsloth_peft_forward = module.forward+ module.forward = types.MethodType(_lean_lora_forward, module)++def _parsed(value):if isinstance(value, str) and value.strip()[:1] in ("{", "["):try:@@ -1015,6 +1063,14 @@ class DecisionTrainer(Trainer):kwargs["data_collator"] = DecisionDataCollator(pad_token_id)self.head_learning_rate = head_learning_ratesuper().__init__(model = model, args = args, **kwargs)+ backward = self.accelerator.backward++ def _backward(loss, **backward_kwargs):+ # Checkpointed layers rerun their forward here, so they need the forward's attention.+ with _no_cudnn_attention():+ return backward(loss, **backward_kwargs)++ self.accelerator.backward = _backward@contextlib.contextmanagerdef _dataset_field_order(self):@@ -1036,6 +1092,19 @@ class DecisionTrainer(Trainer):with self._dataset_field_order():return super().predict(*args, **kwargs)+ def train(self, *args, **kwargs):+ forwards = self.args.max_steps * self.args.gradient_accumulation_steps+ if forwards <= 0 and self.train_dataset is not None:+ batches = math.ceil(len(self.train_dataset) / self.args.train_batch_size)+ forwards = int(batches * self.args.num_train_epochs)+ amp_dtype = torch.bfloat16 if self.args.bf16 else torch.float16 if self.args.fp16 else None+ try:+ max_length = max(len(item["input_ids"]) for item in self.train_dataset)+ except (TypeError, KeyError, ValueError):+ max_length = None+ with compiled_encoder(self.model, forwards, amp_dtype, max_length):+ return super().train(*args, **kwargs)+def _get_train_sampler(self, train_dataset = None):dataset = self.train_dataset if train_dataset is None else train_dataset# With one micro-batch per step, length grouping would make each step one question type.@@ -1054,7 +1123,9 @@ class DecisionTrainer(Trainer):):target = inputs.pop("target")ordinal = inputs.pop("ordinal", None)- logits, _ = model(**inputs)+ inputs = pad_length(model, inputs)+ with _no_cudnn_attention():+ logits, _ = model(**inputs)mask = inputs["marker_mask"]loss = _decision_loss(logits,@@ -1067,7 +1138,7 @@ class DecisionTrainer(Trainer):)if self._reference_head is not None:unwrapped = self.accelerator.unwrap_model(model)- with torch.no_grad(), unwrapped.encoder.disable_adapter():+ with torch.no_grad(), unwrapped.encoder.disable_adapter(), _no_cudnn_attention():reference, _ = unwrapped(**inputs, head = self._reference_head)log_ref = torch.log_softmax(reference.float().masked_fill(~mask, -1e4), -1)log_p = torch.log_softmax(logits.float().masked_fill(~mask, -1e4), -1)@@ -1114,15 +1185,20 @@ def _logits(collate = DecisionDataCollator(pad_token_id)was_training = model.trainingmodel.eval()- out = []+ # Similar lengths share a batch, so little is padded; logits go back in the callers' order.+ order = sorted(range(len(items)), key = lambda i: len(items[i]["input_ids"]))+ out = [None] * len(items)for start in range(0, len(items), batch_size):- chunk = items[start : start + batch_size]+ indices = order[start : start + batch_size]+ chunk = [items[i] for i in indices]batch = collate(chunk)batch.pop("target")- with torch.autocast(device.type, dtype = amp_dtype, enabled = amp_dtype is not None):+ autocast = torch.autocast(device.type, dtype = amp_dtype, enabled = amp_dtype is not None)+ with autocast, _no_cudnn_attention():logits, _ = model(**{k: v.to(device) for k, v in batch.items()})logits = logits.float().cpu()- out.extend(logits[row, : len(item["markers"])] for row, item in enumerate(chunk))+ for row, i in enumerate(indices):+ out[i] = logits[row, : len(items[i]["markers"])]model.train(was_training)return out@@ -1470,6 +1546,7 @@ class FastDecisionModel:**kwargs,),)+ _lean_lora(model.encoder)_gradient_checkpointing(model, use_gradient_checkpointing)return model