Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Compile the Laya encoder for decision model training (#12778)

Daniel Hancb7829375e90
diff --git a/tests/test_decision_model.py b/tests/test_decision_model.py
index c8348e664f3..07643d357a9 100644
--- a/tests/test_decision_model.py
+++ b/tests/test_decision_model.py
@@ -436,6 +436,124 @@ def test_train_calibrate_save_and_serve(checkpoint, tmp_path, lora):
assert all(torch.equal(weights[k], v.detach().cpu().half()) for k, v in expected.items())
+def test_long_runs_compile_the_encoder_layers_and_leave_them_eager(
+ checkpoint, tmp_path, monkeypatch
+):
+ from unsloth.models import _decision_fast
+
+ model, tokenizer = FastDecisionModel.from_pretrained(
+ str(checkpoint), use_gradient_checkpointing = False
+ )
+ model = FastDecisionModel.get_peft_model(model, r = 4, lora_alpha = 8)
+ items, _ = FastDecisionModel.build_dataset([_row(i) for i in range(8)], tokenizer, model)
+ compiled, static = [], []
+ monkeypatch.setattr(torch.nn.Module, "compile", lambda self, **kw: compiled.append((self, kw)))
+ monkeypatch.setattr(_decision_fast, "_compile_static", lambda fn: static.append(fn) or fn)
+ layers = _decision_fast._encoder_layers(model)
+ assert len(layers) == 2
+ trainer = DecisionTrainer(model = model, args = _args(tmp_path, max_steps = 1), train_dataset = items)
+ # Short runs and CPUs stay eager; UNSLOTH_DECISION_COMPILE=1 forces it, =0 refuses it.
+ monkeypatch.delenv("UNSLOTH_DECISION_COMPILE", raising = False)
+ trainer.train()
+ assert not compiled
+ assert _decision_fast._wants_compile(model, 10**6) == next(model.parameters()).is_cuda
+ monkeypatch.setattr(_decision_fast, "_on_gpu", lambda model: True)
+ assert _decision_fast._wants_compile(model, _decision_fast.COMPILE_MIN_FORWARDS)
+ assert not _decision_fast._wants_compile(model, _decision_fast.COMPILE_MIN_FORWARDS - 1)
+ monkeypatch.setenv("UNSLOTH_DECISION_COMPILE", "1")
+ trainer.train()
+ # The test decisions are short, so each layer compiles once for static 64-token buckets.
+ assert not compiled and len(static) == len(layers)
+ assert all(layer._compiled_call_impl is None for layer in layers)
+ monkeypatch.setattr(_decision_fast, "STATIC_MAX_LEN", 0)
+ trainer.train()
+ assert [m for m, _ in compiled] == layers and all(kw == {"dynamic": True} for _, kw in compiled)
+ assert all(layer._compiled_call_impl is None for layer in layers)
+ monkeypatch.setenv("UNSLOTH_DECISION_COMPILE", "0")
+ assert not _decision_fast._wants_compile(model, 10**6)
+
+
+def test_a_failing_compile_trains_eagerly(checkpoint, tmp_path, monkeypatch):
+ from unsloth.models import _decision_fast as fast
+
+ model, tokenizer = FastDecisionModel.from_pretrained(
+ str(checkpoint), use_gradient_checkpointing = False
+ )
+ model = FastDecisionModel.get_peft_model(model, r = 4, lora_alpha = 8)
+ items, _ = FastDecisionModel.build_dataset([_row(i) for i in range(8)], tokenizer, model)
+
+ def broken(*args, **kwargs):
+ raise RuntimeError("inductor cannot serve this platform")
+
+ def compile(self, **kwargs):
+ self._compiled_call_impl = broken
+
+ monkeypatch.setattr(torch.nn.Module, "compile", compile)
+ monkeypatch.setattr(fast, "_compile_static", lambda fn: broken)
+ monkeypatch.setenv("UNSLOTH_DECISION_COMPILE", "1")
+ trainer = DecisionTrainer(model = model, args = _args(tmp_path, max_steps = 2), train_dataset = items)
+ trainer.train()
+ assert trainer.state.global_step == 2
+ assert model._unsloth_decision_compiled is False
+ assert all(layer._compiled_call_impl is None for layer in fast._encoder_layers(model))
+ assert all(p.grad is None or torch.isfinite(p.grad).all() for p in model.parameters())
+
+
+def test_compiled_layers_checkpoint_with_torch_and_get_unsloths_back(
+ checkpoint, tmp_path, monkeypatch
+):
+ import functools
+
+ from unsloth.models import _decision_fast as fast
+
+ torch_checkpoint = fast._torch_checkpoint()
+ assert torch_checkpoint.__module__ == "torch.utils.checkpoint"
+
+ model, _ = FastDecisionModel.from_pretrained(str(checkpoint), use_gradient_checkpointing = False)
+ model = FastDecisionModel.get_peft_model(model, r = 4, lora_alpha = 8)
+ layers = fast._encoder_layers(model)
+
+ def unsloth_checkpoint(*args, **kwargs):
+ raise AssertionError("compiled layers must not use Unsloth's checkpoint")
+
+ offloaded = functools.partial(unsloth_checkpoint, use_reentrant = True)
+ # unsloth_zoo patches torch.utils.checkpoint.checkpoint once a model loads with its checkpointing.
+ monkeypatch.setattr(torch.utils.checkpoint, "_old_checkpoint", torch_checkpoint, raising = False)
+ monkeypatch.setattr(torch.utils.checkpoint, "checkpoint", unsloth_checkpoint)
+ for layer in layers:
+ layer._gradient_checkpointing_func = offloaded
+ monkeypatch.setattr(torch.nn.Module, "compile", lambda self, **kw: None)
+ monkeypatch.setattr(fast, "_wants_compile", lambda model, forwards: True)
+ monkeypatch.setattr(fast, "_warm_up", lambda model, amp_dtype: None)
+ with fast.compiled_encoder(model, 10**6) as compiled:
+ assert compiled
+ for layer in layers:
+ func = layer._gradient_checkpointing_func
+ assert func.func is torch_checkpoint and func.keywords == {"use_reentrant": False}
+ assert all(layer._gradient_checkpointing_func is offloaded for layer in layers)
+
+ def broken(model, amp_dtype):
+ raise RuntimeError("inductor cannot serve this platform")
+
+ monkeypatch.setattr(fast, "_warm_up", broken)
+ with fast.compiled_encoder(model, 10**6) as compiled:
+ assert not compiled
+ assert all(layer._gradient_checkpointing_func is offloaded for layer in layers)
+
+ # torch.compile itself can refuse (an unsupported Python): the second layer raises here.
+ def refuse(self, **kwargs):
+ if self is layers[1]:
+ raise RuntimeError("Dynamo is not supported on this Python")
+ self._compiled_call_impl = lambda *a, **k: None
+
+ monkeypatch.setattr(torch.nn.Module, "compile", refuse)
+ monkeypatch.setattr(fast, "_warm_up", lambda model, amp_dtype: None)
+ with fast.compiled_encoder(model, 10**6) as compiled:
+ assert not compiled
+ assert all(layer._compiled_call_impl is None for layer in layers)
+ assert all(layer._gradient_checkpointing_func is offloaded for layer in layers)
+
+
def test_a_resized_laya_vocabulary_saves_and_reloads(checkpoint, tmp_path):
model, tokenizer = FastDecisionModel.from_pretrained(
str(checkpoint), use_gradient_checkpointing = False
@@ -470,6 +588,44 @@ def test_full_clef_finetuning_never_quantizes_the_backbone(tmp_path, monkeypatch
assert seen[1].get("quantization_config") is not None
+def test_the_compile_warm_up_leaves_the_rng_where_it_was(checkpoint, monkeypatch):
+ from unsloth.models import _decision_fast as fast
+
+ model, _ = FastDecisionModel.from_pretrained(str(checkpoint), use_gradient_checkpointing = False)
+ monkeypatch.setattr(torch.nn.Module, "compile", lambda self, **kw: None)
+ monkeypatch.setattr(fast, "_wants_compile", lambda model, forwards: True)
+ torch.manual_seed(0)
+ before = torch.get_rng_state()
+ with fast.compiled_encoder(model, 10**6) as compiled:
+ assert compiled
+ assert torch.equal(torch.get_rng_state(), before)
+
+
+def test_the_compile_warm_up_forks_the_rng_of_the_models_own_accelerator(monkeypatch):
+ from unsloth.models import _decision_fast as fast
+
+ forked = []
+
+ class Forked(Exception):
+ pass
+
+ def fork_rng(**kwargs):
+ forked.append(kwargs)
+ raise Forked
+
+ model = types.SimpleNamespace(
+ training = True,
+ parameters = lambda: iter([torch.empty(1, device = "meta")]),
+ train = lambda mode = True: None,
+ encoder = types.SimpleNamespace(config = types.SimpleNamespace(vocab_size = 100)),
+ )
+ monkeypatch.setattr(torch.random, "fork_rng", fork_rng)
+ with pytest.raises(Forked):
+ fast._warm_up(model, None)
+ # An XPU / MPS model forks its own RNG, not CUDA's (meta stands in for any accelerator).
+ assert forked == [{"devices": [0], "device_type": "meta"}]
+
+
def test_clef_backbone_stays_on_one_device_unless_the_caller_places_it(tmp_path, monkeypatch):
from unsloth.models import loader, loader_utils
@@ -499,6 +655,8 @@ def test_toy_task_beats_the_base_model(checkpoint, tmp_path, monkeypatch):
# only after ~100 steps on a GPU (same curve otherwise, any precision), so 80 steps is a threshold
# tuned to CPU arithmetic, not a GPU defect.
monkeypatch.setattr(decision, "_device", lambda: torch.device("cpu"))
+ # Eager too: a compiled encoder draws the same accuracy from a different 45-decision RNG stream.
+ monkeypatch.setenv("UNSLOTH_DECISION_COMPILE", "0")
model, tokenizer = FastDecisionModel.from_pretrained(
str(checkpoint), use_gradient_checkpointing = False
)
@@ -1287,3 +1445,209 @@ def test_clef_save_keeps_the_source_tokenizer_files_byte_identical(clef_checkpoi
assert (out / name).read_bytes() == (clef_checkpoint / name).read_bytes(), name
# Without it transformers warns of an "incorrect regex pattern" when loading the tokenizer.
assert "transformers_version" in json.loads((out / "config.json").read_text())
+
+
+def test_decision_forward_never_picks_cudnn_attention(checkpoint, tmp_path):
+ model, tokenizer = FastDecisionModel.from_pretrained(
+ str(checkpoint), full_finetuning = True, use_gradient_checkpointing = False
+ )
+ items, _ = FastDecisionModel.build_dataset([_row(i) for i in range(4)], tokenizer, model)
+ cudnn = []
+ forward = model.forward
+
+ def spy(*args, **kwargs):
+ cudnn.append(torch.backends.cuda.cudnn_sdp_enabled())
+ return forward(*args, **kwargs)
+
+ model.forward = spy
+ trainer = DecisionTrainer(
+ model = model, args = _args(tmp_path), train_dataset = items, processing_class = tokenizer
+ )
+ device = next(model.parameters()).device
+ batch = {
+ k: v.to(device) for k, v in DecisionDataCollator(tokenizer.pad_token_id)(items).items()
+ }
+ trainer.compute_loss(model, batch)
+ decision._logits(model, items, tokenizer.pad_token_id)
+ assert cudnn and not any(cudnn)
+
+
+def test_checkpointed_recompute_never_picks_cudnn_attention(checkpoint, tmp_path):
+ # The recompute runs in backward: cuDNN there saved other tensors than the forward (CheckpointError).
+ model, tokenizer = FastDecisionModel.from_pretrained(
+ str(checkpoint), full_finetuning = True, use_gradient_checkpointing = True
+ )
+ items, _ = FastDecisionModel.build_dataset([_row(i) for i in range(4)], tokenizer, model)
+ layer = model.encoder.layers[0]
+ cudnn = []
+ forward = layer.forward
+
+ def spy(*args, **kwargs):
+ cudnn.append(torch.backends.cuda.cudnn_sdp_enabled())
+ return forward(*args, **kwargs)
+
+ layer.forward = spy
+ trainer = DecisionTrainer(
+ model = model, args = _args(tmp_path), train_dataset = items, processing_class = tokenizer
+ )
+ device = next(model.parameters()).device
+ batch = {
+ k: v.to(device) for k, v in DecisionDataCollator(tokenizer.pad_token_id)(items).items()
+ }
+ trainer.accelerator.backward(trainer.compute_loss(model, batch))
+ assert len(cudnn) == 2 and not any(cudnn)
+
+
+def test_logits_batch_similar_lengths_and_keep_the_callers_order(checkpoint):
+ model, tokenizer = FastDecisionModel.from_pretrained(
+ str(checkpoint), full_finetuning = True, use_gradient_checkpointing = False
+ )
+ items, _ = FastDecisionModel.build_dataset([_row(i) for i in range(4)], tokenizer, model)
+ assert len({len(item["input_ids"]) for item in items}) > 1
+ widths = []
+ forward = model.forward
+
+ def spy(*args, **kwargs):
+ widths.append(kwargs["input_ids"].shape[1])
+ return forward(*args, **kwargs)
+
+ model.forward = spy
+ batched = decision._logits(model, items, tokenizer.pad_token_id, batch_size = 3)
+ assert widths == sorted(widths)
+ model.forward = forward
+ for item, logits in zip(items, batched):
+ alone = decision._logits(model, [item], tokenizer.pad_token_id)[0]
+ assert logits.shape == (len(item["markers"]),)
+ torch.testing.assert_close(logits, alone, rtol = 2e-2, atol = 2e-2)
+
+
[email protected]("scaling", [1, 2])
+def test_lean_lora_forward_matches_peft(checkpoint, scaling):
+ model, _ = FastDecisionModel.from_pretrained(
+ str(checkpoint), dtype = torch.float32, use_gradient_checkpointing = False
+ )
+ model = FastDecisionModel.get_peft_model(model, r = 4, lora_alpha = 4 * scaling)
+ layers = [m for m in model.encoder.modules() if hasattr(m, "_unsloth_peft_forward")]
+ from peft.tuners.lora.layer import Linear
+
+ if Linear.forward.__name__ == "unsloth_forward":
+ # An earlier FastModel load in this process compiled PEFT's forward; the lean one stays out.
+ assert not layers
+ return
+ assert layers and all(m.forward.__func__ is decision._lean_lora_forward for m in layers)
+ layer = layers[0]
+ torch.manual_seed(0)
+ with torch.no_grad():
+ layer.lora_B["default"].weight.normal_()
+ device = layer.lora_A["default"].weight.device
+ autocast = [False] + ([True] if device.type == "cuda" else [])
+ for enabled in autocast:
+ x = torch.randn(2, 5, layer.in_features, device = device, requires_grad = True)
+ outputs = []
+ for forward in (layer.forward, layer._unsloth_peft_forward):
+ # The dtype the model trains in on this GPU: fp16 on a T4, which cannot run bf16.
+ with torch.autocast(device.type, dtype = decision._amp_dtype(device), enabled = enabled):
+ out = forward(x)
+ grads = torch.autograd.grad(
+ out.float().square().sum(), [x, layer.lora_A["default"].weight]
+ )
+ outputs.append((out, *grads))
+ for lean, peft in zip(*outputs):
+ assert lean.dtype == peft.dtype and torch.equal(lean, peft)
+ # Disabled or merged adapters take PEFT's own path.
+ with model.encoder.disable_adapter():
+ x = torch.randn(2, 5, layer.in_features, device = device)
+ torch.testing.assert_close(layer(x), layer.base_layer(x))
+
+
+def test_compiled_layers_use_plain_sdpa_and_put_the_attention_back(checkpoint):
+ from transformers.integrations.sdpa_attention import sdpa_attention_forward
+
+ from unsloth.models import _decision_fast
+
+ model, _ = FastDecisionModel.from_pretrained(
+ str(checkpoint), full_finetuning = True, use_gradient_checkpointing = False
+ )
+ config = model.encoder.config
+ assert config._attn_implementation == "sdpa"
+ with _decision_fast._compilable_attention(model, False):
+ assert config._attn_implementation == "sdpa"
+ with _decision_fast._compilable_attention(model, True):
+ assert config._attn_implementation == "unsloth_decision_sdpa"
+ assert config._attn_implementation == "sdpa"
+ q, k, v = (torch.randn(2, 4, 6, 8) for _ in range(3))
+ mask = torch.ones(2, 1, 6, 6, dtype = torch.bool)
+ mask[1, ..., 4:] = False
+ module = model.encoder.layers[0].attn
+ ours, _ = _decision_fast._encoder_sdpa(module, q, k, v, mask, scaling = 0.5)
+ reference, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 0.5)
+ torch.testing.assert_close(ours, reference)
+
+
+def test_static_length_padding_leaves_the_loss_unchanged(checkpoint, tmp_path):
+ from unsloth.models import _decision_fast
+
+ model, tokenizer = FastDecisionModel.from_pretrained(
+ str(checkpoint), full_finetuning = True, use_gradient_checkpointing = False
+ )
+ for module in model.modules():
+ if isinstance(module, torch.nn.Dropout):
+ module.p = 0.0
+ if isinstance(module, torch.nn.MultiheadAttention):
+ module.dropout = 0.0
+ items, _ = FastDecisionModel.build_dataset([_row(i) for i in range(4)], tokenizer, model)
+ trainer = DecisionTrainer(
+ model = model, args = _args(tmp_path), train_dataset = items, processing_class = tokenizer
+ )
+ device = next(model.parameters()).device
+ batch = DecisionDataCollator(tokenizer.pad_token_id)(items)
+ assert batch["input_ids"].shape[1] % _decision_fast.STATIC_MULTIPLE
+ losses = []
+ for multiple in (0, _decision_fast.STATIC_MULTIPLE):
+ model._unsloth_pad_multiple = multiple
+ inputs = {k: v.to(device) for k, v in batch.items()}
+ padded = _decision_fast.pad_length(model, dict(inputs))
+ assert padded["input_ids"].shape[1] % (multiple or 1) == 0
+ with torch.no_grad():
+ losses.append(trainer.compute_loss(model, inputs))
+ model._unsloth_pad_multiple = 0
+ torch.testing.assert_close(losses[0], losses[1])
+
+
+def test_static_compiled_layers_run_eagerly_outside_training():
+ from unsloth.models import _decision_fast
+
+ calls = []
+ layer = torch.nn.Linear(2, 2)
+ call = _decision_fast._training_only(
+ layer, lambda *a, **k: calls.append("compiled") or layer._call_impl(*a, **k)
+ )
+ x = torch.randn(1, 2)
+ call(x)
+ layer.eval()
+ call(x)
+ assert calls == ["compiled"]
+
+
[email protected]("max_length, static", [(634, True), (4096, False), (None, False)])
+def test_short_data_compiles_static_buckets_and_long_data_dynamic(
+ checkpoint, monkeypatch, max_length, static
+):
+ from unsloth.models import _decision_fast
+
+ monkeypatch.setenv("UNSLOTH_DECISION_COMPILE", "1")
+ monkeypatch.setattr(_decision_fast, "_warm_up", lambda model, amp_dtype: None)
+ model, _ = FastDecisionModel.from_pretrained(
+ str(checkpoint), full_finetuning = True, use_gradient_checkpointing = False
+ )
+ layers = _decision_fast._encoder_layers(model)
+ limit = torch._dynamo.config.recompile_limit
+ with _decision_fast.compiled_encoder(model, 10, None, max_length) as compiled:
+ assert compiled
+ assert model._unsloth_pad_multiple == (_decision_fast.STATIC_MULTIPLE if static else 0)
+ assert all(layer._compiled_call_impl is not None for layer in layers)
+ assert all((layer._compiled_call_impl.__name__ == "call") == static for layer in layers)
+ assert (torch._dynamo.config.recompile_limit > limit) == (static and limit < 28)
+ assert all(layer._compiled_call_impl is None for layer in layers)
+ assert model._unsloth_pad_multiple == 0
+ assert torch._dynamo.config.recompile_limit == limit
diff --git a/unsloth/models/_decision_fast.py b/unsloth/models/_decision_fast.py
new file mode 100644
index 00000000000..776386b09ae
--- /dev/null
+++ b/unsloth/models/_decision_fast.py
@@ -0,0 +1,223 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+# Laya's encoder is launch bound at decision sizes, so long runs compile each layer.
+# UNSLOTH_DECISION_COMPILE=0 or 1 turns it off or forces it on.
+
+import contextlib
+import functools
+import importlib.util
+import os
+from typing import Optional
+
+import torch
+import torch.nn as nn
+
+
+__all__ = ["compiled_encoder", "pad_length"]
+
+# Shorter runs spend more on a cold compile than it saves.
+COMPILE_MIN_FORWARDS = 4000
+# Static shapes launch far cheaper than dynamic ones on short decisions, one graph per 64-token bucket
+# (G4: full fine-tune 0.528 -> 0.482 s/step at 2 x 16). Past STATIC_MAX_LEN there are too many buckets
+# to compile, so long-context data keeps dynamic shapes.
+STATIC_MULTIPLE = 64
+STATIC_MAX_LEN = 1024
+
+
+def _encoder_sdpa(
+ module,
+ query,
+ key,
+ value,
+ attention_mask,
+ dropout = 0.0,
+ scaling = None,
+ **kwargs,
+):
+ # transformers' SDPA attention without unsloth_zoo's wrappers, which carry a __module__ that
+ # torch 2.11's dynamo cannot guard, so the compiled layers fell back to eager on Linux.
+ out = torch.nn.functional.scaled_dot_product_attention(
+ query, key, value, attn_mask = attention_mask, dropout_p = dropout, scale = scaling
+ )
+ return out.transpose(1, 2).contiguous(), None
+
+
+def _compilable_attention(model, enabled: bool):
+ config = model.encoder.config
+ original = config._attn_implementation
+ if enabled and original == "sdpa":
+ from transformers import AttentionInterface, AttentionMaskInterface
+ from transformers.masking_utils import sdpa_mask
+
+ AttentionInterface.register("unsloth_decision_sdpa", _encoder_sdpa)
+ AttentionMaskInterface.register("unsloth_decision_sdpa", sdpa_mask)
+ config._attn_implementation = "unsloth_decision_sdpa"
+ try:
+ yield
+ finally:
+ config._attn_implementation = original
+
+
+def _compile_static(function):
+ return torch.compile(function, dynamic = False)
+
+
+def _training_only(layer, compiled):
+ # Evaluation batches have arbitrary lengths, which static graphs would compile one by one.
+ def call(*args, **kwargs):
+ return compiled(*args, **kwargs) if layer.training else layer._call_impl(*args, **kwargs)
+
+ return call
+
+
+def _recompile_limit(limit: int):
+ config = torch._dynamo.config
+ name = "recompile_limit" if hasattr(config, "recompile_limit") else "cache_size_limit"
+ original = getattr(config, name)
+ setattr(config, name, max(original, limit))
+ try:
+ yield
+ finally:
+ setattr(config, name, original)
+
+
+def _torch_checkpoint():
+ # torch's own checkpoint, also once unsloth_zoo has patched torch.utils.checkpoint.checkpoint.
+ module = torch.utils.checkpoint
+ for func in (module.checkpoint, getattr(module, "_old_checkpoint", None)):
+ if getattr(func, "__module__", None) == module.__name__:
+ return func
+ return None
+
+
+def _encoder_layers(model):
+ if not isinstance(getattr(model, "head", None), nn.TransformerEncoder):
+ return []
+ encoder = getattr(model, "encoder", None)
+ if encoder is not None and hasattr(encoder, "get_base_model"):
+ encoder = encoder.get_base_model()
+ layers = getattr(encoder, "layers", None)
+ return list(layers) if isinstance(layers, nn.ModuleList) else []
+
+
+def _on_gpu(model) -> bool:
+ return next(model.parameters()).device.type == "cuda"
+
+
+def _wants_compile(model, forwards: int) -> bool:
+ choice = os.environ.get("UNSLOTH_DECISION_COMPILE", "auto")
+ if choice in ("0", "1"):
+ return choice == "1"
+ return (
+ _on_gpu(model)
+ and forwards >= COMPILE_MIN_FORWARDS
+ and importlib.util.find_spec("triton") is not None
+ )
+
+
+def _warm_up(model, amp_dtype) -> None:
+ # Fails here, not mid-run, on a platform Inductor cannot serve; RNG and gradients are restored.
+ device = next(model.parameters()).device
+ vocab = int(getattr(model.encoder.config, "vocab_size", 1000))
+ params = [p for p in model.parameters() if p.requires_grad]
+ grads = [p.grad for p in params]
+ training = model.training
+ accelerator = device.type != "cpu"
+ devices = [device.index or 0] if accelerator else []
+ try:
+ model.train()
+ with torch.random.fork_rng(
+ devices = devices, device_type = device.type if accelerator else "cuda"
+ ):
+ ids = torch.randint(5, min(vocab, 1000), (2, 64), device = device)
+ mask = torch.ones_like(ids)
+ mask[1, 48:] = 0
+ with torch.autocast(device.type, dtype = amp_dtype, enabled = amp_dtype is not None):
+ h = model.encoder(input_ids = ids, attention_mask = mask).last_hidden_state
+ if h.requires_grad:
+ h.float().sum().backward()
+ finally:
+ model.train(training)
+ for p, g in zip(params, grads):
+ p.grad = g
+
+
+def compiled_encoder(
+ model,
+ forwards: int,
+ amp_dtype = None,
+ max_length: Optional[int] = None,
+):
+ """Compile each Laya encoder layer in place for one training run, then go back to eager.
+
+ In place keeps parameter names, saving and LoRA merging unchanged.
+ """
+ layers = _encoder_layers(model)
+ if not (layers and _wants_compile(model, forwards)):
+ layers = []
+ static = bool(layers) and max_length is not None and max_length <= STATIC_MAX_LEN
+ # Every bucket compiles once with and once without grad (the KL reference forward), plus slack.
+ buckets = -(-max_length // STATIC_MULTIPLE) if static else 0
+ # Unsloth's reentrant offloaded checkpoint gives compiled bf16 layers wrong gradients (cosine 0.68
+ # to eager, 0.30 full fine-tune), so compiled runs checkpoint with torch's own.
+ swapped = {}
+
+ def restore():
+ for layer in layers:
+ layer._compiled_call_impl = None
+ for layer, func in swapped.items():
+ layer._gradient_checkpointing_func = func
+ model.__dict__["_unsloth_pad_multiple"] = 0
+
+ with _compilable_attention(model, bool(layers)), _recompile_limit(2 * buckets + 8):
+ if layers:
+ try:
+ torch_checkpoint = _torch_checkpoint()
+ if torch_checkpoint is None:
+ raise RuntimeError("torch's own checkpoint is not reachable")
+ for layer in layers:
+ func = getattr(layer, "_gradient_checkpointing_func", None)
+ if func is not None and getattr(func, "func", func) is not torch_checkpoint:
+ swapped[layer] = func
+ layer._gradient_checkpointing_func = functools.partial(
+ torch_checkpoint, use_reentrant = False
+ )
+ if static:
+ layer._compiled_call_impl = _training_only(
+ layer, _compile_static(layer._call_impl)
+ )
+ else:
+ layer.compile(dynamic = True)
+ model.__dict__["_unsloth_pad_multiple"] = STATIC_MULTIPLE if static else 0
+ _warm_up(model, amp_dtype)
+ except Exception as error:
+ restore()
+ torch._dynamo.reset()
+ print(
+ f"Unsloth: compiling the Laya encoder failed ({type(error).__name__}), training eagerly."
+ )
+ layers, swapped = [], {}
+ model.__dict__["_unsloth_decision_compiled"] = bool(layers)
+ try:
+ yield bool(layers)
+ finally:
+ restore()
+
+
+def pad_length(model, inputs: dict) -> dict:
+ # Pads a training batch to the static bucket; padded positions are masked keys and never
+ # gathered, so the loss is unchanged.
+ multiple = model.__dict__.get("_unsloth_pad_multiple", 0)
+ extra = -inputs["input_ids"].shape[1] % multiple if multiple and model.training else 0
+ if extra:
+ pad = model.encoder.config.pad_token_id or 0
+ inputs = {
+ **inputs,
+ "input_ids": torch.nn.functional.pad(inputs["input_ids"], (0, extra), value = pad),
+ "attention_mask": torch.nn.functional.pad(inputs["attention_mask"], (0, extra)),
+ }
+ return inputs
diff --git a/unsloth/models/decision.py b/unsloth/models/decision.py
index a5fece628b6..ffb02831787 100644
--- a/unsloth/models/decision.py
+++ b/unsloth/models/decision.py
@@ -31,6 +31,7 @@ from ._utils import (
is_bfloat16_supported,
)
from .loader_utils import is_distributed
+from ._decision_fast import compiled_encoder, pad_length
TRAIN_MAX_LEN, TRAIN_HEAD_MAX_LEN = 1024, 256
HOLDOUT_MAX = 400
@@ -208,6 +209,14 @@ def _amp_dtype(device):
return torch.bfloat16 if device.type == "xpu" else None
+def _no_cudnn_attention():
+ # cuDNN SDPA rebuilds its bf16 plan for every new sequence length, about 100x slower steps on a B200.
+ from torch.nn.attention import SDPBackend, sdpa_kernel
+ return sdpa_kernel(
+ [SDPBackend.FLASH_ATTENTION, SDPBackend.EFFICIENT_ATTENTION, SDPBackend.MATH]
+ )
+
+
def _gradient_checkpointing(model, use_gradient_checkpointing) -> None:
device = next(model.parameters()).device
# Unsloth's offloaded checkpointing needs an accelerator, and its re-entrant backward breaks DDP (#3713).
@@ -226,6 +235,45 @@ def _gradient_checkpointing(model, use_gradient_checkpointing) -> None:
model.encoder.gradient_checkpointing_disable()
+def _lean_lora_forward(self, x, *args, **kwargs):
+ adapter = self._unsloth_adapter
+ if (
+ self.disable_adapters
+ or self.merged
+ or args
+ or kwargs
+ or (
+ not torch.is_autocast_enabled(x.device.type)
+ and x.dtype != self.lora_A[adapter].weight.dtype
+ )
+ ):
+ return self._unsloth_peft_forward(x, *args, **kwargs)
+ # PEFT's maths without its per-call checks, its round trip of x through the fp32 adapter
+ # dtype, and the multiply when scaling is 1.
+ lora = self.lora_B[adapter](self.lora_A[adapter](x))
+ scaling = self.scaling[adapter]
+ return self.base_layer(x) + (lora if scaling == 1 else lora * scaling)
+
+
+def _lean_lora(encoder) -> None:
+ # Plain LoRA (one adapter, no dropout, DoRA or other variant) skips PEFT's per-call checks and casts.
+ from peft.tuners.lora.layer import Linear
+
+ # Unsloth's compiler (a FastModel load earlier in this process) already gave PEFT a compiled forward.
+ if Linear.forward.__name__ == "unsloth_forward":
+ return
+ for module in encoder.modules():
+ if type(module) is not Linear or len(module.lora_A) != 1 or module.lora_variant:
+ continue
+ adapter = next(iter(module.lora_A))
+ if not isinstance(module.lora_dropout[adapter], torch.nn.Identity):
+ continue
+ # Bound methods, so the deepcopy that merges for saving rebinds them to the copy.
+ module._unsloth_adapter = adapter
+ module._unsloth_peft_forward = module.forward
+ module.forward = types.MethodType(_lean_lora_forward, module)
+
+
def _parsed(value):
if isinstance(value, str) and value.strip()[:1] in ("{", "["):
try:
@@ -1015,6 +1063,14 @@ class DecisionTrainer(Trainer):
kwargs["data_collator"] = DecisionDataCollator(pad_token_id)
self.head_learning_rate = head_learning_rate
super().__init__(model = model, args = args, **kwargs)
+ backward = self.accelerator.backward
+
+ def _backward(loss, **backward_kwargs):
+ # Checkpointed layers rerun their forward here, so they need the forward's attention.
+ with _no_cudnn_attention():
+ return backward(loss, **backward_kwargs)
+
+ self.accelerator.backward = _backward
@contextlib.contextmanager
def _dataset_field_order(self):
@@ -1036,6 +1092,19 @@ class DecisionTrainer(Trainer):
with self._dataset_field_order():
return super().predict(*args, **kwargs)
+ def train(self, *args, **kwargs):
+ forwards = self.args.max_steps * self.args.gradient_accumulation_steps
+ if forwards <= 0 and self.train_dataset is not None:
+ batches = math.ceil(len(self.train_dataset) / self.args.train_batch_size)
+ forwards = int(batches * self.args.num_train_epochs)
+ amp_dtype = torch.bfloat16 if self.args.bf16 else torch.float16 if self.args.fp16 else None
+ try:
+ max_length = max(len(item["input_ids"]) for item in self.train_dataset)
+ except (TypeError, KeyError, ValueError):
+ max_length = None
+ with compiled_encoder(self.model, forwards, amp_dtype, max_length):
+ return super().train(*args, **kwargs)
+
def _get_train_sampler(self, train_dataset = None):
dataset = self.train_dataset if train_dataset is None else train_dataset
# With one micro-batch per step, length grouping would make each step one question type.
@@ -1054,7 +1123,9 @@ class DecisionTrainer(Trainer):
):
target = inputs.pop("target")
ordinal = inputs.pop("ordinal", None)
- logits, _ = model(**inputs)
+ inputs = pad_length(model, inputs)
+ with _no_cudnn_attention():
+ logits, _ = model(**inputs)
mask = inputs["marker_mask"]
loss = _decision_loss(
logits,
@@ -1067,7 +1138,7 @@ class DecisionTrainer(Trainer):
)
if self._reference_head is not None:
unwrapped = self.accelerator.unwrap_model(model)
- with torch.no_grad(), unwrapped.encoder.disable_adapter():
+ with torch.no_grad(), unwrapped.encoder.disable_adapter(), _no_cudnn_attention():
reference, _ = unwrapped(**inputs, head = self._reference_head)
log_ref = torch.log_softmax(reference.float().masked_fill(~mask, -1e4), -1)
log_p = torch.log_softmax(logits.float().masked_fill(~mask, -1e4), -1)
@@ -1114,15 +1185,20 @@ def _logits(
collate = DecisionDataCollator(pad_token_id)
was_training = model.training
model.eval()
- out = []
+ # Similar lengths share a batch, so little is padded; logits go back in the callers' order.
+ order = sorted(range(len(items)), key = lambda i: len(items[i]["input_ids"]))
+ out = [None] * len(items)
for start in range(0, len(items), batch_size):
- chunk = items[start : start + batch_size]
+ indices = order[start : start + batch_size]
+ chunk = [items[i] for i in indices]
batch = collate(chunk)
batch.pop("target")
- with torch.autocast(device.type, dtype = amp_dtype, enabled = amp_dtype is not None):
+ autocast = torch.autocast(device.type, dtype = amp_dtype, enabled = amp_dtype is not None)
+ with autocast, _no_cudnn_attention():
logits, _ = model(**{k: v.to(device) for k, v in batch.items()})
logits = logits.float().cpu()
- out.extend(logits[row, : len(item["markers"])] for row, item in enumerate(chunk))
+ for row, i in enumerate(indices):
+ out[i] = logits[row, : len(items[i]["markers"])]
model.train(was_training)
return out
@@ -1470,6 +1546,7 @@ class FastDecisionModel:
**kwargs,
),
)
+ _lean_lora(model.encoder)
_gradient_checkpointing(model, use_gradient_checkpointing)
return model