Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
import copy
import json
import math
import sys
import types
import warnings
from pathlib import Path
import pytest
from real_accelerator import has_real_cuda
torch = pytest.importorskip("torch")
import unsloth
# The torch backend: on Apple Silicon unsloth serves decision models from decision_mlx instead.
if unsloth._IS_MLX:
pytest.skip("the torch backend", allow_module_level = True)
from transformers import TrainerCallback, TrainingArguments
from unsloth import DecisionTrainer, FastDecisionModel
from unsloth.models import _decision_common, decision
from unsloth.models.decision import (
DecisionDataCollator,
DecisionDataError,
_LengthGroupedBatches,
_laya,
_target,
)
# torch < 2.3 has only the CUDA scaler.
GRAD_SCALER = getattr(torch.amp, "GradScaler", None) or torch.cuda.amp.GradScaler
laya = _laya()
# Every word the states, questions and options use, so the toy task below is learnable.
WORDS = (
"the server is down again refund my card charge twice please help now noul choice score "
"question : ? , does this need a reply which team should handle it how upset customer outage "
"service billing charges and refunds level 0 1 2 calm annoyed angry false no statement not hold "
"true yes holds"
).split()
QUESTIONS = {
"urgent": {"type": "noul", "instructions": "Does this need a reply now?"},
"team": {
"type": "choice",
"instructions": "Which team should handle it?",
"criteria": {"outage": "service down", "billing": "charges and refunds"},
},
"mood": {
"type": "score",
"instructions": "How upset is the customer?",
"criteria": ["calm", "annoyed", "angry"],
},
}
def _row(i):
outage = i % 2 == 0
return {
"state": "the server is down again help now" if outage else "refund my card charge twice",
"questions": QUESTIONS,
"gold": {
"urgent": {"label": "true" if outage else "false"},
"team": {"label": "outage" if outage else "billing"},
"mood": 2 if outage else 0,
},
}
def _args(tmp_path, **overrides):
return TrainingArguments(
**{
"output_dir": str(tmp_path / "run"),
"per_device_train_batch_size": 8,
"max_steps": 4,
"learning_rate": 1e-3,
"report_to": "none",
"save_strategy": "no",
**overrides,
}
)
def _tiny_encoder_config(**rope):
from transformers import ModernBertConfig
config = ModernBertConfig(
vocab_size = 64,
hidden_size = 64,
intermediate_size = 96,
num_hidden_layers = 2,
num_attention_heads = 4,
pad_token_id = 0,
cls_token_id = 2,
sep_token_id = 1,
global_attn_every_n_layers = 2,
local_attention = 16,
).to_dict()
for key in ("rope_parameters", "global_rope_theta", "local_rope_theta", "rope_theta"):
config.pop(key, None)
return {**config, **rope}
@pytest.fixture
def checkpoint(tmp_path):
from safetensors.torch import save_file
from tokenizers import Tokenizer, models, normalizers, pre_tokenizers
from transformers import PreTrainedTokenizerFast
specials = ["[PAD]", "[SEP]", "[CLS]", "[UNK]", "[MASK]"]
vocab = {token: i for i, token in enumerate(specials + sorted(set(WORDS)))}
tokenizer = Tokenizer(models.WordLevel(vocab, unk_token = "[UNK]"))
tokenizer.normalizer = normalizers.Lowercase()
tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()
PreTrainedTokenizerFast(
tokenizer_object = tokenizer,
pad_token = "[PAD]",
sep_token = "[SEP]",
cls_token = "[CLS]",
unk_token = "[UNK]",
mask_token = "[MASK]",
).save_pretrained(str(tmp_path / "base" / "tokenizer"))
(tmp_path / "base" / "encoder").mkdir()
# The layout transformers 5 writes, as the published Laya checkpoints have it.
(tmp_path / "base" / "encoder" / "config.json").write_text(
json.dumps(
_tiny_encoder_config(
rope_parameters = {
"full_attention": {"rope_theta": 160000.0, "rope_type": "default"},
"sliding_attention": {"rope_theta": 160000.0, "rope_type": "default"},
}
),
indent = 2,
)
)
config = {
"encoder": "tiny",
"head_layers": 1,
"act_costs": {"escalate": 0.5},
"max_len": 96,
"head_max_len": 48,
"temperature": [1.2, 1.1, 1.3],
"temperature_by_options": {"noul:2": 0.7, "score:3-5": 0.9},
"training": {"updates": 1000},
}
torch.manual_seed(0)
model = laya.common.build_model(config, encoder_dir = str(tmp_path / "base" / "encoder"))
save_file(
{k: v.half().contiguous() for k, v in model.state_dict().items()},
str(tmp_path / "base" / "model.safetensors"),
)
(tmp_path / "base" / "rl_agent_config.json").write_text(json.dumps(config))
return tmp_path / "base"
@pytest.fixture
def fresh_laya():
decision._laya.cache_clear()
yield
decision._laya.cache_clear()
def test_laya_is_the_copy_vendored_with_unsloth(monkeypatch, fresh_laya):
# A pip installed laya (0.3.9+ has no agent or common attributes) is ignored and left in place.
pip_laya = types.ModuleType("laya")
monkeypatch.setitem(sys.modules, "laya", pip_laya)
vendored = decision._laya()
assert vendored is not pip_laya and sys.modules["laya"] is pip_laya
assert Path(vendored.__file__).resolve() == decision._VENDORED_LAYA
assert vendored.__version__ == "0.3.5" and hasattr(vendored.agent.Agent, "_to_internal")
# Studio registers the same files as "laya" before training, and that module is the one used.
studio_laya = types.ModuleType("laya")
studio_laya.__file__ = str(decision._VENDORED_LAYA)
monkeypatch.setitem(sys.modules, "laya", studio_laya)
decision._laya.cache_clear()
assert decision._laya() is studio_laya
def test_missing_vendored_laya_is_a_clear_error(monkeypatch, tmp_path, fresh_laya):
monkeypatch.setattr(_decision_common, "_VENDORED_LAYA", tmp_path / "laya" / "__init__.py")
with pytest.raises(ImportError, match = "Unsloth: decision models need laya"):
decision._laya()
def test_gold_labels_and_probabilities():
noul, team, mood = (
laya.agent.Agent._to_internal(QUESTIONS[n]) for n in ("urgent", "team", "mood")
)
assert _target(noul, {"probabilities": {"true": 0.3}}) == ([pytest.approx(0.7), 0.3], 0)
assert _target(noul, {"noul": 0.8}) == ([pytest.approx(0.2), 0.8], 1)
assert _target(team, {"probabilities": {"outage": 3, "billing": 1}}) == ([0.75, 0.25], 0)
for label in (True, "TRUE", "True", " true"):
assert _target(noul, label) == ([0.0, 1.0], 1)
for label in (2, 2.0, "2", "2.0"):
assert _target(mood, {"label": label}) == ([0.0, 0.0, 1.0], 2)
unusable = [
(mood, {"label": 2.5}),
(mood, {"label": True}),
(mood, math.inf),
(mood, {"label": math.nan}),
(noul, 1),
(noul, {"noul": math.inf}),
(noul, {"probabilities": {"true": math.inf}}),
(team, {"probabilities": {"outage": math.nan, "billing": 1}}),
]
for internal, gold in unusable:
with pytest.raises(DecisionDataError):
_target(internal, gold)
def test_temperature_fit_finds_the_temperature_of_the_soft_gold():
# softmax([2, 0] / T) is [0.6, 0.4] at T = 2 / ln(1.5).
logits = [torch.tensor([2.0, 0.0])] * 10
items = [{"target": [0.6, 0.4]}] * 10
assert decision._fit_temperature(logits, items) == pytest.approx(2 / math.log(1.5), rel = 1e-2)
def test_dataset_skips_bad_rows_and_names_the_most_common_reason(checkpoint):
model, tokenizer = FastDecisionModel.from_pretrained(
str(checkpoint), use_gradient_checkpointing = False
)
assert model.decision_config["max_len"] == 1024
assert model.decision_config["head_max_len"] == 256
assert "training" not in model.decision_config
nested = {**QUESTIONS, "team": {**QUESTIONS["team"], "criteria": [["outage"], ["billing"]]}}
numbered = {**QUESTIONS, "team": {**QUESTIONS["team"], "criteria": [1, 2]}}
rows = [
{**_row(0), "gold": {"urgent": "true", "team": "outage"}},
*[{**_row(i), "questions": nested} for i in range(1, 4)],
{**_row(4), "questions": numbered},
"not a row",
{**_row(6), "gold": {**_row(6)["gold"], "urgent": {"noul": math.inf}}},
{**_row(7), "gold": {**_row(7)["gold"], "mood": 1.5}},
{"state": "x", "questions": "{}", "gold": {}},
*[_row(i) for i in range(9, 30)],
]
items, report = FastDecisionModel.build_dataset(rows, tokenizer, model)
assert report["total"] == 3 * 28 + 1 and report["skipped"] == 8
assert len(items) == report["total"] - report["skipped"]
assert (
report["reason"] == 'row 2: "team" needs criteria naming its options (and 3 more like it)'
)
batch = DecisionDataCollator(tokenizer.pad_token_id)(items[-3:])
assert batch["marker_mask"].sum(-1).tolist() == [2, 2, 3]
assert torch.allclose(batch["target"].sum(-1), torch.ones(3))
@pytest.mark.parametrize("clef", [False, True])
def test_rows_problem_needs_no_model_and_stops_at_the_first_usable_row(clef):
assert FastDecisionModel.rows_problem([], clef) == ""
no_gold = {**_row(0), "gold": {}}
no_questions = {"state": "only a state"}
assert FastDecisionModel.rows_problem([no_questions], clef) == (
"row 1 needs state, questions and gold"
)
assert FastDecisionModel.rows_problem([no_questions, no_questions], clef) == (
"row 1 needs state, questions and gold (and 1 more like it)"
)
assert FastDecisionModel.rows_problem([no_gold], clef).startswith('row 1: "')
def record(name, question):
seen.append(name)
seen = []
assert FastDecisionModel.rows_problem([no_gold, _row(1), no_questions], clef, record) is None
assert len(seen) == 1
def test_rows_problem_applies_the_callers_validation():
def refuse(name, question):
raise ValueError("is over the limit")
problem = FastDecisionModel.rows_problem([_row(0)], False, refuse)
assert problem.endswith("is over the limit (and 2 more like it)")
def test_holdout_takes_whole_rows_and_never_overshoots():
items = [{"row": row} for row in range(50) for _ in range(3)]
train, held = FastDecisionModel.split_holdout(items, 3407)
assert len(held) == 15 and not {i["row"] for i in train} & {i["row"] for i in held}
# One row with 100 decisions among single-decision rows: the target is 10.
items = [{"row": row} for row, size in enumerate([1] * 9 + [100]) for _ in range(size)]
for seed in range(20):
train, held = FastDecisionModel.split_holdout(items, seed)
assert len(held) <= 10 and len(train) >= 100
single = [{"row": 0}, {"row": 0}]
assert FastDecisionModel.split_holdout(single, 3407, fraction = 0.5) == (single, [])
def test_full_finetuning_trains_every_weight(checkpoint, capsys):
model, _ = FastDecisionModel.from_pretrained(
str(checkpoint), full_finetuning = True, use_gradient_checkpointing = False
)
assert all(p.requires_grad and p.dtype == torch.float32 for p in model.parameters())
assert FastDecisionModel.get_peft_model(model) is model
assert "Full finetuning is enabled" in capsys.readouterr().out
assert not hasattr(model.encoder, "peft_config")
with pytest.raises(NotImplementedError, match = "16-bit"):
FastDecisionModel.from_pretrained(str(checkpoint), load_in_4bit = True)
def test_lora_freezes_a_16_bit_encoder_and_trains_the_head_in_fp32(checkpoint):
model, _ = FastDecisionModel.from_pretrained(str(checkpoint), use_gradient_checkpointing = False)
stored = torch.float32 if decision._device().type == "cpu" else torch.float16
assert {p.dtype for p in model.encoder.parameters()} == {stored}
assert not any(p.requires_grad for p in model.encoder.parameters())
head = [p for n, p in model.named_parameters() if not n.startswith("encoder.")]
assert head and all(p.requires_grad and p.dtype == torch.float32 for p in head)
model, _ = FastDecisionModel.from_pretrained(
str(checkpoint), dtype = torch.float16, use_gradient_checkpointing = False
)
model = FastDecisionModel.get_peft_model(model, r = 4, lora_alpha = 4)
lora = [p for n, p in model.named_parameters() if "lora_" in n]
assert lora and all(p.requires_grad and p.dtype == torch.float32 for p in lora)
frozen = [p for n, p in model.encoder.named_parameters() if "lora_" not in n]
assert all(not p.requires_grad and p.dtype == torch.float16 for p in frozen)
assert all(b.dtype == torch.float32 for b in model.encoder.buffers() if b.is_floating_point())
targets = {n.split(".")[-1] for n, m in model.encoder.named_modules() if hasattr(m, "lora_A")}
assert targets == {"Wqkv", "Wo", "Wi"}
with pytest.raises(RuntimeError, match = "already added LoRA"):
FastDecisionModel.get_peft_model(model)
@pytest.mark.parametrize("lora", [True, False])
def test_head_learning_rate_and_weight_decay(checkpoint, tmp_path, lora):
model, tokenizer = FastDecisionModel.from_pretrained(
str(checkpoint), full_finetuning = not lora, use_gradient_checkpointing = False
)
model = FastDecisionModel.get_peft_model(model, r = 4, lora_alpha = 4)
items, _ = FastDecisionModel.build_dataset([_row(i) for i in range(4)], tokenizer, model)
names = {id(p): n for n, p in model.named_parameters()}
no_decay = {
f"{prefix}.{name}"
for prefix, module in model.named_modules()
for name, _ in module.named_parameters(recurse = False)
if isinstance(module, torch.nn.LayerNorm) or name.endswith("bias")
}
for head_learning_rate, head_lr in ((None, 1e-4), (0.0, 0.0)):
trainer = DecisionTrainer(
model = model,
args = _args(tmp_path, learning_rate = 8e-4, weight_decay = 0.01),
train_dataset = items,
processing_class = tokenizer,
head_learning_rate = head_learning_rate,
)
optimizer = trainer.create_optimizer()
grouped = [
(names[id(p)], group) for group in optimizer.param_groups for p in group["params"]
]
assert {name for name, _ in grouped} == {
n for n, p in model.named_parameters() if p.requires_grad
}
for name, group in grouped:
assert group["lr"] == (8e-4 if name.startswith("encoder.") else head_lr)
assert group["weight_decay"] == (0.0 if name in no_decay else 0.01)
@pytest.mark.skipif(
not hasattr(TrainingArguments, "activation_checkpointing"),
reason = "transformers < 5.23 reads gradient_checkpointing",
)
def test_trainer_checkpoints_the_encoder_not_the_wrapper(checkpoint, tmp_path):
model, tokenizer = FastDecisionModel.from_pretrained(
str(checkpoint), use_gradient_checkpointing = False
)
items, _ = FastDecisionModel.build_dataset([_row(i) for i in range(4)], tokenizer, model)
trainer = DecisionTrainer(
model = model,
args = _args(tmp_path, max_steps = 1, gradient_checkpointing = True),
train_dataset = items,
processing_class = tokenizer,
)
assert trainer.args.activation_checkpointing is False
assert model.encoder.is_gradient_checkpointing
trainer.train()
def test_length_grouping_only_with_gradient_accumulation(checkpoint, tmp_path):
model, tokenizer = FastDecisionModel.from_pretrained(
str(checkpoint), use_gradient_checkpointing = False
)
items, _ = FastDecisionModel.build_dataset([_row(i) for i in range(8)], tokenizer, model)
for accumulation in (1, 4):
trainer = DecisionTrainer(
model = model,
args = _args(tmp_path, gradient_accumulation_steps = accumulation),
train_dataset = items,
processing_class = tokenizer,
)
assert isinstance(trainer._get_train_sampler(), _LengthGroupedBatches) == (accumulation > 1)
def test_length_grouped_batches_mix_lengths_and_replay_an_epoch():
lengths = [10] * 64 + [500] * 64
sampler = _LengthGroupedBatches(lengths, 8, 3407)
order = list(sampler)
assert sorted(order) == list(range(128))
batches = [order[i : i + 8] for i in range(0, 128, 8)]
assert all(len({lengths[i] for i in batch}) == 1 for batch in batches)
assert lengths[batches[0][0]] == 500
assert {lengths[i] for batch in batches[:8] for i in batch} == {10, 500}
# The order depends only on the epoch, so a resumed run replays the epoch it stopped in.
assert list(sampler) == order
sampler.set_epoch(1)
assert list(sampler) != order
sampler.set_epoch(0)
assert list(sampler) == order
@pytest.mark.parametrize("lora", [True, False])
def test_train_calibrate_save_and_serve(checkpoint, tmp_path, lora):
from safetensors.torch import load_file
model, tokenizer = FastDecisionModel.from_pretrained(
str(checkpoint), full_finetuning = not lora, use_gradient_checkpointing = False
)
model = FastDecisionModel.get_peft_model(model, r = 4, lora_alpha = 4)
items, _ = FastDecisionModel.build_dataset([_row(i) for i in range(80)], tokenizer, model)
train, held = FastDecisionModel.split_holdout(items, 3407, fraction = 0.5)
midway = {}
class SaveMidway(TrainerCallback):
def on_step_end(
self,
args,
state,
control,
model = None,
**kwargs,
):
if state.global_step == 2:
model.save_pretrained_merged(tmp_path / "midway")
midway.update(
{n: p.detach().clone() for n, p in model.named_parameters() if p.requires_grad}
)
def accuracy(prediction):
hits = prediction.predictions.argmax(-1) == prediction.label_ids.argmax(-1)
return {"accuracy": float(hits.mean())}
trainer = DecisionTrainer(
model = model,
args = _args(tmp_path),
train_dataset = train,
eval_dataset = held,
tokenizer = tokenizer,
compute_metrics = accuracy,
callbacks = [SaveMidway()],
)
trainer.train()
# Saving midway kept the adapters on the model, so the steps after it still trained them.
assert decision.is_decision_checkpoint(tmp_path / "midway")
after = dict(model.named_parameters())
assert set(midway) <= set(after)
assert any(not torch.equal(after[n], p) for n, p in midway.items() if n.startswith("encoder."))
metrics = trainer.evaluate()
assert metrics["eval_loss"] > 0 and 0 <= metrics["eval_accuracy"] <= 1
assert trainer.predict(held).predictions.shape == (len(held), 3)
calibrated = FastDecisionModel.calibrate(model, tokenizer, held)
assert calibrated["fitted_types"] == [0, 1, 2] and 0 <= calibrated["accuracy"] <= 1
assert model.decision_config["temperature"] != [1.2, 1.1, 1.3]
assert "temperature_by_options" not in model.decision_config
model.decision_config["training"] = {"steps": 4}
model.save_pretrained_merged(tmp_path / "out")
saved = json.loads((tmp_path / "out" / "rl_agent_config.json").read_text())
assert saved["fine_tuned"] is True and saved["training"] == {"steps": 4}
assert saved["temperature"] == model.decision_config["temperature"]
encoder_config = (checkpoint / "encoder" / "config.json").read_bytes()
assert (tmp_path / "out" / "encoder" / "config.json").read_bytes() == encoder_config
agent = laya.load(str(tmp_path / "out"), device = "cpu")
served = agent.predict("the server is down again", QUESTIONS)["answers"]
assert served["team"]["choice"] in ("outage", "billing")
# predict() answers like the served checkpoint, from the model in memory.
ours = FastDecisionModel.predict(model, tokenizer, "the server is down again", QUESTIONS)
assert set(ours) == set(QUESTIONS)
for name, answer in served.items():
assert ours[name]["type"] == answer["type"]
for key, value in (answer.get("probabilities") or {}).items():
assert ours[name]["probabilities"][key] == pytest.approx(value, abs = 0.02)
FastDecisionModel.for_inference(model)
assert not model.training
FastDecisionModel.for_training(model)
assert model.training
weights = load_file(str(tmp_path / "out" / "model.safetensors"))
merged = model.encoder.merge_and_unload() if lora else model.encoder
expected = {f"encoder.{k}": v for k, v in merged.state_dict().items()}
expected.update((k, v) for k, v in model.state_dict().items() if not k.startswith("encoder."))
assert weights.keys() == expected.keys()
assert all(torch.equal(weights[k], v.detach().cpu().half()) for k, v in expected.items())
def test_long_runs_compile_the_encoder_layers_and_leave_them_eager(
checkpoint, tmp_path, monkeypatch
):
from unsloth.models import _decision_fast
model, tokenizer = FastDecisionModel.from_pretrained(
str(checkpoint), use_gradient_checkpointing = False
)
model = FastDecisionModel.get_peft_model(model, r = 4, lora_alpha = 8)
items, _ = FastDecisionModel.build_dataset([_row(i) for i in range(8)], tokenizer, model)
compiled, static = [], []
monkeypatch.setattr(torch.nn.Module, "compile", lambda self, **kw: compiled.append((self, kw)))
monkeypatch.setattr(_decision_fast, "_compile_static", lambda fn: static.append(fn) or fn)
layers = _decision_fast._encoder_layers(model)
assert len(layers) == 2
trainer = DecisionTrainer(model = model, args = _args(tmp_path, max_steps = 1), train_dataset = items)
# Short runs and CPUs stay eager; UNSLOTH_DECISION_COMPILE=1 forces it, =0 refuses it.
monkeypatch.delenv("UNSLOTH_DECISION_COMPILE", raising = False)
trainer.train()
assert not compiled
assert _decision_fast._wants_compile(model, 10**6) == next(model.parameters()).is_cuda
monkeypatch.setattr(_decision_fast, "_on_gpu", lambda model: True)
assert _decision_fast._wants_compile(model, _decision_fast.COMPILE_MIN_FORWARDS)
assert not _decision_fast._wants_compile(model, _decision_fast.COMPILE_MIN_FORWARDS - 1)
monkeypatch.setenv("UNSLOTH_DECISION_COMPILE", "1")
trainer.train()
# The test decisions are short, so each layer compiles once for static 64-token buckets.
assert not compiled and len(static) == len(layers)
assert all(layer._compiled_call_impl is None for layer in layers)
monkeypatch.setattr(_decision_fast, "STATIC_MAX_LEN", 0)
trainer.train()
assert [m for m, _ in compiled] == layers and all(kw == {"dynamic": True} for _, kw in compiled)
assert all(layer._compiled_call_impl is None for layer in layers)
monkeypatch.setenv("UNSLOTH_DECISION_COMPILE", "0")
assert not _decision_fast._wants_compile(model, 10**6)
def test_a_failing_compile_trains_eagerly(checkpoint, tmp_path, monkeypatch):
from unsloth.models import _decision_fast as fast
model, tokenizer = FastDecisionModel.from_pretrained(
str(checkpoint), use_gradient_checkpointing = False
)
model = FastDecisionModel.get_peft_model(model, r = 4, lora_alpha = 8)
items, _ = FastDecisionModel.build_dataset([_row(i) for i in range(8)], tokenizer, model)
def broken(*args, **kwargs):
raise RuntimeError("inductor cannot serve this platform")
def compile(self, **kwargs):
self._compiled_call_impl = broken
monkeypatch.setattr(torch.nn.Module, "compile", compile)
monkeypatch.setattr(fast, "_compile_static", lambda fn: broken)
monkeypatch.setenv("UNSLOTH_DECISION_COMPILE", "1")
trainer = DecisionTrainer(model = model, args = _args(tmp_path, max_steps = 2), train_dataset = items)
trainer.train()
assert trainer.state.global_step == 2
assert model._unsloth_decision_compiled is False
assert all(layer._compiled_call_impl is None for layer in fast._encoder_layers(model))
assert all(p.grad is None or torch.isfinite(p.grad).all() for p in model.parameters())
def test_compiled_layers_checkpoint_with_torch_and_get_unsloths_back(
checkpoint, tmp_path, monkeypatch
):
import functools
from unsloth.models import _decision_fast as fast
torch_checkpoint = fast._torch_checkpoint()
assert torch_checkpoint.__module__ == "torch.utils.checkpoint"
model, _ = FastDecisionModel.from_pretrained(str(checkpoint), use_gradient_checkpointing = False)
model = FastDecisionModel.get_peft_model(model, r = 4, lora_alpha = 8)
layers = fast._encoder_layers(model)
def unsloth_checkpoint(*args, **kwargs):
raise AssertionError("compiled layers must not use Unsloth's checkpoint")
offloaded = functools.partial(unsloth_checkpoint, use_reentrant = True)
# unsloth_zoo patches torch.utils.checkpoint.checkpoint once a model loads with its checkpointing.
monkeypatch.setattr(torch.utils.checkpoint, "_old_checkpoint", torch_checkpoint, raising = False)
monkeypatch.setattr(torch.utils.checkpoint, "checkpoint", unsloth_checkpoint)
for layer in layers:
layer._gradient_checkpointing_func = offloaded
monkeypatch.setattr(torch.nn.Module, "compile", lambda self, **kw: None)
monkeypatch.setattr(fast, "_wants_compile", lambda model, forwards: True)
monkeypatch.setattr(fast, "_warm_up", lambda model, amp_dtype: None)
with fast.compiled_encoder(model, 10**6) as compiled:
assert compiled
for layer in layers:
func = layer._gradient_checkpointing_func
assert func.func is torch_checkpoint and func.keywords == {"use_reentrant": False}
assert all(layer._gradient_checkpointing_func is offloaded for layer in layers)
def broken(model, amp_dtype):
raise RuntimeError("inductor cannot serve this platform")
monkeypatch.setattr(fast, "_warm_up", broken)
with fast.compiled_encoder(model, 10**6) as compiled:
assert not compiled
assert all(layer._gradient_checkpointing_func is offloaded for layer in layers)
# torch.compile itself can refuse (an unsupported Python): the second layer raises here.
def refuse(self, **kwargs):
if self is layers[1]:
raise RuntimeError("Dynamo is not supported on this Python")
self._compiled_call_impl = lambda *a, **k: None
monkeypatch.setattr(torch.nn.Module, "compile", refuse)
monkeypatch.setattr(fast, "_warm_up", lambda model, amp_dtype: None)
with fast.compiled_encoder(model, 10**6) as compiled:
assert not compiled
assert all(layer._compiled_call_impl is None for layer in layers)
assert all(layer._gradient_checkpointing_func is offloaded for layer in layers)
@pytest.mark.parametrize("instructions", [None, ""])
def test_a_question_without_instructions_is_read_by_its_id(checkpoint, instructions):
model, tokenizer = FastDecisionModel.from_pretrained(
str(checkpoint), use_gradient_checkpointing = False
)
state = "the server is down again"
blank = {"type": "noul", "instructions": instructions}
named = {name: {"type": "noul", "instructions": name} for name in ("outage", "billing")}
answers = FastDecisionModel.predict(model, tokenizer, state, dict.fromkeys(named, blank))
assert answers == FastDecisionModel.predict(model, tokenizer, state, named)
def tokens(questions):
gold = dict.fromkeys(questions, {"label": "true"})
row = {"state": state, "questions": questions, "gold": gold}
return [
item["input_ids"]
for item in FastDecisionModel.build_dataset([row], tokenizer, model)[0]
]
assert tokens(dict.fromkeys(named, blank)) == tokens(named)
def test_a_resized_laya_vocabulary_saves_and_reloads(checkpoint, tmp_path):
model, tokenizer = FastDecisionModel.from_pretrained(
str(checkpoint), use_gradient_checkpointing = False
)
grown = model.encoder.get_input_embeddings().num_embeddings + 8
# The fixture's vocab equals its hidden size, so the covariance mean resizing samples from
# is singular and can come out NaN on some GPUs.
model.encoder.resize_token_embeddings(grown, mean_resizing = False)
model.save_pretrained_merged(str(tmp_path / "out"), tokenizer)
reloaded, _ = FastDecisionModel.from_pretrained(str(tmp_path / "out"))
assert reloaded.encoder.get_input_embeddings().num_embeddings == grown
def test_full_clef_finetuning_never_quantizes_the_backbone(tmp_path, monkeypatch):
from unsloth.models import loader
seen = []
class Captured(Exception):
pass
def capture(*args, **kwargs):
seen.append(kwargs)
raise Captured
monkeypatch.setattr(decision, "_device", lambda: torch.device("cuda"))
monkeypatch.setattr(loader.FastModel, "from_pretrained", capture)
for full in (True, False):
with pytest.raises(Captured):
decision._load_clef(tmp_path, None, torch.bfloat16, True, full, None, False, {})
assert seen[0].get("quantization_config") is None
assert seen[1].get("quantization_config") is not None
def test_the_compile_warm_up_leaves_the_rng_where_it_was(checkpoint, monkeypatch):
from unsloth.models import _decision_fast as fast
model, _ = FastDecisionModel.from_pretrained(str(checkpoint), use_gradient_checkpointing = False)
monkeypatch.setattr(torch.nn.Module, "compile", lambda self, **kw: None)
monkeypatch.setattr(fast, "_wants_compile", lambda model, forwards: True)
torch.manual_seed(0)
before = torch.get_rng_state()
with fast.compiled_encoder(model, 10**6) as compiled:
assert compiled
assert torch.equal(torch.get_rng_state(), before)
def test_the_compile_warm_up_forks_the_rng_of_the_models_own_accelerator(monkeypatch):
from unsloth.models import _decision_fast as fast
forked = []
class Forked(Exception):
pass
def fork_rng(**kwargs):
forked.append(kwargs)
raise Forked
model = types.SimpleNamespace(
training = True,
parameters = lambda: iter([torch.empty(1, device = "meta")]),
train = lambda mode = True: None,
encoder = types.SimpleNamespace(config = types.SimpleNamespace(vocab_size = 100)),
)
monkeypatch.setattr(torch.random, "fork_rng", fork_rng)
with pytest.raises(Forked):
fast._warm_up(model, None)
# An XPU / MPS model forks its own RNG, not CUDA's (meta stands in for any accelerator).
assert forked == [{"devices": [0], "device_type": "meta"}]
def test_clef_backbone_stays_on_one_device_unless_the_caller_places_it(tmp_path, monkeypatch):
from unsloth.models import loader, loader_utils
seen = []
class Captured(Exception):
pass
def capture(*args, **kwargs):
seen.append(kwargs.get("device_map"))
raise Captured
monkeypatch.setattr(decision, "_device", lambda: torch.device("cuda"))
monkeypatch.setattr(loader.FastModel, "from_pretrained", capture)
index = torch.cuda.current_device() if torch.cuda.is_available() else 0
for kwargs in ({}, {"device_map": None}, {"device_map": "auto"}):
with pytest.raises(Captured):
decision._load_clef(tmp_path, None, torch.bfloat16, False, False, None, False, kwargs)
monkeypatch.setattr(loader_utils, "prepare_device_map", lambda: ({"": "cuda:1"}, True))
with pytest.raises(Captured):
decision._load_clef(tmp_path, None, torch.bfloat16, False, False, None, False, {})
assert seen == [{"": f"cuda:{index}"}, {"": f"cuda:{index}"}, "auto", {"": "cuda:1"}]
def test_toy_task_beats_the_base_model(checkpoint, tmp_path, monkeypatch):
# The toy task plateaus near loss 0.45 and leaves it by step ~70 on CPU but only after ~100
# steps on GPU, so 80 steps is tuned to CPU arithmetic, not a GPU defect.
monkeypatch.setattr(decision, "_device", lambda: torch.device("cpu"))
# Eager too: a compiled encoder draws the same accuracy from a different 45-decision RNG stream.
monkeypatch.setenv("UNSLOTH_DECISION_COMPILE", "0")
model, tokenizer = FastDecisionModel.from_pretrained(
str(checkpoint), use_gradient_checkpointing = False
)
model = FastDecisionModel.get_peft_model(model, r = 8, lora_alpha = 16)
items, _ = FastDecisionModel.build_dataset([_row(i) for i in range(120)], tokenizer, model)
train, held = FastDecisionModel.split_holdout(items, 3407, fraction = 0.25)
base = FastDecisionModel.evaluate(model, tokenizer, held)
DecisionTrainer(
model = model,
args = _args(
tmp_path,
max_steps = 80,
learning_rate = 5e-3,
per_device_train_batch_size = 16,
use_cpu = True,
),
train_dataset = train,
processing_class = tokenizer,
head_learning_rate = 5e-3,
).train()
tuned = FastDecisionModel.evaluate(model, tokenizer, held)
# The random base can already score high on some transformers versions; 1.0 is the ceiling.
assert tuned["accuracy"] >= min(base["accuracy"] + 0.25, 1.0)
assert tuned["loss"] < base["loss"] / 2
def test_trainer_leaves_the_callers_arguments_alone(checkpoint, tmp_path, monkeypatch):
model, tokenizer = FastDecisionModel.from_pretrained(
str(checkpoint), use_gradient_checkpointing = False
)
items, _ = FastDecisionModel.build_dataset([_row(i) for i in range(4)], tokenizer, model)
args = _args(tmp_path, max_steps = 1, gradient_checkpointing = True)
trainer = DecisionTrainer(model = model, args = args, train_dataset = items)
assert args.remove_unused_columns is True and args.label_names is None
assert args.gradient_checkpointing is True
trainer.train()
assert model.encoder.is_gradient_checkpointing
# Without arguments, as Trainer allows.
monkeypatch.chdir(tmp_path)
trainer = DecisionTrainer(model = model, train_dataset = items, processing_class = tokenizer)
batch = next(iter(trainer.get_train_dataloader()))
assert {"marker_pos", "marker_mask", "target"} <= set(batch)
def test_for_training_turns_gradient_checkpointing_back_on(checkpoint):
model, _ = FastDecisionModel.from_pretrained(str(checkpoint), use_gradient_checkpointing = False)
assert not model.encoder.is_gradient_checkpointing
FastDecisionModel.for_training(model)
assert model.training and model.encoder.is_gradient_checkpointing
FastDecisionModel.for_training(model, use_gradient_checkpointing = False)
assert not model.encoder.is_gradient_checkpointing
def test_trainer_uses_one_gpu_and_the_set_batch_on_a_multi_gpu_machine(checkpoint, tmp_path):
model, tokenizer = FastDecisionModel.from_pretrained(
str(checkpoint), use_gradient_checkpointing = False
)
args = _args(tmp_path, max_steps = 1)
args._n_gpu = 2
trainer = DecisionTrainer(model = model, args = args, processing_class = tokenizer)
assert trainer.args.n_gpu == 1 and args.n_gpu == 2
assert trainer.args.train_batch_size == args.per_device_train_batch_size
assert trainer._wrap_model(model) is model
@pytest.mark.skipif(not has_real_cuda(), reason = "measures GPU memory")
def test_lora_save_does_not_copy_the_encoder_on_the_gpu(checkpoint, tmp_path):
model, tokenizer = FastDecisionModel.from_pretrained(
str(checkpoint), use_gradient_checkpointing = False
)
model = FastDecisionModel.get_peft_model(model, r = 8, lora_alpha = 16)
torch.cuda.synchronize()
before = torch.cuda.memory_allocated()
torch.cuda.reset_peak_memory_stats()
model.save_pretrained_merged(tmp_path / "out", tokenizer)
torch.cuda.synchronize()
assert torch.cuda.max_memory_allocated() <= before
assert next(model.encoder.parameters()).is_cuda and hasattr(model.encoder, "peft_config")
def test_save_refuses_weights_float16_cannot_hold(checkpoint, tmp_path):
model, _ = FastDecisionModel.from_pretrained(
str(checkpoint), full_finetuning = True, use_gradient_checkpointing = False
)
model.save_pretrained_merged(tmp_path / "out")
with torch.no_grad():
model.scorer[1].weight[0, 0] = 7e4
with pytest.raises(ValueError, match = "float16"):
model.save_pretrained_merged(tmp_path / "out")
assert decision.is_decision_checkpoint(tmp_path / "out")
def test_push_to_hub_merged_uploads_a_complete_checkpoint(checkpoint, monkeypatch):
import huggingface_hub
pushed = {}
class Api:
def __init__(self, token = None):
pushed["token"] = token
def create_repo(
self,
repo_id,
private = None,
exist_ok = False,
):
pushed["private"] = private
return types.SimpleNamespace(repo_id = f"me/{repo_id}")
def upload_folder(self, folder_path, repo_id, **kwargs):
pushed["repo_id"] = repo_id
pushed["complete"] = decision.is_decision_checkpoint(folder_path)
monkeypatch.setattr(huggingface_hub, "HfApi", Api)
model, _ = FastDecisionModel.from_pretrained(str(checkpoint), use_gradient_checkpointing = False)
model.push_to_hub_merged("laya-ft", token = "hf_token", private = True)
assert pushed == {
"token": "hf_token",
"private": True,
"repo_id": "me/laya-ft",
"complete": True,
}
def test_gradient_accumulation_matches_one_large_batch(checkpoint, tmp_path):
grads = {}
class Grab(TrainerCallback):
def __init__(self, key):
self.key = key
def on_pre_optimizer_step(
self,
args,
state,
control,
model = None,
**kwargs,
):
grads[self.key] = torch.cat(
[p.grad.flatten() for p in model.parameters() if p.grad is not None]
)
for lora in (False, True):
for accumulation in (1, 4):
model, tokenizer = FastDecisionModel.from_pretrained(
str(checkpoint),
full_finetuning = not lora,
dtype = torch.float32,
use_gradient_checkpointing = False,
)
model = FastDecisionModel.get_peft_model(model, r = 4, lora_alpha = 4)
model.eval()
items, _ = FastDecisionModel.build_dataset(
[_row(i) for i in range(16)], tokenizer, model
)
trainer = DecisionTrainer(
model = model,
args = _args(
tmp_path,
per_device_train_batch_size = 48 // accumulation,
gradient_accumulation_steps = accumulation,
max_steps = 1,
max_grad_norm = 0.0,
use_cpu = True,
),
train_dataset = items,
processing_class = tokenizer,
callbacks = [Grab((lora, accumulation))],
)
trainer.model.train = lambda mode = True: trainer.model
trainer.train()
torch.testing.assert_close(grads[(lora, 1)], grads[(lora, 4)], rtol = 1e-3, atol = 1e-6)
def test_rope_parameters_build_the_trained_rope_on_every_transformers(tmp_path):
from transformers import AutoModel
def build(name, **rope):
folder = tmp_path / name
folder.mkdir()
(folder / "config.json").write_text(json.dumps(_tiny_encoder_config(**rope)))
return AutoModel.from_config(decision._encoder_config(folder), attn_implementation = "sdpa")
def rope_parameters(sliding):
return {
"full_attention": {"rope_theta": 160000.0, "rope_type": "default"},
"sliding_attention": {"rope_theta": sliding, "rope_type": "default"},
}
# How transformers 5 saves mmBERT, the same with both key styles, and with the 10000 default.
saved = build("saved", rope_parameters = rope_parameters(160000.0))
both = build(
"both",
rope_parameters = rope_parameters(160000.0),
global_rope_theta = 160000.0,
local_rope_theta = 160000.0,
)
default = build(
"default",
rope_parameters = rope_parameters(10000.0),
global_rope_theta = 160000.0,
local_rope_theta = 10000.0,
)
with torch.no_grad():
# Random weights attend almost uniformly, which hides the rotary angles; sharpen the attention.
for name, weight in saved.named_parameters():
if name.endswith("Wqkv.weight"):
weight.mul_(10)
for model in (both, default):
model.load_state_dict(saved.state_dict())
ids = torch.randint(5, 64, (2, 24))
outputs = [model.eval()(input_ids = ids).last_hidden_state for model in (saved, both, default)]
torch.testing.assert_close(outputs[0], outputs[1])
assert not torch.allclose(outputs[0], outputs[2], atol = 1e-2)
TINY_QWEN3_5 = "trl-internal-testing/tiny-Qwen3_5ForConditionalGeneration-NoThink"
CLEF_HEAD = {
"hidden_size": 16,
"width": 32,
"routing_layers": 1,
"layers": 1,
"heads": 4,
"feedforward": 64,
}
def _clef_reference():
from huggingface_hub import hf_hub_download
try:
path = hf_hub_download("Cloudflare/clef-flash", "joint_schema_model.py")
except Exception as exc:
pytest.skip(f"Clef reference code unavailable: {exc}")
import importlib.util
spec = importlib.util.spec_from_file_location("clef_reference", path)
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module, path
@pytest.fixture
def clef_checkpoint(tmp_path):
import shutil
from safetensors.torch import save_file
from transformers import AutoProcessor, Qwen3_5ForConditionalGeneration
from unsloth.models.clef import JointSchemaHead
if not has_real_cuda():
try:
import causal_conv1d # noqa: F401
except ImportError:
pass
else:
pytest.skip("transformers calls causal_conv1d's CUDA-only kernel on CPU tensors")
reference, path = _clef_reference()
folder = tmp_path / "clef"
torch.manual_seed(0)
Qwen3_5ForConditionalGeneration.from_pretrained(
TINY_QWEN3_5, dtype = torch.bfloat16
).save_pretrained(str(folder))
AutoProcessor.from_pretrained(TINY_QWEN3_5).save_pretrained(str(folder))
head = JointSchemaHead(**CLEF_HEAD)
with torch.no_grad():
for param in head.parameters():
param.add_(torch.randn_like(param) * 0.1)
save_file(
{k: v.to(torch.bfloat16).contiguous() for k, v in head.state_dict().items()},
str(folder / "joint_head.safetensors"),
)
(folder / "joint_head_config.json").write_text(json.dumps(CLEF_HEAD))
shutil.copyfile(path, folder / "joint_schema_model.py")
return folder
def _clef_rows(n):
rows = [_row(i) for i in range(n)]
for row in rows:
row["questions"] = {
**QUESTIONS,
"tags": {"type": "choice", "criteria": ["billing", "outage", "other"]},
}
row["gold"]["tags"] = row["gold"]["team"]["label"]
return rows
def _clef_items_with(tokenizer, rows):
report = {"total": 0, "skipped": 0, "reason": None, "truncated": 0}
items = decision._clef_items(rows, tokenizer, 4096, None, report, lambda *args: None)
return [
(item["input_ids"], item["record"].input_ids, item["record"].questions, item["targets"])
for item in items
], report
def test_clef_items_tokenize_each_repeated_string_once():
from transformers import AutoTokenizer
from unsloth.models.clef import encode_record
tokenizer = AutoTokenizer.from_pretrained(TINY_QWEN3_5)
rows = [{**_row(i), "state": {"ticket": i, "note": "word " * (40 * i)}} for i in range(24)]
texts = []
class Counting:
def __call__(self, text, **kwargs):
texts.append(text)
return tokenizer(text, **kwargs)
items, report = _clef_items_with(Counting(), rows)
# Each row on its own, as encode_record sees a plain tokenizer.
plain = [
encode_record(tokenizer, {"state": r["state"], "questions": QUESTIONS}, max_length = 4096)
for r in rows
]
assert [item[1] for item in items] == [record.input_ids for record in plain]
assert [item[2] for item in items] == [record.questions for record in plain]
assert all(list(item[1]) == item[0] for item in items)
assert report["skipped"] == 0
# Every distinct string once: the fixed prompt and option texts are shared by every row.
assert len(texts) == len(set(texts))
assert len(texts) < 2 * len(rows) + 30
def test_memo_tokenizer_returns_fresh_lists_and_passes_other_calls_through():
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(TINY_QWEN3_5)
memo = decision._MemoTokenizer(tokenizer)
first = memo("OPTION 1: ", add_special_tokens = False).input_ids
first.extend([1, 2, 3])
assert (
memo("OPTION 1: ", add_special_tokens = False).input_ids
== tokenizer("OPTION 1: ", add_special_tokens = False).input_ids
)
long = "state " * 200
assert (
memo(long, add_special_tokens = False).input_ids
== tokenizer(long, add_special_tokens = False).input_ids
)
assert long not in memo.cache
assert memo("hi").input_ids == tokenizer("hi").input_ids
assert (
memo(["a", "b"], add_special_tokens = False).input_ids
== tokenizer(["a", "b"], add_special_tokens = False).input_ids
)
def test_clef_encoding_is_token_identical_to_cloudflares():
from transformers import AutoTokenizer
from unsloth.models.clef import encode_record
reference, _ = _clef_reference()
tokenizer = AutoTokenizer.from_pretrained(TINY_QWEN3_5)
for record in (
{"state": {"invoice": {"total": 1250.0, "vendor": "Acmé"}}, "questions": QUESTIONS},
{"state": "plain text", "questions": {"x": {"type": "noul", "criteria": {"true": "yes"}}}},
{
"state": "s" * 500,
"questions": {"lvl": {"type": "score", "instructions": "", "criteria": [1, "b"]}},
},
):
for max_length in (100, 400, 16384):
if max_length == 100:
with pytest.raises(ValueError):
reference.encode_record(tokenizer, record, max_length = max_length)
with pytest.raises(ValueError):
encode_record(tokenizer, record, max_length = max_length)
continue
ours = encode_record(tokenizer, record, max_length = max_length)
theirs = reference.encode_record(tokenizer, record, max_length = max_length)
assert ours.input_ids == theirs.input_ids
assert [
(q.question_id, q.question_type, q.question_span, q.option_spans, q.option_ids)
for q in ours.questions
] == [
(q.question_id, q.question_type, q.question_span, q.option_spans, q.option_ids)
for q in theirs.questions
]
@pytest.mark.parametrize(
"option",
[
{"use_dora": True},
{"layers_to_transform": [0]},
{"layers_pattern": "layers"},
{"loftq_config": {"loftq_bits": 4}},
{"init_lora_weights": "gaussian"},
],
)
def test_clef_lora_refuses_options_it_would_drop(monkeypatch, option):
monkeypatch.setattr(decision, "_clef_peft_model", lambda model, **kwargs: "plain lora")
model = types.SimpleNamespace(is_clef = True, encoder = types.SimpleNamespace())
with pytest.raises(NotImplementedError, match = next(iter(option))):
FastDecisionModel.get_peft_model(model, **option)
assert FastDecisionModel.get_peft_model(model) == "plain lora"
def test_clef_metrics_score_in_the_runs_batch_size(monkeypatch):
seen = []
def clef_logits(
model,
items,
pad_token_id,
batch_size = 4,
processor = None,
):
seen.append(batch_size)
raise StopIteration
monkeypatch.setattr(decision, "_clef_logits", clef_logits)
model = types.SimpleNamespace(is_clef = True, decision_config = {})
tokenizer = types.SimpleNamespace(pad_token_id = 0)
for call in (FastDecisionModel.evaluate, FastDecisionModel.calibrate):
with pytest.raises(StopIteration):
call(model, tokenizer, [], batch_size = 1)
with pytest.raises(StopIteration):
FastDecisionModel.evaluate(model, tokenizer, [])
assert seen == [1, 1, 4]
def test_clef_loads_scores_like_cloudflares_model_and_trains(clef_checkpoint, tmp_path):
reference, _ = _clef_reference()
model, processor = FastDecisionModel.from_pretrained(str(clef_checkpoint), max_seq_length = 512)
assert getattr(model, "is_clef", False) and model.head.hidden_norm.weight.dtype == torch.float32
device = next(model.parameters()).device
backbone_dtype = next(model.encoder.parameters()).dtype
released, _ = reference.load_release_model(
str(clef_checkpoint), device = device, dtype = backbone_dtype
)
record = {"state": "the server is down again", "questions": QUESTIONS}
encoded = reference.encode_record(processor.tokenizer, record)
batch = reference.collate_records([encoded], processor.tokenizer.pad_token_id, device)
with torch.no_grad():
theirs = [z.float() for z in released(batch)[0]]
ours, _ = model(batch["input_ids"], batch["attention_mask"], batch["records"])
fp32_head = model.head
model.head = copy.deepcopy(fp32_head).to(backbone_dtype)
ours_same_dtype, _ = model(batch["input_ids"], batch["attention_mask"], batch["records"])
model.head = fp32_head
for row, z in enumerate(theirs):
same = ours_same_dtype[row, : len(z)].float()
assert torch.allclose(same, z, atol = 2e-2, rtol = 2e-2), (row, (same - z).abs().max(), same, z)
fp32 = ours[row, : len(z)].float()
assert torch.allclose(fp32, z, atol = 0.1), (row, (fp32 - z).abs().max(), fp32, z)
items, report = FastDecisionModel.build_dataset(_clef_rows(64), processor, model)
assert report["skipped"] == 0 and len(items) == 64 and len(items[0]["targets"]) == 4
train, holdout = FastDecisionModel.split_holdout(items, fraction = 0.25)
assert sum(len(i["labels"]) for i in holdout) <= 64
before = FastDecisionModel.evaluate(model, processor, holdout)
model = FastDecisionModel.get_peft_model(model, r = 8, lora_alpha = 8)
losses = []
class Losses(TrainerCallback):
def on_log(
self,
args,
state,
control,
logs = None,
**kwargs,
):
if logs and "loss" in logs:
losses.append(logs["loss"])
trainer = DecisionTrainer(
model = model,
tokenizer = processor,
train_dataset = train,
args = _args(tmp_path, max_steps = 30, learning_rate = 5e-3, logging_steps = 1),
head_learning_rate = 5e-3,
callbacks = [Losses()],
)
trainer.train()
after = FastDecisionModel.evaluate(model, processor, holdout)
assert min(losses[-5:]) < losses[0] and after["loss"] < before["loss"], (losses, before, after)
calibration = FastDecisionModel.calibrate(model, processor, holdout)
assert "accuracy" in calibration
# Fitted to the gold labels, so the calibrated confidence tracks being right.
head_temperature = model.decision_config["head_temperature"]
assert (
decision.HEAD_TEMPERATURE_RANGE[0] <= head_temperature <= decision.HEAD_TEMPERATURE_RANGE[1]
)
model.decision_config["training"] = {"steps": 30}
model.save_pretrained_merged(str(tmp_path / "out"))
assert (tmp_path / "out" / "joint_schema_model.py").is_file()
reloaded, _ = FastDecisionModel.from_pretrained(str(tmp_path / "out"), max_seq_length = 512)
# The parent run's record stays on disk but does not describe the next fine-tune.
assert "training" not in reloaded.decision_config
# The calibrated temperature over all questions is folded into the saved head; the per-type
# temperatures are relative to it, so the reload serves the same probabilities.
folded = reloaded.decision_config["folded_temperature"]
assert folded == pytest.approx(model.decision_config["head_temperature"])
assert "head_temperature" not in reloaded.decision_config
assert reloaded.decision_config["temperature"] == model.decision_config["temperature"]
released, _ = reference.load_release_model(
str(tmp_path / "out"), device = device, dtype = torch.float32
)
with torch.no_grad():
trained, _ = model(batch["input_ids"], batch["attention_mask"], batch["records"])
again, _ = reloaded(batch["input_ids"], batch["attention_mask"], batch["records"])
theirs = released(batch)[0]
# bf16 reload rounding grows with the per-run logit scale, so the bound is relative (10 B200 runs
# peaked at 1.6%); a fold left out or doubled is off by >= 20%.
mask = trained > -1e3
expected = (trained / folded).float()[mask]
error = (again.float()[mask] - expected).abs().max().item()
scale = expected.abs().max().item()
assert error <= 0.03 * scale + 0.05, (error, scale, folded)
for row, z in enumerate(theirs):
assert int(z.argmax()) == int(again[row, : len(z)].argmax()), (row, z, again[row, : len(z)])
# predict() serves the same calibrated answer from memory, the merged reload and the adapters.
state = "the server is down again"
ours = FastDecisionModel.predict(model, processor, state, QUESTIONS)
merged = FastDecisionModel.predict(reloaded, processor, state, QUESTIONS)
model.save_pretrained(str(tmp_path / "adapters"))
from_adapters, _ = FastDecisionModel.from_pretrained(
str(tmp_path / "adapters"), max_seq_length = 512
)
assert from_adapters.decision_config["base_model"] == str(clef_checkpoint)
adapters = FastDecisionModel.predict(from_adapters, processor, state, QUESTIONS)
for name in QUESTIONS:
for other in (merged, adapters):
assert other[name]["type"] == ours[name]["type"]
for key, value in (ours[name].get("probabilities") or {}).items():
assert other[name]["probabilities"][key] == pytest.approx(value, abs = 0.03)
@pytest.mark.skipif(not has_real_cuda(), reason = "the fast kernels need a CUDA device")
def test_clef_backbone_runs_the_compiled_gated_delta_and_conv_kernels(clef_checkpoint):
from transformers.utils.import_utils import is_causal_conv1d_available
model, _ = FastDecisionModel.from_pretrained(str(clef_checkpoint), max_seq_length = 512)
from transformers.utils.import_utils import is_flash_linear_attention_available
if not is_flash_linear_attention_available():
pytest.skip("flash-linear-attention is disabled on this GPU / Triton")
layers = [m for m in model.encoder.modules() if type(m).__name__.endswith("GatedDeltaNet")]
assert layers
for layer in layers:
forward = type(layer).forward
assert forward.__code__.co_filename.endswith("unsloth_compiled_module_qwen3_5.py")
delta = _chosen_kernel(
layer, forward, "torch_chunk_gated_delta_rule", "chunk_gated_delta_rule"
)
conv = _chosen_kernel(layer, forward, "causal_conv1d_fn", "causal_conv1d_fn")
assert delta is not None and ".fla." in "." + delta, delta
if is_causal_conv1d_available():
assert conv is not None and conv.startswith("causal_conv1d"), conv
def _chosen_kernel(layer, forward, global_name, attr):
# transformers < 5.6 stores the picked kernel on the layer; later versions freeze it in a
# closure under the hub-kernel wrappers of the module-level function.
fn = vars(layer)[attr] if attr in vars(layer) else forward.__globals__.get(global_name)
top = fn
while fn is not None and hasattr(fn, "__code__"):
cells = dict(zip(fn.__code__.co_freevars, fn.__closure__ or ()))
if "implementation" in cells:
return cells["implementation"].cell_contents.__module__
fn = getattr(fn, "__wrapped__", None)
return getattr(top, "__module__", None)
def test_decision_loss_recipe_terms():
torch.manual_seed(0)
logits = torch.randn(5, 4)
mask = torch.tensor([[1, 1, 1, 0]] * 3 + [[1, 1, 0, 0]] * 2, dtype = torch.bool)
target = torch.zeros(5, 4)
target[torch.arange(5), torch.tensor([0, 2, 1, 1, 0])] = 1.0
ordinal = torch.tensor([True, True, False, False, False])
plain = decision._soft_cross_entropy(logits, target, mask)
assert torch.equal(decision._decision_loss(logits, target, mask, ordinal), plain)
log_p = torch.log_softmax(logits.masked_fill(~mask, -1e4), -1)
p = log_p.exp()
k = mask.sum(-1, keepdim = True).float()
smoothed = 0.9 * target + 0.1 * mask.float() / k
expected_ce = -(smoothed * log_p).sum(-1).mean()
expected_brier = ((p - target) ** 2 * mask).sum(-1).mean()
levels = torch.arange(4.0)
gold = target.argmax(-1).float()
distance = (p * (levels - gold[:, None]).abs()).sum(-1) / (k[:, 0] - 1)
expected_ordinal = distance[ordinal].mean()
got = decision._decision_loss(
logits,
target,
mask,
ordinal,
label_smoothing = 0.1,
brier_weight = 0.5,
ordinal_weight = 2.0,
)
torch.testing.assert_close(got, expected_ce + 0.5 * expected_brier + 2.0 * expected_ordinal)
def test_record_accuracy_needs_every_question_of_a_row_right():
logits = [torch.tensor([2.0, 0.0]), torch.tensor([0.0, 2.0]), torch.tensor([2.0, 0.0])]
items = [
{"label": 0, "target": [1.0, 0.0], "row": 0, "qtype": 0},
{"label": 0, "target": [1.0, 0.0], "row": 0, "qtype": 0},
{"label": 0, "target": [1.0, 0.0], "row": 1, "qtype": 0},
]
metrics = decision._metrics(logits, items, [1.0] * 3)
assert metrics["accuracy"] == pytest.approx(2 / 3) and metrics["record_accuracy"] == 0.5
def _clef_head_inputs():
from transformers import AutoTokenizer
from unsloth.models.clef import JointSchemaHead, encode_record
tokenizer = AutoTokenizer.from_pretrained(TINY_QWEN3_5)
encoded = [
encode_record(tokenizer, {"state": "state " * (i + 1), "questions": QUESTIONS})
for i in range(2)
]
batch = decision.ClefDataCollator(tokenizer.pad_token_id)(
[
{"input_ids": list(r.input_ids), "record": r, "targets": [[1.0]], "qtypes": [0]}
for r in encoded
]
)
torch.manual_seed(0)
head = JointSchemaHead(**CLEF_HEAD)
with torch.no_grad():
for param in head.parameters():
param.add_(torch.randn_like(param) * 0.1)
hidden = torch.randn(*batch["input_ids"].shape, CLEF_HEAD["hidden_size"])
lexical = torch.randn(int(batch["input_ids"].max()) + 1, CLEF_HEAD["hidden_size"])
def run(module):
with torch.no_grad():
return [
z
for record in module(
hidden, batch["input_ids"], batch["attention_mask"], batch["records"], lexical
)
for z in record
]
return head, run
@pytest.mark.parametrize("temperature", [1.7, 0.6])
def test_clef_temperature_folds_into_the_head_exactly(temperature):
head, run = _clef_head_inputs()
state = {k: v.detach().clone() for k, v in head.state_dict().items()}
assert decision._fold_temperature(state, temperature)
folded = copy.deepcopy(head)
folded.load_state_dict(state)
for original, scaled in zip(run(head), run(folded)):
torch.testing.assert_close(scaled, original / temperature, rtol = 1e-5, atol = 1e-6)
def test_clef_temperature_is_not_folded_past_the_heads_scale_clamp():
head, _ = _clef_head_inputs()
state = {k: v.detach().clone() for k, v in head.state_dict().items()}
state["joint_logit_scale"] = torch.tensor(math.log(90.0))
before = {k: v.clone() for k, v in state.items()}
# Sharpening by 2 would need a scale of 180, past the clamp at 100.
assert not decision._fold_temperature(state, 0.5)
assert all(torch.equal(before[k], v) for k, v in state.items())
def test_clef_collator_permutes_fields_but_keeps_targets_with_their_questions():
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(TINY_QWEN3_5)
model = types.SimpleNamespace(is_clef = True, decision_config = {"max_len": 2048})
items, report = FastDecisionModel.build_dataset(_clef_rows(4), tokenizer, model)
assert report["skipped"] == 0
fixed = decision.ClefDataCollator(tokenizer.pad_token_id)
shuffled = decision.ClefDataCollator(
tokenizer.pad_token_id, tokenizer = tokenizer, max_len = 2048, permute_fields = True, seed = 1
)
orders = set()
for _ in range(8):
batch = shuffled(items[:1])
record = batch["records"][0]
orders.add(tuple(q.question_id for q in record.questions))
by_name = dict(zip(items[0]["source"]["questions"], items[0]["targets"]))
for row, question in enumerate(record.questions):
assert (
batch["target"][row, : len(question.option_ids)].tolist()
== by_name[question.question_id]
)
assert bool(batch["ordinal"][row]) == (question.question_type == 2)
assert len(orders) > 1
assert fixed(items[:1])["records"][0] == items[0]["record"]
def test_a_failed_save_leaves_the_previous_laya_checkpoint_loadable(
checkpoint, tmp_path, monkeypatch
):
model, tokenizer = FastDecisionModel.from_pretrained(
str(checkpoint), use_gradient_checkpointing = False
)
out = tmp_path / "out"
model.save_pretrained_merged(out)
before = sorted(p.name for p in out.iterdir())
original = (out / "model.safetensors").read_bytes()
with torch.no_grad():
for param in model.head.parameters():
param.add_(1.0)
def broken(*args, **kwargs):
raise OSError("disk full")
monkeypatch.setattr(type(tokenizer), "save_pretrained", broken)
with pytest.raises(OSError):
model.save_pretrained_merged(out)
assert sorted(p.name for p in out.iterdir()) == before
assert (out / "model.safetensors").read_bytes() == original
assert decision.is_decision_checkpoint(out)
FastDecisionModel.from_pretrained(str(out), use_gradient_checkpointing = False)
def test_a_failed_save_leaves_the_previous_clef_checkpoint_loadable(
clef_checkpoint, tmp_path, monkeypatch
):
import safetensors.torch
model, processor = FastDecisionModel.from_pretrained(str(clef_checkpoint), max_seq_length = 512)
out = tmp_path / "out"
model.save_pretrained_merged(str(out))
head = (out / "joint_head.safetensors").read_bytes()
before = sorted(p.name for p in out.iterdir())
with torch.no_grad():
for param in model.head.parameters():
param.add_(1.0)
# The backbone shards save, then the head write fails.
original = safetensors.torch.save_file
def failing_head(tensors, path, *args, **kwargs):
if str(path).endswith("joint_head.safetensors"):
raise OSError("disk full")
return original(tensors, path, *args, **kwargs)
monkeypatch.setattr(safetensors.torch, "save_file", failing_head)
with pytest.raises(OSError):
model.save_pretrained_merged(str(out))
assert sorted(p.name for p in out.iterdir()) == before
assert (out / "joint_head.safetensors").read_bytes() == head
FastDecisionModel.from_pretrained(str(out), max_seq_length = 512)
def test_structured_instructions_train_as_written():
question = {"type": "noul", "instructions": {"ask": "これは緊急ですか"}}
assert "これは緊急ですか" in _decision_common._internal(question)["ins"]
def test_kl_regularization_needs_a_lora_clef(clef_checkpoint, tmp_path):
model, processor = FastDecisionModel.from_pretrained(
str(clef_checkpoint), max_seq_length = 512, full_finetuning = True
)
with pytest.raises(NotImplementedError, match = "LoRA Clef"):
DecisionTrainer(model = model, args = _args(tmp_path), train_dataset = [], kl_weight = 0.1)
@pytest.mark.skipif(not has_real_cuda(), reason = "Unsloth's float16 path for Qwen3.5 needs a GPU")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_clef_trains_in_float16_and_bfloat16(clef_checkpoint, tmp_path, dtype):
if dtype == torch.bfloat16 and not torch.cuda.is_bf16_supported(including_emulation = False):
pytest.skip("this GPU has no bfloat16")
model, processor = FastDecisionModel.from_pretrained(
str(clef_checkpoint), max_seq_length = 512, dtype = dtype
)
# float16 puts Qwen3.5 on Unsloth's float32 path (bfloat16 weights, no autocast), as on a T4.
assert decision._clef_forced_float32(model) == (dtype == torch.float16)
model = FastDecisionModel.get_peft_model(model, r = 8, lora_alpha = 8)
items, _ = FastDecisionModel.build_dataset(_clef_rows(32), processor, model)
grads = []
class Grads(TrainerCallback):
def on_log(
self,
args,
state,
control,
logs = None,
**kwargs,
):
if logs and "loss" in logs:
grads.append((logs["loss"], logs.get("grad_norm")))
trainer = DecisionTrainer(
model = model,
tokenizer = processor,
train_dataset = items,
args = _args(
tmp_path,
max_steps = 6,
logging_steps = 1,
fp16 = dtype == torch.float16,
bf16 = dtype == torch.bfloat16,
),
callbacks = [Grads()],
)
assert trainer.args.fp16 is False and trainer.args.bf16 == (dtype == torch.bfloat16)
assert isinstance(trainer.accelerator.scaler, GRAD_SCALER) == (dtype == torch.float16)
trainer.train()
assert grads and all(math.isfinite(loss) for loss, _ in grads)
# The scaler skips the steps that overflow while it calibrates; they log a nan grad norm.
stepped = [g for _, g in grads if math.isfinite(g)]
assert len(stepped) * 2 >= len(grads) and all(g > 0 for g in stepped), grads
assert math.isfinite(FastDecisionModel.evaluate(model, processor, items)["loss"])
def test_clef_calibration_fits_being_right_and_serves_through_the_head_temperature():
# Underconfident logits: right 80% of the time with a soft gold that says 60%.
generator = torch.Generator().manual_seed(0)
logits, items = [], []
for i in range(400):
label = int(torch.randint(0, 3, (1,), generator = generator))
right = bool(torch.rand(1, generator = generator) < 0.8)
top = label if right else (label + 1) % 3
z = torch.zeros(3)
z[top] = 0.6
target = [0.2, 0.2, 0.2]
target[label] = 0.6
logits.append(z)
items.append({"label": label, "target": target, "qtype": i % 3, "row": i // 2})
config = {"temperature": [1.0] * 3}
plain = decision._metrics(logits, items, [1.0] * len(items))
calibrated = decision._calibrate_clef(config, logits, items)
assert calibrated["ece"] < plain["ece"] / 2 and calibrated["accuracy"] == plain["accuracy"]
assert config["head_temperature"] < 1.0
served = decision._served_temperatures(config, logits, items)
assert served[0] == pytest.approx(
config["head_temperature"]
* decision._laya().common.clamp_temperature(config["temperature"][0])
)
def test_clef_autocasts_as_it_trains_and_never_on_the_float32_path(monkeypatch):
model = torch.nn.Linear(1, 1)
cuda = torch.device("cuda")
monkeypatch.setattr(decision, "is_bfloat16_supported", lambda: True)
assert decision._clef_amp_dtype(model, cuda) == torch.bfloat16
model._unsloth_forced_float32 = True
assert decision._clef_amp_dtype(model, cuda) is None
model._unsloth_forced_float32 = False
# A T4: an fp16 model autocasts like its fp16 training, else its fp32 norms meet fp16 Linears.
monkeypatch.setattr(decision, "is_bfloat16_supported", lambda: False)
assert decision._clef_amp_dtype(model, cuda) == torch.float16
model._unsloth_forced_float32 = True
assert decision._clef_amp_dtype(model, cuda) is None
model._unsloth_forced_float32 = False
assert decision._clef_amp_dtype(model, torch.device("cpu")) is None
# An XPU without native bf16 loads fp16, so it autocasts fp16 too.
xpu = torch.device("xpu")
assert decision._amp_dtype(xpu) == torch.float16
monkeypatch.setattr(decision, "is_bfloat16_supported", lambda: True)
assert decision._amp_dtype(xpu) == torch.bfloat16
def test_clef_keeps_an_explicit_bf16_model_in_bf16_on_xpu(monkeypatch):
# The loaders keep an explicit bf16 request on an XPU without native bf16; autocast and the
# 4-bit compute dtype must follow it, never fp16 autocast over bf16 weights.
pytest.importorskip("transformers")
monkeypatch.setattr(decision, "is_bfloat16_supported", lambda: False)
xpu, cuda = torch.device("xpu"), torch.device("cuda")
model = torch.nn.Linear(1, 1).to(torch.bfloat16)
assert decision._clef_amp_dtype(model, xpu) == torch.bfloat16
assert decision._clef_amp_dtype(model, cuda) == torch.float16
assert decision._clef_amp_dtype(torch.nn.Linear(1, 1).half(), xpu) == torch.float16
model._unsloth_forced_float32 = True
assert decision._clef_amp_dtype(model, xpu) is None
monkeypatch.setattr(decision, "_device", lambda: xpu)
monkeypatch.setattr(decision, "xpu_keeps_explicit_bf16", lambda: True)
assert decision._clef_bnb_config(torch.bfloat16).bnb_4bit_compute_dtype == torch.bfloat16
assert decision._clef_bnb_config(None).bnb_4bit_compute_dtype == torch.float16
monkeypatch.setattr(decision, "_device", lambda: cuda)
assert decision._clef_bnb_config(torch.bfloat16).bnb_4bit_compute_dtype == torch.float16
def test_a_clef_prompt_that_fits_exactly_is_not_truncated(monkeypatch):
# Serving (Studio's Decision API) and predict share this rule.
from unsloth.models import clef
def encode(
tokenizer,
record,
max_length,
processor = None,
):
# The state needs `natural` tokens; anything over max_length is cut to fit.
return types.SimpleNamespace(input_ids = [0] * min(record["state"], max_length))
monkeypatch.setattr(clef, "encode_record", encode)
size = 16384
def truncated(natural):
encoded = encode(None, {"state": natural}, size)
return decision._clef_truncated(None, natural, {}, encoded, size)
assert (truncated(size - 1), truncated(size), truncated(size + 1)) == (False, False, True)
@pytest.mark.parametrize("forced", [False, True])
def test_clef_trains_under_the_autocast_it_evaluates_in(tmp_path, monkeypatch, forced):
# A previous trainer leaves fp16 in ACCELERATE_MIXED_PRECISION, which transformers 4.x reads.
monkeypatch.setenv("ACCELERATE_MIXED_PRECISION", "fp16")
monkeypatch.setattr(decision, "_amp_dtype", lambda device: torch.bfloat16)
model = torch.nn.Linear(1, 1)
model._unsloth_forced_float32 = forced
args = _args(tmp_path)
assert not args.bf16 and not args.fp16
decision._clef_mixed_precision(model, args)
expected = "no" if forced else "bf16"
assert args.bf16 == (not forced) and not args.fp16, (args.bf16, args.fp16)
assert decision.os.environ["ACCELERATE_MIXED_PRECISION"] == expected
assert getattr(args, "mixed_precision", expected) == expected
@pytest.mark.parametrize(
"case, wants",
[
("float32_path", True),
("deepspeed", False),
("fsdp", False),
("cpu", False),
("not_forced", False),
],
)
def test_clef_loss_scaler_only_on_the_float32_path(tmp_path, case, wants):
device = torch.device("cpu" if case == "cpu" else "cuda")
model = types.SimpleNamespace(
parameters = lambda: iter([types.SimpleNamespace(device = device)]),
_unsloth_forced_float32 = case != "not_forced",
)
args = _args(tmp_path)
if case == "deepspeed":
args.deepspeed = {"zero_optimization": {"stage": 2}}
if case == "fsdp":
args.fsdp = ["full_shard"]
with warnings.catch_warnings():
warnings.simplefilter("ignore") # GradScaler warns when this host has no CUDA
scaler = decision._clef_loss_scaler(model, args)
assert isinstance(scaler, GRAD_SCALER) == wants
def test_clef_loss_scaler_on_torch_without_torch_amp_grad_scaler(tmp_path, monkeypatch):
monkeypatch.delattr(torch.amp, "GradScaler", raising = False)
made = []
monkeypatch.setattr(torch.cuda.amp, "GradScaler", lambda: made.append(1) or "legacy")
for device, wants in (("cuda", "legacy"), ("xpu", None)):
model = types.SimpleNamespace(
parameters = lambda device = device: iter(
[types.SimpleNamespace(device = torch.device(device))]
),
_unsloth_forced_float32 = True,
)
assert decision._clef_loss_scaler(model, _args(tmp_path)) == wants
assert made == [1]
def test_clef_loss_scaler_unscales_once_before_clipping():
# A prior fp16 Trainer in the process can leave Accelerate on native fp16, which unscales itself.
calls = []
scaler = types.SimpleNamespace(unscale_ = lambda optimizer: calls.append(("unscale", optimizer)))
for native, wants in ((False, [("unscale", "opt"), "clip"]), (True, ["clip"])):
calls.clear()
accelerator = types.SimpleNamespace(
native_amp = native,
mixed_precision = "fp16" if native else "no",
clip_grad_norm_ = lambda p, m, n = 2: calls.append("clip"),
)
decision._install_clef_loss_scaler(accelerator, scaler, lambda: "opt")
assert accelerator.scaler is scaler
accelerator.clip_grad_norm_([], 1.0)
assert calls == wants
def test_clef_trains_under_fp16_autocast_on_a_gpu_without_bf16(tmp_path, monkeypatch):
# A T4: evaluate and serving autocast fp16, so training does too.
monkeypatch.setenv("ACCELERATE_MIXED_PRECISION", "no")
monkeypatch.setattr(decision, "_amp_dtype", lambda device: torch.float16)
model = torch.nn.Linear(1, 1)
model._unsloth_forced_float32 = False
args = _args(tmp_path)
decision._clef_mixed_precision(model, args)
assert args.fp16 and not args.bf16, (args.bf16, args.fp16)
assert decision.os.environ["ACCELERATE_MIXED_PRECISION"] == "fp16"
def test_clef_calibration_with_every_holdout_decision_from_one_row():
# A holdout row with many questions passes the item minimum with nothing to cross-validate on.
logits = [torch.tensor([1.0, 0.0, -1.0]) for _ in range(12)]
items = [{"label": i % 3, "target": [1 / 3] * 3, "qtype": i % 3, "row": 0} for i in range(12)]
config = {"temperature": [1.0] * 3}
calibrated = decision._calibrate_clef(config, logits, items)
assert math.isfinite(calibrated["ece"]) and "head_temperature" in config
def test_clef_save_keeps_the_source_tokenizer_files_byte_identical(clef_checkpoint, tmp_path):
model, processor = FastDecisionModel.from_pretrained(str(clef_checkpoint), max_seq_length = 512)
out = tmp_path / "out"
model.save_pretrained_merged(str(out))
for name in ("tokenizer.json", "tokenizer_config.json", "processor_config.json"):
if (clef_checkpoint / name).is_file():
assert (out / name).read_bytes() == (clef_checkpoint / name).read_bytes(), name
# Without it transformers warns of an "incorrect regex pattern" when loading the tokenizer.
assert "transformers_version" in json.loads((out / "config.json").read_text())
def test_decision_forward_never_picks_cudnn_attention(checkpoint, tmp_path):
model, tokenizer = FastDecisionModel.from_pretrained(
str(checkpoint), full_finetuning = True, use_gradient_checkpointing = False
)
items, _ = FastDecisionModel.build_dataset([_row(i) for i in range(4)], tokenizer, model)
cudnn = []
forward = model.forward
def spy(*args, **kwargs):
cudnn.append(torch.backends.cuda.cudnn_sdp_enabled())
return forward(*args, **kwargs)
model.forward = spy
trainer = DecisionTrainer(
model = model, args = _args(tmp_path), train_dataset = items, processing_class = tokenizer
)
device = next(model.parameters()).device
batch = {
k: v.to(device) for k, v in DecisionDataCollator(tokenizer.pad_token_id)(items).items()
}
trainer.compute_loss(model, batch)
decision._logits(model, items, tokenizer.pad_token_id)
assert cudnn and not any(cudnn)
def test_checkpointed_recompute_never_picks_cudnn_attention(checkpoint, tmp_path):
# The recompute runs in backward: cuDNN there saved other tensors than the forward (CheckpointError).
model, tokenizer = FastDecisionModel.from_pretrained(
str(checkpoint), full_finetuning = True, use_gradient_checkpointing = True
)
items, _ = FastDecisionModel.build_dataset([_row(i) for i in range(4)], tokenizer, model)
layer = model.encoder.layers[0]
cudnn = []
forward = layer.forward
def spy(*args, **kwargs):
cudnn.append(torch.backends.cuda.cudnn_sdp_enabled())
return forward(*args, **kwargs)
layer.forward = spy
trainer = DecisionTrainer(
model = model, args = _args(tmp_path), train_dataset = items, processing_class = tokenizer
)
device = next(model.parameters()).device
batch = {
k: v.to(device) for k, v in DecisionDataCollator(tokenizer.pad_token_id)(items).items()
}
trainer.accelerator.backward(trainer.compute_loss(model, batch))
assert len(cudnn) == 2 and not any(cudnn)
def test_logits_batch_similar_lengths_and_keep_the_callers_order(checkpoint):
model, tokenizer = FastDecisionModel.from_pretrained(
str(checkpoint), full_finetuning = True, use_gradient_checkpointing = False
)
items, _ = FastDecisionModel.build_dataset([_row(i) for i in range(4)], tokenizer, model)
assert len({len(item["input_ids"]) for item in items}) > 1
widths = []
forward = model.forward
def spy(*args, **kwargs):
widths.append(kwargs["input_ids"].shape[1])
return forward(*args, **kwargs)
model.forward = spy
batched = decision._logits(model, items, tokenizer.pad_token_id, batch_size = 3)
assert widths == sorted(widths)
model.forward = forward
for item, logits in zip(items, batched):
alone = decision._logits(model, [item], tokenizer.pad_token_id)[0]
assert logits.shape == (len(item["markers"]),)
torch.testing.assert_close(logits, alone, rtol = 2e-2, atol = 2e-2)
@pytest.mark.parametrize("scaling", [1, 2])
def test_lean_lora_forward_matches_peft(checkpoint, scaling):
model, _ = FastDecisionModel.from_pretrained(
str(checkpoint), dtype = torch.float32, use_gradient_checkpointing = False
)
model = FastDecisionModel.get_peft_model(model, r = 4, lora_alpha = 4 * scaling)
layers = [m for m in model.encoder.modules() if hasattr(m, "_unsloth_peft_forward")]
from peft.tuners.lora.layer import Linear
if Linear.forward.__name__ == "unsloth_forward":
# An earlier FastModel load in this process compiled PEFT's forward; the lean one stays out.
assert not layers
return
assert layers and all(m.forward.__func__ is decision._lean_lora_forward for m in layers)
layer = layers[0]
torch.manual_seed(0)
with torch.no_grad():
layer.lora_B["default"].weight.normal_()
device = layer.lora_A["default"].weight.device
autocast = [False] + ([True] if device.type == "cuda" else [])
for enabled in autocast:
x = torch.randn(2, 5, layer.in_features, device = device, requires_grad = True)
outputs = []
for forward in (layer.forward, layer._unsloth_peft_forward):
# The dtype the model trains in on this GPU: fp16 on a T4, which cannot run bf16.
with torch.autocast(device.type, dtype = decision._amp_dtype(device), enabled = enabled):
out = forward(x)
grads = torch.autograd.grad(
out.float().square().sum(), [x, layer.lora_A["default"].weight]
)
outputs.append((out, *grads))
for lean, peft in zip(*outputs):
assert lean.dtype == peft.dtype and torch.equal(lean, peft)
# Disabled or merged adapters take PEFT's own path.
with model.encoder.disable_adapter():
x = torch.randn(2, 5, layer.in_features, device = device)
torch.testing.assert_close(layer(x), layer.base_layer(x))
def test_compiled_layers_use_plain_sdpa_and_put_the_attention_back(checkpoint):
from transformers.integrations.sdpa_attention import sdpa_attention_forward
from unsloth.models import _decision_fast
model, _ = FastDecisionModel.from_pretrained(
str(checkpoint), full_finetuning = True, use_gradient_checkpointing = False
)
config = model.encoder.config
assert config._attn_implementation == "sdpa"
with _decision_fast._compilable_attention(model, False):
assert config._attn_implementation == "sdpa"
with _decision_fast._compilable_attention(model, True):
assert config._attn_implementation == "unsloth_decision_sdpa"
assert config._attn_implementation == "sdpa"
q, k, v = (torch.randn(2, 4, 6, 8) for _ in range(3))
mask = torch.ones(2, 1, 6, 6, dtype = torch.bool)
mask[1, ..., 4:] = False
module = model.encoder.layers[0].attn
ours, _ = _decision_fast._encoder_sdpa(module, q, k, v, mask, scaling = 0.5)
reference, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 0.5)
torch.testing.assert_close(ours, reference)
def test_static_length_padding_leaves_the_loss_unchanged(checkpoint, tmp_path):
from unsloth.models import _decision_fast
model, tokenizer = FastDecisionModel.from_pretrained(
str(checkpoint), full_finetuning = True, use_gradient_checkpointing = False
)
for module in model.modules():
if isinstance(module, torch.nn.Dropout):
module.p = 0.0
if isinstance(module, torch.nn.MultiheadAttention):
module.dropout = 0.0
items, _ = FastDecisionModel.build_dataset([_row(i) for i in range(4)], tokenizer, model)
trainer = DecisionTrainer(
model = model, args = _args(tmp_path), train_dataset = items, processing_class = tokenizer
)
device = next(model.parameters()).device
batch = DecisionDataCollator(tokenizer.pad_token_id)(items)
assert batch["input_ids"].shape[1] % _decision_fast.STATIC_MULTIPLE
losses = []
for multiple in (0, _decision_fast.STATIC_MULTIPLE):
model._unsloth_pad_multiple = multiple
inputs = {k: v.to(device) for k, v in batch.items()}
padded = _decision_fast.pad_length(model, dict(inputs))
assert padded["input_ids"].shape[1] % (multiple or 1) == 0
with torch.no_grad():
losses.append(trainer.compute_loss(model, inputs))
model._unsloth_pad_multiple = 0
torch.testing.assert_close(losses[0], losses[1])
def test_static_compiled_layers_run_eagerly_outside_training():
from unsloth.models import _decision_fast
calls = []
layer = torch.nn.Linear(2, 2)
call = _decision_fast._training_only(
layer, lambda *a, **k: calls.append("compiled") or layer._call_impl(*a, **k)
)
x = torch.randn(1, 2)
call(x)
layer.eval()
call(x)
assert calls == ["compiled"]
@pytest.mark.parametrize("max_length, static", [(634, True), (4096, False), (None, False)])
def test_short_data_compiles_static_buckets_and_long_data_dynamic(
checkpoint, monkeypatch, max_length, static
):
from unsloth.models import _decision_fast
monkeypatch.setenv("UNSLOTH_DECISION_COMPILE", "1")
monkeypatch.setattr(_decision_fast, "_warm_up", lambda model, amp_dtype: None)
model, _ = FastDecisionModel.from_pretrained(
str(checkpoint), full_finetuning = True, use_gradient_checkpointing = False
)
layers = _decision_fast._encoder_layers(model)
limit = torch._dynamo.config.recompile_limit
with _decision_fast.compiled_encoder(model, 10, None, max_length) as compiled:
assert compiled
assert model._unsloth_pad_multiple == (_decision_fast.STATIC_MULTIPLE if static else 0)
assert all(layer._compiled_call_impl is not None for layer in layers)
assert all((layer._compiled_call_impl.__name__ == "call") == static for layer in layers)
assert (torch._dynamo.config.recompile_limit > limit) == (static and limit < 28)
assert all(layer._compiled_call_impl is None for layer in layers)
assert model._unsloth_pad_multiple == 0
assert torch._dynamo.config.recompile_limit == limit