Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
[perf] Qwen3.5-9B LoRA SFT with unsloth-cli.py on a B200: 0.84 s to 0.77 s per step at 1 GPU, the model-independent half of #10744 (#11238)
Hexu Zhaoa801caee39d6
diff --git a/tests/test_fla_autotuner_fast_path.py b/tests/test_fla_autotuner_fast_path.pynew file mode 100644index 00000000000..733293af56b--- /dev/null+++ b/tests/test_fla_autotuner_fast_path.py@@ -0,0 +1,262 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""PR #11238, commit 7b45cc8f9: `patch_fla_autotuner_fast_path` rebinds+`fla.ops.utils.cache.CachedAutotuner.run` to launch with a latched config.++Each test compares the patched path against the ORIGINAL `CachedAutotuner.run` on the same+inputs, so "same kernel, same config, same numerics" is measured, not asserted.++ PYTHONPATH=<tree> CUDA_VISIBLE_DEVICES=<n> pytest -q test_fla_autotuner_fast_path.py+"""++import importlib++import pytest++# These runners do not all ship torch/triton/fla; skip rather than erroring at collection.+torch = pytest.importorskip("torch")+triton = pytest.importorskip("triton")+tl = pytest.importorskip("triton.language")+fla_cache = pytest.importorskip("fla.ops.utils.cache")+U = pytest.importorskip("unsloth.models._utils")++PATCHER = getattr(U, "patch_fla_autotuner_fast_path", None)+needs_patch = pytest.mark.skipif(+ PATCHER is None, reason = "base tree: no patch_fla_autotuner_fast_path"+)+CUDA = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a GPU")+++class _ReuseBestCache(dict):+ """Byte-copy of unsloth_zoo/compiler.py:4808 so the tests reproduce the real shape."""++ def __contains__(self, key):+ return len(self) > 0 or super().__contains__(key)++ def __getitem__(self, key):+ if not super().__contains__(key) and len(self) > 0:+ return next(iter(self.values()))+ return super().__getitem__(key)+++def _add_kernel(x_ptr, y_ptr, o_ptr, n, BLOCK: tl.constexpr):+ pid = tl.program_id(0)+ offs = pid * BLOCK + tl.arange(0, BLOCK)+ mask = offs < n+ tl.store(+ o_ptr + offs, tl.load(x_ptr + offs, mask = mask) + tl.load(y_ptr + offs, mask = mask), mask = mask+ )+++def _make(+ configs,+ key = ("n",),+ pre_hook = None,+):+ from fla.ops.utils.cache import fla_cache_autotune+ if pre_hook is not None:+ configs = [+ triton.Config(+ c.kwargs, num_warps = c.num_warps, num_stages = c.num_stages, pre_hook = pre_hook+ )+ for c in configs+ ]+ return fla_cache_autotune(configs = configs, key = list(key))(_add_kernel)+++CFGS2 = [triton.Config({"BLOCK": 64}, num_warps = 2), triton.Config({"BLOCK": 128}, num_warps = 4)]+CFGS1 = [triton.Config({"BLOCK": 128}, num_warps = 4)]+++def _grid(n):+ # A meta-dependent grid, as every real fla kernel uses. A hardcoded grid would silently+ # under-cover the tensor whenever the tuned BLOCK is smaller than the one assumed here,+ # which makes the harness, not the patch, the source of any mismatch.+ return lambda meta: (triton.cdiv(n, meta["BLOCK"]),)+++def _run(kern, n = 4096):+ x = torch.randn(n, device = "cuda")+ y = torch.randn(n, device = "cuda")+ o = torch.empty_like(x)+ kern[_grid(n)](x, y, o, n)+ return x, y, o+++def _unpatch(cls, original):+ cls.run = original+++def fresh():+ """Reload the module so each test starts from an unpatched CachedAutotuner."""+ importlib.reload(fla_cache)+ orig = fla_cache.CachedAutotuner.run+ yield fla_cache.CachedAutotuner+ fla_cache.CachedAutotuner.run = orig+++# --------------------------------------------------------------------------- install+@needs_patch+def test_patch_installs_once_and_is_idempotent(fresh):+ assert not getattr(fresh.run, "_unsloth_fast_path", False)+ PATCHER()+ first = fresh.run+ assert getattr(first, "_unsloth_fast_path", False) is True+ PATCHER()+ assert fresh.run is first, "a second call must not stack another wrapper"+++@needs_patch+def test_patch_declines_under_FLA_CACHE_MODE_always(fresh, monkeypatch):+ class _Mode:+ value = "always"++ monkeypatch.setattr(fla_cache, "FLA_CACHE_MODE", _Mode(), raising = False)+ PATCHER()+ assert not getattr(+ fresh.run, "_unsloth_fast_path", False+ ), "the debug mode re-reads config files on every launch; the fast path must stand down"+++# --------------------------------------------------------------------------- behaviour+@CUDA+@needs_patch+def test_multi_config_plain_dict_cache_falls_back(fresh):+ kern = _make(CFGS2)+ PATCHER()+ _run(kern)+ at = kern+ while not hasattr(at, "configs"):+ at = at.fn+ assert type(at.cache) is not _ReuseBestCache+ assert (+ getattr(at, "_unsloth_fixed_config", None) is None+ ), "without _ReuseBestCache the autotuner must keep triton's own lookup"+++@CUDA+@needs_patch+def test_reuse_best_cache_fast_path_matches_the_original_bit_for_bit(fresh):+ kern_a = _make(CFGS2)+ kern_b = _make(CFGS2)+ # settle both the same way with the ORIGINAL run+ _run(kern_a)+ _run(kern_b)+ at_a, at_b = kern_a, kern_b+ while not hasattr(at_a, "configs"):+ at_a = at_a.fn+ while not hasattr(at_b, "configs"):+ at_b = at_b.fn+ at_a.cache = _ReuseBestCache(at_a.cache)+ at_b.cache = _ReuseBestCache(at_b.cache)++ n = 4096+ x = torch.randn(n, device = "cuda")+ y = torch.randn(n, device = "cuda")+ o_ref = torch.empty_like(x)+ o_fast = torch.empty_like(x)+ kern_a[_grid(n)](x, y, o_ref, n) # unpatched++ PATCHER()+ kern_b[_grid(n)](x, y, o_fast, n) # patched++ assert getattr(at_b, "_unsloth_fixed_config", None) is not None, "fast path did not latch"+ assert at_b._unsloth_fixed_config.all_kwargs() == at_a.best_config.all_kwargs()+ assert at_b.best_config is at_b._unsloth_fixed_config+ assert torch.equal(o_ref, o_fast)+ assert torch.equal(o_ref, x + y)+++@CUDA+@needs_patch+def test_single_config_fast_path_first_and_subsequent_launches(fresh):+ kern = _make(CFGS1)+ PATCHER()+ x, y, o = _run(kern)+ assert torch.equal(o, x + y)+ at = kern+ while not hasattr(at, "configs"):+ at = at.fn+ assert at._unsloth_fixed_config is at.configs[0]+ x2, y2, o2 = _run(kern) # second launch uses the latch+ assert torch.equal(o2, x2 + y2)+++@CUDA+@needs_patch+def test_per_config_pre_hook_falls_back_and_the_hook_still_fires(fresh):+ fired = []+ kern = _make(CFGS1, pre_hook = lambda nargs: fired.append(1))+ PATCHER()+ _run(kern)+ _run(kern)+ at = kern+ while not hasattr(at, "configs"):+ at = at.fn+ assert getattr(at, "_unsloth_fixed_config", None) is None+ assert len(fired) >= 2, "a per-Config pre_hook must run on every launch"+++# --------------------------------------------------------------------------- absence+def test_patch_is_a_no_op_when_fla_is_missing(monkeypatch):+ """fla < 0.5.1 has no fla.ops.utils.cache; the patch must not raise."""+ if PATCHER is None:+ pytest.skip("base tree")+ import builtins++ real_import = builtins.__import__++ def _blocked(name, *a, **kw):+ if name.startswith("fla"):+ raise ImportError("no fla")+ return real_import(name, *a, **kw)++ monkeypatch.setattr(builtins, "__import__", _blocked)+ PATCHER() # must not raise+++# --------------------------------------------------------------------------- cache probe+CACHE_PROBE = getattr(U, "_unsloth_cache_reuses_one_config", None)+needs_probe = pytest.mark.skipif(CACHE_PROBE is None, reason = "tree without the behaviour probe")+++@needs_probe+def test_cache_probe_detects_the_reuse_cache_under_any_name():+ """unsloth_zoo defines _ReuseBestCache inside a function, so there is no symbol to import.+ The probe must key off behaviour, not the class name, or a rename silently turns the fast+ path off."""++ class RenamedTomorrow(_ReuseBestCache):+ pass++ for cls in (_ReuseBestCache, RenamedTomorrow):+ c = cls()+ assert CACHE_PROBE(c) is False, "an empty reuse cache reuses nothing yet"+ c["k"] = object()+ assert CACHE_PROBE(c) is True+++@needs_probe+def test_cache_probe_rejects_a_plain_dict_and_anything_that_raises():+ assert CACHE_PROBE({}) is False+ assert CACHE_PROBE({"k": object()}) is False++ class Hostile(dict):+ def __contains__(self, key):+ raise RuntimeError("no")++ h = Hostile()+ h["k"] = object()+ assert CACHE_PROBE(h) is False, "a cache that raises must fall back, not propagate"+++@needs_probe+def test_cache_probe_does_not_mutate_the_cache():+ c = _ReuseBestCache()+ c["k"] = object()+ before = dict(c)+ CACHE_PROBE(c)+ assert dict(c) == before and len(c) == 1diff --git a/tests/test_train_mode_asserted_once.py b/tests/test_train_mode_asserted_once.pynew file mode 100644index 00000000000..8789a8a0569--- /dev/null+++ b/tests/test_train_mode_asserted_once.py@@ -0,0 +1,182 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Behavioural difference between calling `model.train()` every micro-step and asserting+train mode once, which is the only thing that can make that optimisation unsafe.+"""++import copy++import pytest++# These runners do not all ship torch; skip the module rather than erroring at collection.+pytest.importorskip("torch")+nn = pytest.importorskip("torch.nn")+U = pytest.importorskip("unsloth.models._utils")++HELPER = getattr(U, "_unsloth_train_if_needed", None)+needs_helper = pytest.mark.skipif(HELPER is None, reason = "base tree: no _unsloth_train_if_needed")+++class _Tree(nn.Module):+ """Root with a deep child, so a submodule can be flipped on its own."""++ def __init__(self):+ super().__init__()+ self.block = nn.Sequential(nn.Linear(4, 4), nn.Dropout(0.5))++ def forward(self, x):+ return self.block(x)+++class _Wrapper(nn.Module):+ """DDP shape: `training_step` gets this, other code holds `.module`."""++ def __init__(self, inner):+ super().__init__()+ self.module = inner++ def forward(self, x):+ return self.module(x)+++def _modes(m):+ return {n or "<root>": mod.training for n, mod in m.named_modules()}+++@needs_helper+def test_first_call_asserts_train_mode_on_every_module():+ m = _Tree()+ m.eval()+ assert not any(_modes(m).values())+ HELPER(m)+ assert all(_modes(m).values())+ assert getattr(m, "_unsloth_train_mode_asserted", False) is True+++@needs_helper+def test_root_eval_rearms_the_walk():+ m = _Tree()+ HELPER(m)+ m.eval()+ assert m.training is False+ HELPER(m)+ assert all(_modes(m).values()), "a root .eval() must be repaired by the next micro-step"+++@needs_helper+def test_marker_is_not_in_state_dict_and_does_not_survive_a_fresh_module():+ m = _Tree()+ HELPER(m)+ assert "_unsloth_train_mode_asserted" not in m.state_dict()+ fresh = _Tree()+ assert getattr(fresh, "_unsloth_train_mode_asserted", None) is None+ clone = copy.deepcopy(m)+ assert clone.training == m.training+++def test_reference_submodule_only_eval_is_repaired_by_stock_train():+ """What the stock per-micro-step `model.train()` does. Passes on both arms."""+ m = _Tree()+ m.train()+ m.block[1].eval()+ assert m.block[1].training is False+ m.train()+ assert m.block[1].training is True+++@needs_helper+def test_submodule_only_eval_is_NOT_repaired_by_the_helper():+ """Documented semantic change: a submodule flipped to eval on its own keeps that mode."""+ m = _Tree()+ HELPER(m)+ m.block[1].eval()+ HELPER(m)+ assert m.block[1].training is False, "if this repairs, the PR body's claim is wrong"+++def test_reference_wrapper_inner_eval_is_repaired_by_stock_train():+ inner = _Tree()+ w = _Wrapper(inner)+ w.train()+ inner.eval()+ assert w.training is True and inner.training is False+ w.train()+ assert inner.training is True+ assert all(_modes(w).values())+++@needs_helper+ getattr(U, "_unsloth_wrappees_are_in_train_mode", None) is not None,+ reason = "the wrappee check repairs this; kept to document what it repairs",+)+def test_wrapper_inner_eval_is_permanently_stale_under_the_helper():+ """A wrapper's own `.training` never flips when the module inside it is eval'd, so a+ root-flag check strands the whole model in eval for the rest of the run."""+ inner = _Tree()+ w = _Wrapper(inner)+ HELPER(w)+ inner.eval()+ for _ in range(100):+ HELPER(w)+ assert w.training is True+ assert inner.training is False+ assert w.module.block[1].training is False+++@needs_helper+def test_helper_survives_a_model_that_rejects_attribute_assignment():+ class _Frozen(_Tree):+ def __setattr__(self, k, v):+ if k == "_unsloth_train_mode_asserted":+ raise AttributeError("read-only")+ super().__setattr__(k, v)++ m = _Frozen()+ HELPER(m)+ assert m.training is True+ HELPER(m)+ assert m.training is True+++WRAPPEE_CHECK = getattr(U, "_unsloth_wrappees_are_in_train_mode", None)+needs_fix = pytest.mark.skipif(WRAPPEE_CHECK is None, reason = "tree without the wrappee check")+++@needs_fix+def test_wrapper_inner_eval_is_repaired_once_wrappees_are_checked():+ """DDP + eval_strategy: evaluation_loop evals `self.model`, training_step gets+ `self.model_wrapped`."""+ inner = _Tree()+ w = _Wrapper(inner)+ HELPER(w)+ inner.eval()+ HELPER(w)+ assert inner.training is True+ assert all(_modes(w).values())+++@needs_fix+def test_wrappee_check_is_cheap_and_stops_at_the_first_non_wrapper():+ m = _Tree()+ HELPER(m)+ assert WRAPPEE_CHECK(m) is True+ calls = []+ orig = nn.Module.train+ try:+ nn.Module.train = lambda self, mode = True: calls.append(1) or orig(self, mode)+ HELPER(m)+ finally:+ nn.Module.train = orig+ assert calls == [], "an unwrapped model must still skip the walk"+++@needs_fix+def test_wrappee_check_follows_a_nested_wrapper_chain():+ inner = _Tree()+ w = _Wrapper(_Wrapper(inner))+ HELPER(w)+ assert WRAPPEE_CHECK(w) is True+ inner.eval()+ assert WRAPPEE_CHECK(w) is Falsediff --git a/tests/version_compat/test_transformers_pinned_symbols.py b/tests/version_compat/test_transformers_pinned_symbols.pyindex 96d91aa2b62..b0188ccd414 100644--- a/tests/version_compat/test_transformers_pinned_symbols.py+++ b/tests/version_compat/test_transformers_pinned_symbols.py@@ -332,3 +332,25 @@ def test_training_args_parallel_mode_importable(tag: str):f"{tag}: transformers.training_args.ParallelMode missing; "f"unsloth-zoo loss_utils.py:232 ImportError")+++def test_trainer_training_step_model_train_call_is_standalone(tag: str):+ """unsloth#11238 rewrites the FIRST `model.train()` inside Trainer.training_step into+ `_unsloth_train_if_needed(model)`. A release that wrote `self.model.train()` earlier in that+ method would turn the same replace into `self._unsloth_train_if_needed(model)`, and every+ training step would die with AttributeError. Nothing else in the rewrite guards that."""+ candidates = ["src/transformers/trainer.py", "src/transformers/trainer/__init__.py"]+ hit = first_match("huggingface/transformers", tag, candidates)+ assert hit is not None+ _, src = hit+ m = re.search(r"\n def training_step\(.*?\n def ", src, re.DOTALL)+ if m is None:+ pytest.skip(f"{tag}: Trainer.training_step not found")+ body = m.group(0)+ idx = body.find("model.train()")+ if idx < 0:+ pytest.skip(f"{tag}: Trainer.training_step no longer calls model.train()")+ assert body[max(0, idx - 11) : idx] != "self.model.", (+ f"{tag}: the first `model.train()` in Trainer.training_step is `self.model.train()`; "+ f"unsloth/models/_utils.py would rewrite it into `self._unsloth_train_if_needed(model)`"+ )diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 9df944f352a..a0c479dfac1 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -3169,10 +3169,128 @@ def _unsloth_pre_compute_loss(self, model, inputs, *args, **kwargs):return outputs+# The attributes a training wrapper keeps its wrappee under. DDP, FSDP1 and DataParallel use+# `module`; `torch.compile` uses `_orig_mod`; FSDP's own wrapper policy uses `_fsdp_wrapped_module`.+_UNSLOTH_WRAPPED_MODULE_ATTRS = ("module", "_orig_mod", "_fsdp_wrapped_module")+++def _unsloth_wrappees_are_in_train_mode(model):+ """Whether every module `model` merely wraps also reports training mode.++ `training_step` is handed `self.model_wrapped`, but `Trainer.evaluation_loop` calls `.eval()`+ on `self._wrap_model(self.model, training=False)`, which under DDP is the module INSIDE the+ wrapper. That leaves the wrapper's own `.training` True while the whole model sits in eval,+ so the root flag alone cannot tell the two apart and training would silently continue with+ every dropout disabled. This is a handful of attribute reads, not the module walk being+ skipped, and on an unwrapped model it stops at the first miss.+ """+ inner = model+ for _ in range(4):+ for attr in _UNSLOTH_WRAPPED_MODULE_ATTRS:+ wrappee = getattr(inner, attr, None)+ if isinstance(wrappee, torch.nn.Module):+ break+ else:+ return True+ if not wrappee.training:+ return False+ inner = wrappee+ return True+++def _unsloth_train_if_needed(model):+ """`Trainer.training_step` calls `model.train()` on every micro-step. On a PEFT-wrapped 9B+ model that is a recursive walk over ~2k modules with a `__setattr__` each, several ms of+ pure Python per micro-step while the GPU waits. The mode only has to be asserted once: skip+ the walk when the root already reports training mode and we were the ones who set it. A root+ `.eval()` (evaluation, `for_inference`) flips `model.training`, so the next call walks again.+ """+ if (+ model.training+ and getattr(model, "_unsloth_train_mode_asserted", False)+ and _unsloth_wrappees_are_in_train_mode(model)+ ):+ return model+ model.train()+ try:+ model._unsloth_train_mode_asserted = True+ except Exception:+ pass+ return model+++def _unsloth_cache_reuses_one_config(cache):+ """Whether this autotuner cache answers every key with its one stored config.++ That is what unsloth_zoo's `compile_fla_no_autotune` installs, but its `_ReuseBestCache` is+ defined inside that function, so there is no symbol to import and no class to isinstance+ against. Probe the behaviour instead of the class name: `dict.keys()` membership always uses+ dict's own lookup, so a key absent from the raw mapping yet reported present by `in` is+ exactly the reuse behaviour, under any name a future rewrite gives it.+ """+ try:+ if len(cache) == 0:+ return False+ probe = ("__unsloth_probe__",) * 2+ return probe not in cache.keys() and probe in cache+ except Exception:+ return False+++def patch_fla_autotuner_fast_path():+ """unsloth_zoo's `compile_fla_no_autotune` makes every fla Triton autotuner reuse its first+ tuned config for every key (`_ReuseBestCache`). After that, fla's `CachedAutotuner.run` still+ builds its own `AutotuneKey` (dict zips, dtype strings, JSON-able normalisation) and then+ Triton's `Autotuner.run` builds a key a second time, on every launch, for a lookup whose answer+ is always the same config. Linear-attention layers make thousands of such launches per optimizer+ step. Once the config is settled, launch the kernel with it directly: same config, same kernel,+ same numerics.+ """+ try:+ import fla.ops.utils.cache as fla_cache+ except Exception:+ return+ CachedAutotuner = getattr(fla_cache, "CachedAutotuner", None)+ if CachedAutotuner is None or getattr(CachedAutotuner.run, "_unsloth_fast_path", False):+ return+ # FLA_CACHE_MODE=always re-reads the config files on every launch on purpose (a debug mode).+ cache_mode = getattr(fla_cache, "FLA_CACHE_MODE", None)+ if getattr(cache_mode, "value", None) == "always":+ return+ original_run = CachedAutotuner.run++ @functools.wraps(original_run)+ def run(self, *args, **kwargs):+ cfg = getattr(self, "_unsloth_fixed_config", None)+ if cfg is None:+ cache = self.cache+ if len(self.configs) == 1:+ cfg = self.configs[0]+ elif _unsloth_cache_reuses_one_config(cache):+ cfg = next(iter(cache.values()))+ else:+ return original_run(self, *args, **kwargs)+ if cfg.pre_hook is not None:+ return original_run(self, *args, **kwargs)+ self._unsloth_fixed_config = cfg+ self._unsloth_fixed_kwargs = cfg.all_kwargs()+ self.best_config = cfg+ return self.fn.run(*args, **kwargs, **self._unsloth_fixed_kwargs)++ run._unsloth_fast_path = True+ CachedAutotuner.run = run++def patch_gradient_accumulation_fix(Trainer):# Fixes "Output 0 of UnslothFusedLossBackward is a view and is being modified inplace" and gradient accumulation.import inspect+ # Before the early returns below: the fla patch is unrelated to gradient accumulation, and it+ # can only install once fla has been imported. Loading a non-fla model first (llama.py's+ # FastLlamaModel.from_pretrained never imports fla) and a gated-deltanet model second would+ # otherwise leave it permanently uninstalled, since the second call returns here.+ patch_fla_autotuner_fast_path()+if hasattr(Trainer, "get_batch_samples"):if Trainer.get_batch_samples.__name__ == "_unsloth_get_batch_samples":return@@ -3254,6 +3372,9 @@ def patch_gradient_accumulation_fix(Trainer):)function = function.replace("def training_step", "def _unsloth_training_step", 1)+ # Skip the per-micro-step recursive `model.train()` walk once train mode is set.+ function = function.replace("model.train()", "_unsloth_train_if_needed(model)", 1)+# Fix 4.47.0 removing num_items_in_batch (huggingface/transformers#35121) and the case where it is nothing (huggingface/transformers#35207).function = function.replace("if self.model_accepts_loss_kwargs:",