Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Qwen3.5: skip compiled regions on eager decode steps, make CUDA graph decode opt-in (#12213)
Daniel Han34f345543188
diff --git a/tests/test_compile_decode_selection.py b/tests/test_compile_decode_selection.pynew file mode 100644index 00000000000..78f686b5586--- /dev/null+++ b/tests/test_compile_decode_selection.py@@ -0,0 +1,276 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Which models compile their decode step, and the static cache length they get."""++import inspect+import types++import pytest++import unsloth.models.vision as vision+import contextlib++from unsloth.models.vision import (+ _CompileDecodeOnRepeat,+ _EagerDecodeSteps,+ _compiles_decode,+ _decode_cache_bucket,+ _eager_decodes,+ _match_compiled_call,+)+++def _model(model_type, text_type = None):+ config = types.SimpleNamespace(model_type = model_type)+ if text_type is not None:+ config.text_config = types.SimpleNamespace(model_type = text_type)+ return types.SimpleNamespace(config = config, generation_config = types.SimpleNamespace())++[email protected](autouse = True)+def _zoo_supports_it(monkeypatch):+ monkeypatch.setattr(vision, "unsloth_decode_compile", object())+ monkeypatch.setenv("UNSLOTH_COMPILE_DECODE", "1")+ monkeypatch.delenv("UNSLOTH_COMPILE_DISABLE", raising = False)+ monkeypatch.delenv("UNSLOTH_EAGER_DECODE", raising = False)+++def test_decode_compile_is_opt_in(monkeypatch):+ monkeypatch.delenv("UNSLOTH_COMPILE_DECODE")+ assert not _compiles_decode(_model("qwen3_5", "qwen3_5_text"))+++ "model",+ [+ _model("qwen3_5", "qwen3_5_text"),+ _model("qwen3_5_moe", "qwen3_5_moe_text"),+ _model("qwen3_5_text"),+ _model("qwen3_5_moe_text"),+ ],+)+def test_qwen3_5_compiles_decode(model):+ assert _compiles_decode(model)+++ "model",+ [+ _model("llama"),+ _model("gemma3", "gemma3_text"),+ _model("qwen3_next"),+ _model("qwen3_vl", "qwen3_vl_text"),+ ],+)+def test_other_models_keep_eager_decode(model):+ assert not _compiles_decode(model)+++ "env",+ [+ ("UNSLOTH_COMPILE_DECODE", "0"),+ ("UNSLOTH_COMPILE_DISABLE", "1"),+ ("UNSLOTH_COMPILE_DISABLE", "partial"),+ ],+)+def test_opt_outs(monkeypatch, env):+ monkeypatch.setenv(*env)+ assert not _compiles_decode(_model("qwen3_5", "qwen3_5_text"))+++def test_uncompileable_quantizer_keeps_eager_decode():+ model = _model("qwen3_5", "qwen3_5_text")+ model.hf_quantizer = types.SimpleNamespace(is_compileable = False)+ assert not _compiles_decode(model)+ model.hf_quantizer = types.SimpleNamespace(is_compileable = True)+ assert _compiles_decode(model)++[email protected]("device", ["cpu", "disk"])+def test_offloaded_model_keeps_eager_decode(device):+ model = _model("qwen3_5", "qwen3_5_text")+ model.hf_device_map = {"model.layers.0": 0, "model.layers.1": device}+ assert not _compiles_decode(model)+ model.hf_device_map = {"model.layers.0": 0, "model.layers.1": 1}+ assert _compiles_decode(model)+++def test_old_zoo_keeps_eager_decode(monkeypatch):+ monkeypatch.setattr(vision, "unsloth_decode_compile", None)+ assert not _compiles_decode(_model("qwen3_5", "qwen3_5_text"))+++ "length, bucket", [(74, 1024), (1024, 1024), (1025, 2048), (1501, 2048), (3100, 4096)]+)+def test_bucket_rounds_up(length, bucket):+ assert _decode_cache_bucket(length) == bucket+++class _FakeGenerating:+ """Stands in for the model: records the length transformers would allocate and whether+ it would compile the decode step."""++ def __init__(self):+ self.allocated = []++ def _prepare_static_cache(+ self, cache_implementation, batch_size, max_cache_len, *rest, **kwargs+ ):+ self.allocated.append(max_cache_len)+ return types.SimpleNamespace(max_cache_len = max_cache_len)++ def _valid_auto_compile_criteria(self, model_kwargs, generation_config):+ return True+++def _generate(+ model,+ scopes,+ batch_size,+ length,+ positional = False,+):+ with _CompileDecodeOnRepeat(model):+ if positional: # transformers 5.2 - 5.5 call shape+ model._prepare_static_cache("static", batch_size, length, {})+ else: # 5.17 passes keywords+ model._prepare_static_cache(+ cache_implementation = "static",+ batch_size = batch_size,+ max_cache_len = length,+ prefill_chunk_size = None,+ model_kwargs = {},+ )+ compiled = model._valid_auto_compile_criteria({}, None)+ return compiled, len(scopes)+++def scopes(monkeypatch):+ entered = []++ @contextlib.contextmanager+ def fake_scope():+ entered.append(1)+ yield++ monkeypatch.setattr(vision, "unsloth_decode_compile", fake_scope)+ return entered+++def test_first_shape_stays_eager_then_compiles_on_repeat(scopes):+ model = _FakeGenerating()+ assert _generate(model, scopes, 1, 200) == (False, 0)+ assert _generate(model, scopes, 1, 300) == (True, 1) # same 1024 bucket+ assert _generate(model, scopes, 3, 300) == (False, 1) # new batch size+ assert _generate(model, scopes, 1, 1500) == (False, 1) # new 2048 bucket+ assert _generate(model, scopes, 3, 90, positional = True) == (True, 2)+ assert model.allocated == [1024, 1024, 1024, 2048, 1024]+++def test_batch_dynamic_graph_skips_eager_warm_up(scopes):+ model = _FakeGenerating()+ _generate(model, scopes, 1, 100)+ _generate(model, scopes, 2, 100)+ assert _generate(model, scopes, 4, 100) == (False, 0) # one compiled batch: still static+ _generate(model, scopes, 1, 100)+ _generate(model, scopes, 2, 100)+ assert len(scopes) == 2+ assert _generate(model, scopes, 3, 100) == (True, 3) # batch now dynamic: no warm-up+ assert _generate(model, scopes, 5, 1500) == (False, 3) # other length: its own graphs+ assert _generate(model, scopes, 1, 100) == (True, 4)+ assert _generate(model, scopes, 6, 100) == (True, 5)+++def test_hooks_are_removed_after_the_call(scopes):+ model = _FakeGenerating()+ _generate(model, scopes, 1, 10)+ assert "_prepare_static_cache" not in model.__dict__+ assert "_valid_auto_compile_criteria" not in model.__dict__+++class _Refusing(_FakeGenerating):+ def _valid_auto_compile_criteria(self, model_kwargs, generation_config):+ return False # e.g. bnb 4-bit or CPU offload+++def test_transformers_refusal_is_respected(scopes):+ model = _Refusing()+ _generate(model, scopes, 1, 10)+ assert _generate(model, scopes, 1, 10) == (False, 0)+++def eager_scope(monkeypatch):+ entered = []++ @contextlib.contextmanager+ def fake_scope():+ entered.append(1)+ yield++ monkeypatch.setattr(vision, "unsloth_eager_decode", fake_scope)+ return entered+++def test_eager_decode_selection(monkeypatch, eager_scope):+ assert _eager_decodes(_model("qwen3_5", "qwen3_5_text"))+ assert _eager_decodes(_model("qwen3_5_moe_text"))+ assert not _eager_decodes(_model("llama"))+ monkeypatch.setenv("UNSLOTH_EAGER_DECODE", "0")+ assert not _eager_decodes(_model("qwen3_5", "qwen3_5_text"))+ monkeypatch.delenv("UNSLOTH_EAGER_DECODE")+ monkeypatch.setattr(vision, "unsloth_eager_decode", None) # older unsloth_zoo+ assert not _eager_decodes(_model("qwen3_5", "qwen3_5_text"))+++class _Forwarding:+ def __init__(self):+ self.seen = []++ def forward(+ self,+ input_ids = None,+ inputs_embeds = None,+ logits_to_keep = 0,+ ):+ self.seen.append((input_ids if input_ids is not None else inputs_embeds).shape)+ return "out"+++def test_only_one_token_steps_run_in_eager_decode(eager_scope):+ import torch++ model = _Forwarding()+ with _EagerDecodeSteps(model):+ # transformers picks logits_to_keep, attention_mask, ... from this signature+ assert list(inspect.signature(model.forward).parameters) == [+ "input_ids",+ "inputs_embeds",+ "logits_to_keep",+ ]+ assert model.forward(input_ids = torch.zeros(2, 7, dtype = torch.long)) == "out"+ assert eager_scope == [] # prefill keeps its compiled regions+ model.forward(input_ids = torch.zeros(2, 1, dtype = torch.long))+ model.forward(inputs_embeds = torch.zeros(3, 1, 8))+ assert len(eager_scope) == 2+ assert "forward" not in model.__dict__+ assert model.seen == [(2, 7), (2, 1), (3, 1, 8)]+++def test_cached_compiled_call_is_dropped_when_the_mode_changes():+ model = types.SimpleNamespace()+ _match_compiled_call(model, False)+ model._compiled_call, model._last_compile_config = "disabled", "cfg"+ _match_compiled_call(model, False)+ assert model._compiled_call == "disabled" # same mode: transformers' cache stands+ _match_compiled_call(model, True)+ assert not hasattr(model, "_compiled_call") and not hasattr(model, "_last_compile_config")+ model._compiled_call = "compiled"+ _match_compiled_call(model, True)+ assert model._compiled_call == "compiled"+ _match_compiled_call(model, False)+ assert not hasattr(model, "_compiled_call")diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex 725ecd37182..531408dab10 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -4580,7 +4580,12 @@ def patch_enable_input_require_grads():returndef _patched_enable_input_require_grads(self):+ import torch+def make_inputs_require_grads(module, input, output):+ # Dynamo graph-breaks on requires_grad_(); a compiled decode step records no grad anyway.+ if torch.compiler.is_compiling() and not torch.is_grad_enabled():+ returnoutput.requires_grad_(True)hooks = []diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 47044a19915..fb2821065f3 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -735,6 +735,171 @@ _compile_config = CompileConfig()_compile_config.disable = True # Must set manually+# For these model types, eager decode steps skip Unsloth's compiled regions (on by default,+# UNSLOTH_EAGER_DECODE=0 opts out), and UNSLOTH_COMPILE_DECODE=1 opts into CUDA graphs over the+# static cache. Off by default: each new shape costs 35-130 s to compile (A100, 2B), so mixed+# sessions came out ~3x slower overall even though a compiled step is ~5x faster.+COMPILE_DECODE_MODELS = ("qwen3_5", "qwen3_5_moe")+_decode_compile_config = CompileConfig(+ fullgraph = False,+ dynamic = None,+ mode = "reduce-overhead",+)+try:+ from unsloth_zoo.temporary_patches.utils import unsloth_decode_compile+except ImportError:+ unsloth_decode_compile = None+try:+ from unsloth_zoo.temporary_patches.utils import unsloth_eager_decode+except ImportError:+ unsloth_eager_decode = None+++def _is_decode_compile_model(model):+ config = model.config+ model_types = (+ getattr(config, "model_type", None),+ getattr(getattr(config, "text_config", None), "model_type", None),+ )+ return any(+ isinstance(mt, str) and mt.removesuffix("_text") in COMPILE_DECODE_MODELS+ for mt in model_types+ )+++def _eager_decodes(model):+ # Eager decode steps skip Unsloth's compiled regions (zoo `unsloth_eager_decode`).+ if unsloth_eager_decode is None or os.environ.get("UNSLOTH_EAGER_DECODE", "1") == "0":+ return False+ return _is_decode_compile_model(model) and "forward" not in model.__dict__+++class _EagerDecodeSteps:+ """One generate() call. Each eager decode step (one new token per row) runs inside zoo's+ `unsloth_eager_decode()`, so compiled regions call their eager originals; prefill and+ a compiled decode step are unchanged."""++ def __init__(self, model):+ self.model = model+ self.forward = model.forward++ def _forward(self, *args, **kwargs):+ if not torch.compiler.is_compiling():+ ids = kwargs.get("input_ids")+ if ids is None:+ ids = kwargs.get("inputs_embeds")+ if ids is not None and ids.dim() >= 2 and ids.shape[1] == 1:+ with unsloth_eager_decode():+ return self.forward(*args, **kwargs)+ return self.forward(*args, **kwargs)++ def __enter__(self):+ # transformers reads signature(self.forward) (logits_to_keep, attention_mask,+ # position_ids), so the stand-in has to present the real one.+ step = self._forward++ @functools.wraps(self.forward)+ def forward(*args, **kwargs):+ return step(*args, **kwargs)++ self.model.forward = forward+ return self++ def __exit__(self, *exc):+ del self.model.forward+ return False+++def _compiles_decode(model):+ if unsloth_decode_compile is None:+ return False+ if os.environ.get("UNSLOTH_COMPILE_DECODE", "0") != "1":+ return False+ if os.environ.get("UNSLOTH_COMPILE_DISABLE", "0") in ("1", "partial"):+ return False+ # Mirror transformers' own auto-compile exceptions: when it will not compile the step,+ # leave the eager decode path exactly as it was.+ quantizer = getattr(model, "hf_quantizer", None)+ if quantizer is not None and not getattr(quantizer, "is_compileable", False):+ return False+ device_map = getattr(model, "hf_device_map", None)+ if isinstance(device_map, dict) and ({"cpu", "disk"} & set(map(str, device_map.values()))):+ return False+ return _is_decode_compile_model(model)+++def _match_compiled_call(model, compile_decode):+ # CompileConfig equality ignores the `disable` attribute, so transformers would keep+ # reusing the callable built for the other mode: drop it when the mode changes.+ if model.__dict__.get("_unsloth_compile_decode", compile_decode) != compile_decode:+ model.__dict__.pop("_compiled_call", None)+ model.__dict__.pop("_last_compile_config", None)+ model._unsloth_compile_decode = compile_decode+++def _decode_cache_bucket(length):+ # A static cache of a new length recompiles the decode step; round it up to a power of+ # two (>= 1024) so later calls with a different max_new_tokens reuse it.+ if type(length) is not int or length <= 0:+ return length+ return max(1024, 1 << (length - 1).bit_length())+++class _CompileDecodeOnRepeat:+ """One generate() call. Buckets the static cache, and compiles the decode step only for a+ (batch, cache length) this model has decoded before: compiling costs ~30-45 s per new shape,+ more than a whole eager call, so a one-off shape stays on the eager path exactly as before.+ Both hooks sit where transformers decides, so its own batch and length are used."""++ def __init__(self, model):+ self.model = model+ self.compile = False+ self.scopes = contextlib.ExitStack()+ self.prepare_static_cache = model._prepare_static_cache+ self.valid_auto_compile_criteria = model._valid_auto_compile_criteria++ def _prepare(self, *args, **kwargs):+ # `max_cache_len` is the third argument on every 5.x release, by keyword since 5.2.+ if "max_cache_len" in kwargs:+ kwargs["max_cache_len"] = _decode_cache_bucket(kwargs["max_cache_len"])+ elif len(args) >= 3:+ args = (*args[:2], _decode_cache_bucket(args[2]), *args[3:])+ cache = self.prepare_static_cache(*args, **kwargs)+ batch_size = kwargs.get("batch_size", args[1] if len(args) > 1 else None)+ key = (batch_size, getattr(cache, "max_cache_len", None))+ seen = getattr(self.model, "_unsloth_decoded_shapes", None)+ if seen is None:+ seen = self.model._unsloth_decoded_shapes = set()+ self.model._unsloth_compiled_batches = {}+ compiled = self.model._unsloth_compiled_batches.setdefault(key[1], set())+ # Once two batch sizes compiled at this length, Dynamo has made batch dynamic (sizes >= 2+ # share that graph), so a new batch size costs no compile: skip its eager warm-up call.+ self.compile = key in seen or (+ type(batch_size) is int and batch_size >= 2 and len(compiled) >= 2+ )+ seen.add(key)+ self.compiled, self.batch_size = compiled, batch_size+ return cache++ def _criteria(self, *args, **kwargs):+ if not self.compile or not self.valid_auto_compile_criteria(*args, **kwargs):+ return False+ self.compiled.add(self.batch_size)+ self.scopes.enter_context(unsloth_decode_compile())+ return True++ def __enter__(self):+ self.model._prepare_static_cache = self._prepare+ self.model._valid_auto_compile_criteria = self._criteria+ return self++ def __exit__(self, *exc):+ del self.model._prepare_static_cache+ del self.model._valid_auto_compile_criteria+ self.scopes.close()+ return False++try:torch_compiler_set_stance = torch.compiler.set_stanceexcept:@@ -1103,6 +1268,22 @@ def unsloth_base_fast_generate(self, *args, **kwargs):cache_implementation = Nonedynamic_implementation = _dynamic_cache_choice(kwargs)+ generation_config = kwargs.get("generation_config") or getattr(self, "generation_config", None)+ compile_decode = (+ cache_implementation == "static"+ and not force_dynamic_cache+ and _compiles_decode(self)+ # transformers compiles only on these devices and without disable_compile; anywhere+ # else a bucketed cache would only cost memory.+ and getattr(getattr(self, "device", None), "type", None) == "cuda"+ and not kwargs.get("disable_compile", getattr(generation_config, "disable_compile", False))+ and hasattr(self, "_prepare_static_cache")+ and hasattr(self, "_valid_auto_compile_criteria")+ and "_prepare_static_cache" not in self.__dict__+ )+ compile_config = _decode_compile_config if compile_decode else _compile_config+ if _is_decode_compile_model(self):+ _match_compiled_call(self, compile_decode)if "generation_config" in kwargs:kwargs["generation_config"].cache_implementation = (dynamic_implementation if force_dynamic_cache else cache_implementation@@ -1111,16 +1292,18 @@ def unsloth_base_fast_generate(self, *args, **kwargs):if force_dynamic_cache:kwargs["cache_implementation"] = dynamic_implementationif cache_implementation is not None:- kwargs["generation_config"].compile_config = _compile_config+ kwargs["generation_config"].compile_config = compile_configelse:kwargs["cache_implementation"] = (dynamic_implementation if force_dynamic_cache else cache_implementation)if cache_implementation is not None:- kwargs["compile_config"] = _compile_config+ kwargs["compile_config"] = compile_config+ decode_scope = _CompileDecodeOnRepeat(self) if compile_decode else contextlib.nullcontext()+ eager_scope = _EagerDecodeSteps(self) if _eager_decodes(self) else contextlib.nullcontext()try:- with torch.inference_mode(), autocaster:+ with decode_scope, eager_scope, torch.inference_mode(), autocaster:output = self._old_generate(*args, **kwargs)finally:_clear_generation_caches(self)