Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Qwen3.5: skip compiled regions on eager decode steps, make CUDA graph decode opt-in (#12213)

Daniel Han34f345543188
diff --git a/tests/test_compile_decode_selection.py b/tests/test_compile_decode_selection.py
new file mode 100644
index 00000000000..78f686b5586
--- /dev/null
+++ b/tests/test_compile_decode_selection.py
@@ -0,0 +1,276 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Which models compile their decode step, and the static cache length they get."""
+
+import inspect
+import types
+
+import pytest
+
+import unsloth.models.vision as vision
+import contextlib
+
+from unsloth.models.vision import (
+ _CompileDecodeOnRepeat,
+ _EagerDecodeSteps,
+ _compiles_decode,
+ _decode_cache_bucket,
+ _eager_decodes,
+ _match_compiled_call,
+)
+
+
+def _model(model_type, text_type = None):
+ config = types.SimpleNamespace(model_type = model_type)
+ if text_type is not None:
+ config.text_config = types.SimpleNamespace(model_type = text_type)
+ return types.SimpleNamespace(config = config, generation_config = types.SimpleNamespace())
+
+
[email protected](autouse = True)
+def _zoo_supports_it(monkeypatch):
+ monkeypatch.setattr(vision, "unsloth_decode_compile", object())
+ monkeypatch.setenv("UNSLOTH_COMPILE_DECODE", "1")
+ monkeypatch.delenv("UNSLOTH_COMPILE_DISABLE", raising = False)
+ monkeypatch.delenv("UNSLOTH_EAGER_DECODE", raising = False)
+
+
+def test_decode_compile_is_opt_in(monkeypatch):
+ monkeypatch.delenv("UNSLOTH_COMPILE_DECODE")
+ assert not _compiles_decode(_model("qwen3_5", "qwen3_5_text"))
+
+
+ "model",
+ [
+ _model("qwen3_5", "qwen3_5_text"),
+ _model("qwen3_5_moe", "qwen3_5_moe_text"),
+ _model("qwen3_5_text"),
+ _model("qwen3_5_moe_text"),
+ ],
+)
+def test_qwen3_5_compiles_decode(model):
+ assert _compiles_decode(model)
+
+
+ "model",
+ [
+ _model("llama"),
+ _model("gemma3", "gemma3_text"),
+ _model("qwen3_next"),
+ _model("qwen3_vl", "qwen3_vl_text"),
+ ],
+)
+def test_other_models_keep_eager_decode(model):
+ assert not _compiles_decode(model)
+
+
+ "env",
+ [
+ ("UNSLOTH_COMPILE_DECODE", "0"),
+ ("UNSLOTH_COMPILE_DISABLE", "1"),
+ ("UNSLOTH_COMPILE_DISABLE", "partial"),
+ ],
+)
+def test_opt_outs(monkeypatch, env):
+ monkeypatch.setenv(*env)
+ assert not _compiles_decode(_model("qwen3_5", "qwen3_5_text"))
+
+
+def test_uncompileable_quantizer_keeps_eager_decode():
+ model = _model("qwen3_5", "qwen3_5_text")
+ model.hf_quantizer = types.SimpleNamespace(is_compileable = False)
+ assert not _compiles_decode(model)
+ model.hf_quantizer = types.SimpleNamespace(is_compileable = True)
+ assert _compiles_decode(model)
+
+
[email protected]("device", ["cpu", "disk"])
+def test_offloaded_model_keeps_eager_decode(device):
+ model = _model("qwen3_5", "qwen3_5_text")
+ model.hf_device_map = {"model.layers.0": 0, "model.layers.1": device}
+ assert not _compiles_decode(model)
+ model.hf_device_map = {"model.layers.0": 0, "model.layers.1": 1}
+ assert _compiles_decode(model)
+
+
+def test_old_zoo_keeps_eager_decode(monkeypatch):
+ monkeypatch.setattr(vision, "unsloth_decode_compile", None)
+ assert not _compiles_decode(_model("qwen3_5", "qwen3_5_text"))
+
+
+ "length, bucket", [(74, 1024), (1024, 1024), (1025, 2048), (1501, 2048), (3100, 4096)]
+)
+def test_bucket_rounds_up(length, bucket):
+ assert _decode_cache_bucket(length) == bucket
+
+
+class _FakeGenerating:
+ """Stands in for the model: records the length transformers would allocate and whether
+ it would compile the decode step."""
+
+ def __init__(self):
+ self.allocated = []
+
+ def _prepare_static_cache(
+ self, cache_implementation, batch_size, max_cache_len, *rest, **kwargs
+ ):
+ self.allocated.append(max_cache_len)
+ return types.SimpleNamespace(max_cache_len = max_cache_len)
+
+ def _valid_auto_compile_criteria(self, model_kwargs, generation_config):
+ return True
+
+
+def _generate(
+ model,
+ scopes,
+ batch_size,
+ length,
+ positional = False,
+):
+ with _CompileDecodeOnRepeat(model):
+ if positional: # transformers 5.2 - 5.5 call shape
+ model._prepare_static_cache("static", batch_size, length, {})
+ else: # 5.17 passes keywords
+ model._prepare_static_cache(
+ cache_implementation = "static",
+ batch_size = batch_size,
+ max_cache_len = length,
+ prefill_chunk_size = None,
+ model_kwargs = {},
+ )
+ compiled = model._valid_auto_compile_criteria({}, None)
+ return compiled, len(scopes)
+
+
+def scopes(monkeypatch):
+ entered = []
+
+ @contextlib.contextmanager
+ def fake_scope():
+ entered.append(1)
+ yield
+
+ monkeypatch.setattr(vision, "unsloth_decode_compile", fake_scope)
+ return entered
+
+
+def test_first_shape_stays_eager_then_compiles_on_repeat(scopes):
+ model = _FakeGenerating()
+ assert _generate(model, scopes, 1, 200) == (False, 0)
+ assert _generate(model, scopes, 1, 300) == (True, 1) # same 1024 bucket
+ assert _generate(model, scopes, 3, 300) == (False, 1) # new batch size
+ assert _generate(model, scopes, 1, 1500) == (False, 1) # new 2048 bucket
+ assert _generate(model, scopes, 3, 90, positional = True) == (True, 2)
+ assert model.allocated == [1024, 1024, 1024, 2048, 1024]
+
+
+def test_batch_dynamic_graph_skips_eager_warm_up(scopes):
+ model = _FakeGenerating()
+ _generate(model, scopes, 1, 100)
+ _generate(model, scopes, 2, 100)
+ assert _generate(model, scopes, 4, 100) == (False, 0) # one compiled batch: still static
+ _generate(model, scopes, 1, 100)
+ _generate(model, scopes, 2, 100)
+ assert len(scopes) == 2
+ assert _generate(model, scopes, 3, 100) == (True, 3) # batch now dynamic: no warm-up
+ assert _generate(model, scopes, 5, 1500) == (False, 3) # other length: its own graphs
+ assert _generate(model, scopes, 1, 100) == (True, 4)
+ assert _generate(model, scopes, 6, 100) == (True, 5)
+
+
+def test_hooks_are_removed_after_the_call(scopes):
+ model = _FakeGenerating()
+ _generate(model, scopes, 1, 10)
+ assert "_prepare_static_cache" not in model.__dict__
+ assert "_valid_auto_compile_criteria" not in model.__dict__
+
+
+class _Refusing(_FakeGenerating):
+ def _valid_auto_compile_criteria(self, model_kwargs, generation_config):
+ return False # e.g. bnb 4-bit or CPU offload
+
+
+def test_transformers_refusal_is_respected(scopes):
+ model = _Refusing()
+ _generate(model, scopes, 1, 10)
+ assert _generate(model, scopes, 1, 10) == (False, 0)
+
+
+def eager_scope(monkeypatch):
+ entered = []
+
+ @contextlib.contextmanager
+ def fake_scope():
+ entered.append(1)
+ yield
+
+ monkeypatch.setattr(vision, "unsloth_eager_decode", fake_scope)
+ return entered
+
+
+def test_eager_decode_selection(monkeypatch, eager_scope):
+ assert _eager_decodes(_model("qwen3_5", "qwen3_5_text"))
+ assert _eager_decodes(_model("qwen3_5_moe_text"))
+ assert not _eager_decodes(_model("llama"))
+ monkeypatch.setenv("UNSLOTH_EAGER_DECODE", "0")
+ assert not _eager_decodes(_model("qwen3_5", "qwen3_5_text"))
+ monkeypatch.delenv("UNSLOTH_EAGER_DECODE")
+ monkeypatch.setattr(vision, "unsloth_eager_decode", None) # older unsloth_zoo
+ assert not _eager_decodes(_model("qwen3_5", "qwen3_5_text"))
+
+
+class _Forwarding:
+ def __init__(self):
+ self.seen = []
+
+ def forward(
+ self,
+ input_ids = None,
+ inputs_embeds = None,
+ logits_to_keep = 0,
+ ):
+ self.seen.append((input_ids if input_ids is not None else inputs_embeds).shape)
+ return "out"
+
+
+def test_only_one_token_steps_run_in_eager_decode(eager_scope):
+ import torch
+
+ model = _Forwarding()
+ with _EagerDecodeSteps(model):
+ # transformers picks logits_to_keep, attention_mask, ... from this signature
+ assert list(inspect.signature(model.forward).parameters) == [
+ "input_ids",
+ "inputs_embeds",
+ "logits_to_keep",
+ ]
+ assert model.forward(input_ids = torch.zeros(2, 7, dtype = torch.long)) == "out"
+ assert eager_scope == [] # prefill keeps its compiled regions
+ model.forward(input_ids = torch.zeros(2, 1, dtype = torch.long))
+ model.forward(inputs_embeds = torch.zeros(3, 1, 8))
+ assert len(eager_scope) == 2
+ assert "forward" not in model.__dict__
+ assert model.seen == [(2, 7), (2, 1), (3, 1, 8)]
+
+
+def test_cached_compiled_call_is_dropped_when_the_mode_changes():
+ model = types.SimpleNamespace()
+ _match_compiled_call(model, False)
+ model._compiled_call, model._last_compile_config = "disabled", "cfg"
+ _match_compiled_call(model, False)
+ assert model._compiled_call == "disabled" # same mode: transformers' cache stands
+ _match_compiled_call(model, True)
+ assert not hasattr(model, "_compiled_call") and not hasattr(model, "_last_compile_config")
+ model._compiled_call = "compiled"
+ _match_compiled_call(model, True)
+ assert model._compiled_call == "compiled"
+ _match_compiled_call(model, False)
+ assert not hasattr(model, "_compiled_call")
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index 725ecd37182..531408dab10 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -4580,7 +4580,12 @@ def patch_enable_input_require_grads():
return
def _patched_enable_input_require_grads(self):
+ import torch
+
def make_inputs_require_grads(module, input, output):
+ # Dynamo graph-breaks on requires_grad_(); a compiled decode step records no grad anyway.
+ if torch.compiler.is_compiling() and not torch.is_grad_enabled():
+ return
output.requires_grad_(True)
hooks = []
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 47044a19915..fb2821065f3 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -735,6 +735,171 @@ _compile_config = CompileConfig(
)
_compile_config.disable = True # Must set manually
+# For these model types, eager decode steps skip Unsloth's compiled regions (on by default,
+# UNSLOTH_EAGER_DECODE=0 opts out), and UNSLOTH_COMPILE_DECODE=1 opts into CUDA graphs over the
+# static cache. Off by default: each new shape costs 35-130 s to compile (A100, 2B), so mixed
+# sessions came out ~3x slower overall even though a compiled step is ~5x faster.
+COMPILE_DECODE_MODELS = ("qwen3_5", "qwen3_5_moe")
+_decode_compile_config = CompileConfig(
+ fullgraph = False,
+ dynamic = None,
+ mode = "reduce-overhead",
+)
+try:
+ from unsloth_zoo.temporary_patches.utils import unsloth_decode_compile
+except ImportError:
+ unsloth_decode_compile = None
+try:
+ from unsloth_zoo.temporary_patches.utils import unsloth_eager_decode
+except ImportError:
+ unsloth_eager_decode = None
+
+
+def _is_decode_compile_model(model):
+ config = model.config
+ model_types = (
+ getattr(config, "model_type", None),
+ getattr(getattr(config, "text_config", None), "model_type", None),
+ )
+ return any(
+ isinstance(mt, str) and mt.removesuffix("_text") in COMPILE_DECODE_MODELS
+ for mt in model_types
+ )
+
+
+def _eager_decodes(model):
+ # Eager decode steps skip Unsloth's compiled regions (zoo `unsloth_eager_decode`).
+ if unsloth_eager_decode is None or os.environ.get("UNSLOTH_EAGER_DECODE", "1") == "0":
+ return False
+ return _is_decode_compile_model(model) and "forward" not in model.__dict__
+
+
+class _EagerDecodeSteps:
+ """One generate() call. Each eager decode step (one new token per row) runs inside zoo's
+ `unsloth_eager_decode()`, so compiled regions call their eager originals; prefill and
+ a compiled decode step are unchanged."""
+
+ def __init__(self, model):
+ self.model = model
+ self.forward = model.forward
+
+ def _forward(self, *args, **kwargs):
+ if not torch.compiler.is_compiling():
+ ids = kwargs.get("input_ids")
+ if ids is None:
+ ids = kwargs.get("inputs_embeds")
+ if ids is not None and ids.dim() >= 2 and ids.shape[1] == 1:
+ with unsloth_eager_decode():
+ return self.forward(*args, **kwargs)
+ return self.forward(*args, **kwargs)
+
+ def __enter__(self):
+ # transformers reads signature(self.forward) (logits_to_keep, attention_mask,
+ # position_ids), so the stand-in has to present the real one.
+ step = self._forward
+
+ @functools.wraps(self.forward)
+ def forward(*args, **kwargs):
+ return step(*args, **kwargs)
+
+ self.model.forward = forward
+ return self
+
+ def __exit__(self, *exc):
+ del self.model.forward
+ return False
+
+
+def _compiles_decode(model):
+ if unsloth_decode_compile is None:
+ return False
+ if os.environ.get("UNSLOTH_COMPILE_DECODE", "0") != "1":
+ return False
+ if os.environ.get("UNSLOTH_COMPILE_DISABLE", "0") in ("1", "partial"):
+ return False
+ # Mirror transformers' own auto-compile exceptions: when it will not compile the step,
+ # leave the eager decode path exactly as it was.
+ quantizer = getattr(model, "hf_quantizer", None)
+ if quantizer is not None and not getattr(quantizer, "is_compileable", False):
+ return False
+ device_map = getattr(model, "hf_device_map", None)
+ if isinstance(device_map, dict) and ({"cpu", "disk"} & set(map(str, device_map.values()))):
+ return False
+ return _is_decode_compile_model(model)
+
+
+def _match_compiled_call(model, compile_decode):
+ # CompileConfig equality ignores the `disable` attribute, so transformers would keep
+ # reusing the callable built for the other mode: drop it when the mode changes.
+ if model.__dict__.get("_unsloth_compile_decode", compile_decode) != compile_decode:
+ model.__dict__.pop("_compiled_call", None)
+ model.__dict__.pop("_last_compile_config", None)
+ model._unsloth_compile_decode = compile_decode
+
+
+def _decode_cache_bucket(length):
+ # A static cache of a new length recompiles the decode step; round it up to a power of
+ # two (>= 1024) so later calls with a different max_new_tokens reuse it.
+ if type(length) is not int or length <= 0:
+ return length
+ return max(1024, 1 << (length - 1).bit_length())
+
+
+class _CompileDecodeOnRepeat:
+ """One generate() call. Buckets the static cache, and compiles the decode step only for a
+ (batch, cache length) this model has decoded before: compiling costs ~30-45 s per new shape,
+ more than a whole eager call, so a one-off shape stays on the eager path exactly as before.
+ Both hooks sit where transformers decides, so its own batch and length are used."""
+
+ def __init__(self, model):
+ self.model = model
+ self.compile = False
+ self.scopes = contextlib.ExitStack()
+ self.prepare_static_cache = model._prepare_static_cache
+ self.valid_auto_compile_criteria = model._valid_auto_compile_criteria
+
+ def _prepare(self, *args, **kwargs):
+ # `max_cache_len` is the third argument on every 5.x release, by keyword since 5.2.
+ if "max_cache_len" in kwargs:
+ kwargs["max_cache_len"] = _decode_cache_bucket(kwargs["max_cache_len"])
+ elif len(args) >= 3:
+ args = (*args[:2], _decode_cache_bucket(args[2]), *args[3:])
+ cache = self.prepare_static_cache(*args, **kwargs)
+ batch_size = kwargs.get("batch_size", args[1] if len(args) > 1 else None)
+ key = (batch_size, getattr(cache, "max_cache_len", None))
+ seen = getattr(self.model, "_unsloth_decoded_shapes", None)
+ if seen is None:
+ seen = self.model._unsloth_decoded_shapes = set()
+ self.model._unsloth_compiled_batches = {}
+ compiled = self.model._unsloth_compiled_batches.setdefault(key[1], set())
+ # Once two batch sizes compiled at this length, Dynamo has made batch dynamic (sizes >= 2
+ # share that graph), so a new batch size costs no compile: skip its eager warm-up call.
+ self.compile = key in seen or (
+ type(batch_size) is int and batch_size >= 2 and len(compiled) >= 2
+ )
+ seen.add(key)
+ self.compiled, self.batch_size = compiled, batch_size
+ return cache
+
+ def _criteria(self, *args, **kwargs):
+ if not self.compile or not self.valid_auto_compile_criteria(*args, **kwargs):
+ return False
+ self.compiled.add(self.batch_size)
+ self.scopes.enter_context(unsloth_decode_compile())
+ return True
+
+ def __enter__(self):
+ self.model._prepare_static_cache = self._prepare
+ self.model._valid_auto_compile_criteria = self._criteria
+ return self
+
+ def __exit__(self, *exc):
+ del self.model._prepare_static_cache
+ del self.model._valid_auto_compile_criteria
+ self.scopes.close()
+ return False
+
+
try:
torch_compiler_set_stance = torch.compiler.set_stance
except:
@@ -1103,6 +1268,22 @@ def unsloth_base_fast_generate(self, *args, **kwargs):
cache_implementation = None
dynamic_implementation = _dynamic_cache_choice(kwargs)
+ generation_config = kwargs.get("generation_config") or getattr(self, "generation_config", None)
+ compile_decode = (
+ cache_implementation == "static"
+ and not force_dynamic_cache
+ and _compiles_decode(self)
+ # transformers compiles only on these devices and without disable_compile; anywhere
+ # else a bucketed cache would only cost memory.
+ and getattr(getattr(self, "device", None), "type", None) == "cuda"
+ and not kwargs.get("disable_compile", getattr(generation_config, "disable_compile", False))
+ and hasattr(self, "_prepare_static_cache")
+ and hasattr(self, "_valid_auto_compile_criteria")
+ and "_prepare_static_cache" not in self.__dict__
+ )
+ compile_config = _decode_compile_config if compile_decode else _compile_config
+ if _is_decode_compile_model(self):
+ _match_compiled_call(self, compile_decode)
if "generation_config" in kwargs:
kwargs["generation_config"].cache_implementation = (
dynamic_implementation if force_dynamic_cache else cache_implementation
@@ -1111,16 +1292,18 @@ def unsloth_base_fast_generate(self, *args, **kwargs):
if force_dynamic_cache:
kwargs["cache_implementation"] = dynamic_implementation
if cache_implementation is not None:
- kwargs["generation_config"].compile_config = _compile_config
+ kwargs["generation_config"].compile_config = compile_config
else:
kwargs["cache_implementation"] = (
dynamic_implementation if force_dynamic_cache else cache_implementation
)
if cache_implementation is not None:
- kwargs["compile_config"] = _compile_config
+ kwargs["compile_config"] = compile_config
+ decode_scope = _CompileDecodeOnRepeat(self) if compile_decode else contextlib.nullcontext()
+ eager_scope = _EagerDecodeSteps(self) if _eager_decodes(self) else contextlib.nullcontext()
try:
- with torch.inference_mode(), autocaster:
+ with decode_scope, eager_scope, torch.inference_mode(), autocaster:
output = self._old_generate(*args, **kwargs)
finally:
_clear_generation_caches(self)