Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Compiled decode for gpt-oss when unsloth_zoo has the routed experts patch (#12713)

Daniel Han1a073ea29cd5
diff --git a/tests/test_compile_decode_selection.py b/tests/test_compile_decode_selection.py
index 78f686b5586..052bc679784 100644
--- a/tests/test_compile_decode_selection.py
+++ b/tests/test_compile_decode_selection.py
@@ -31,6 +31,7 @@ def _model(model_type, text_type = None):
@pytest.fixture(autouse = True)
def _zoo_supports_it(monkeypatch):
monkeypatch.setattr(vision, "unsloth_decode_compile", object())
+ monkeypatch.setattr(vision, "_zoo_module_available", lambda name: True)
monkeypatch.setenv("UNSLOTH_COMPILE_DECODE", "1")
monkeypatch.delenv("UNSLOTH_COMPILE_DISABLE", raising = False)
monkeypatch.delenv("UNSLOTH_EAGER_DECODE", raising = False)
@@ -48,12 +49,21 @@ def test_decode_compile_is_opt_in(monkeypatch):
_model("qwen3_5_moe", "qwen3_5_moe_text"),
_model("qwen3_5_text"),
_model("qwen3_5_moe_text"),
+ _model("gpt_oss"),
],
)
def test_qwen3_5_compiles_decode(model):
assert _compiles_decode(model)
+def test_gpt_oss_needs_the_routed_zoo_patch(monkeypatch):
+ # Older unsloth_zoo cannot trace a gpt-oss decode step into one CUDA graph.
+ monkeypatch.setattr(vision, "_zoo_module_available", lambda name: False)
+ assert not _compiles_decode(_model("gpt_oss"))
+ assert not vision._eager_decodes(_model("gpt_oss"))
+ assert _compiles_decode(_model("qwen3_5", "qwen3_5_text"))
+
+
@pytest.mark.parametrize(
"model",
[
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index bec44dc5961..da4a1e4da53 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -204,6 +204,7 @@ from transformers import __version__ as transformers_version
import types
import functools
+import importlib.util
import os
import copy
import gc
@@ -1015,7 +1016,9 @@ _compile_config.disable = True # Must set manually
# UNSLOTH_EAGER_DECODE=0 opts out), and UNSLOTH_COMPILE_DECODE=1 opts into CUDA graphs over the
# static cache. Off by default: each new shape costs 35-130 s to compile (A100, 2B), so mixed
# sessions came out ~3x slower overall even though a compiled step is ~5x faster.
-COMPILE_DECODE_MODELS = ("qwen3_5", "qwen3_5_moe")
+COMPILE_DECODE_MODELS = ("qwen3_5", "qwen3_5_moe", "gpt_oss")
+# gpt-oss decode only traces as one CUDA graph with unsloth_zoo's routed experts patch.
+_COMPILE_DECODE_NEEDS_ZOO = {"gpt_oss": "unsloth_zoo.temporary_patches.gpt_oss_routed"}
_decode_compile_config = CompileConfig(
fullgraph = False,
dynamic = None,
@@ -1037,10 +1040,20 @@ def _is_decode_compile_model(model):
getattr(config, "model_type", None),
getattr(getattr(config, "text_config", None), "model_type", None),
)
- return any(
- isinstance(mt, str) and mt.removesuffix("_text") in COMPILE_DECODE_MODELS
- for mt in model_types
- )
+ for mt in model_types:
+ if not isinstance(mt, str) or mt.removesuffix("_text") not in COMPILE_DECODE_MODELS:
+ continue
+ needs = _COMPILE_DECODE_NEEDS_ZOO.get(mt.removesuffix("_text"))
+ return needs is None or _zoo_module_available(needs)
+ return False
+
+
[email protected]_cache(maxsize = None)
+def _zoo_module_available(name):
+ try:
+ return importlib.util.find_spec(name) is not None
+ except (ImportError, ValueError):
+ return False
def _eager_decodes(model):