Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Compiled decode for gpt-oss when unsloth_zoo has the routed experts patch (#12713)
Daniel Han1a073ea29cd5
diff --git a/tests/test_compile_decode_selection.py b/tests/test_compile_decode_selection.pyindex 78f686b5586..052bc679784 100644--- a/tests/test_compile_decode_selection.py+++ b/tests/test_compile_decode_selection.py@@ -31,6 +31,7 @@ def _model(model_type, text_type = None):@pytest.fixture(autouse = True)def _zoo_supports_it(monkeypatch):monkeypatch.setattr(vision, "unsloth_decode_compile", object())+ monkeypatch.setattr(vision, "_zoo_module_available", lambda name: True)monkeypatch.setenv("UNSLOTH_COMPILE_DECODE", "1")monkeypatch.delenv("UNSLOTH_COMPILE_DISABLE", raising = False)monkeypatch.delenv("UNSLOTH_EAGER_DECODE", raising = False)@@ -48,12 +49,21 @@ def test_decode_compile_is_opt_in(monkeypatch):_model("qwen3_5_moe", "qwen3_5_moe_text"),_model("qwen3_5_text"),_model("qwen3_5_moe_text"),+ _model("gpt_oss"),],)def test_qwen3_5_compiles_decode(model):assert _compiles_decode(model)+def test_gpt_oss_needs_the_routed_zoo_patch(monkeypatch):+ # Older unsloth_zoo cannot trace a gpt-oss decode step into one CUDA graph.+ monkeypatch.setattr(vision, "_zoo_module_available", lambda name: False)+ assert not _compiles_decode(_model("gpt_oss"))+ assert not vision._eager_decodes(_model("gpt_oss"))+ assert _compiles_decode(_model("qwen3_5", "qwen3_5_text"))++@pytest.mark.parametrize("model",[diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex bec44dc5961..da4a1e4da53 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -204,6 +204,7 @@ from transformers import __version__ as transformers_versionimport typesimport functools+import importlib.utilimport osimport copyimport gc@@ -1015,7 +1016,9 @@ _compile_config.disable = True # Must set manually# UNSLOTH_EAGER_DECODE=0 opts out), and UNSLOTH_COMPILE_DECODE=1 opts into CUDA graphs over the# static cache. Off by default: each new shape costs 35-130 s to compile (A100, 2B), so mixed# sessions came out ~3x slower overall even though a compiled step is ~5x faster.-COMPILE_DECODE_MODELS = ("qwen3_5", "qwen3_5_moe")+COMPILE_DECODE_MODELS = ("qwen3_5", "qwen3_5_moe", "gpt_oss")+# gpt-oss decode only traces as one CUDA graph with unsloth_zoo's routed experts patch.+_COMPILE_DECODE_NEEDS_ZOO = {"gpt_oss": "unsloth_zoo.temporary_patches.gpt_oss_routed"}_decode_compile_config = CompileConfig(fullgraph = False,dynamic = None,@@ -1037,10 +1040,20 @@ def _is_decode_compile_model(model):getattr(config, "model_type", None),getattr(getattr(config, "text_config", None), "model_type", None),)- return any(- isinstance(mt, str) and mt.removesuffix("_text") in COMPILE_DECODE_MODELS- for mt in model_types- )+ for mt in model_types:+ if not isinstance(mt, str) or mt.removesuffix("_text") not in COMPILE_DECODE_MODELS:+ continue+ needs = _COMPILE_DECODE_NEEDS_ZOO.get(mt.removesuffix("_text"))+ return needs is None or _zoo_module_available(needs)+ return False++[email protected]_cache(maxsize = None)+def _zoo_module_available(name):+ try:+ return importlib.util.find_spec(name) is not None+ except (ImportError, ValueError):+ return Falsedef _eager_decodes(model):