Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
gpt-oss MXFP4 without triton_kernels on transformers 4.x: keep dense expert biases in the model dtype (#14275)
Daniel Hanc73389cafafa
diff --git a/tests/test_gpt_oss_mxfp4_bias_upcast.py b/tests/test_gpt_oss_mxfp4_bias_upcast.pynew file mode 100644index 00000000000..48e2c92162d--- /dev/null+++ b/tests/test_gpt_oss_mxfp4_bias_upcast.py@@ -0,0 +1,69 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""gpt-oss MXFP4 expert biases go to fp32, except experts transformers 4.x dequantized (an fp32 bias broke its bmm)."""++import ast+from pathlib import Path++import pytest+import torch++LOADER = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "loader.py"+++def _registered():+ values = []+ for node in ast.walk(ast.parse(LOADER.read_text(encoding = "utf-8"))):+ if isinstance(node, ast.Call) and getattr(node.func, "id", None) == "register_custom_dtype":+ value = ast.literal_eval(node.args[0])+ if "gate_up_proj_bias" in value:+ values.append(value)+ guarded = [v for v in values if "dense" in v]+ plain = [v for v in values if "dense" not in v]+ assert len(guarded) == 1 and len(plain) == 1, values+ return {True: guarded[0], False: plain[0]}+++class _PackedParam(torch.nn.Parameter):+ """Stands in for Mxfp4ExpertParam."""+++def _experts(weight):+ m = torch.nn.Module()+ m.gate_up_proj = weight+ m.gate_up_proj_bias = torch.nn.Parameter(torch.zeros(2, 4, dtype = torch.bfloat16))+ m.down_proj_bias = torch.nn.Parameter(torch.zeros(2, 4, dtype = torch.bfloat16))+ return m+++def _run(code, module):+ fields = code.split(";")+ assert fields[:3] == ["all", "None", "None"] and fields[4] == ""+ exec(fields[3], {"torch": torch, "module": module, "name": "model.layers.0.mlp.experts"})+ return module.gate_up_proj_bias.dtype, module.down_proj_bias.dtype++[email protected]("dense_guard", [True, False])+def test_quantized_experts_get_fp32_biases(dense_guard):+ code = _registered()[dense_guard]+ packed = _experts(_PackedParam(torch.zeros(2, 4, dtype = torch.uint8), requires_grad = False))+ assert _run(code, packed) == (torch.float32, torch.float32)+ # Mxfp4GptOssExperts holds a triton_kernels tensor, not a Parameter.+ native = _experts(None)+ native.gate_up_proj = object()+ assert _run(code, native) == (torch.float32, torch.float32)+++def test_dense_experts_keep_their_bias_dtype_on_transformers_4():+ dense = _experts(torch.nn.Parameter(torch.zeros(2, 4, dtype = torch.bfloat16)))+ assert _run(_registered()[True], dense) == (torch.bfloat16, torch.bfloat16)+ # transformers 5.x is unchanged: Unsloth's own MXFP4 forward reads fp32 biases.+ dense = _experts(torch.nn.Parameter(torch.zeros(2, 4, dtype = torch.bfloat16)))+ assert _run(_registered()[False], dense) == (torch.float32, torch.float32)+++def test_modules_without_expert_biases_are_untouched():+ m = torch.nn.Linear(4, 4, dtype = torch.bfloat16)+ exec(_registered()[True].split(";")[3], {"torch": torch, "module": m, "name": "lm_head"})+ assert m.bias.dtype == torch.bfloat16diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 0321be561bd..5be5fe403a8 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -2595,7 +2595,24 @@ class FastModel(FastBaseModel):elif "gpt_oss" in model_types_all:os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1"# Use the EFFECTIVE bnb state, not the raw flag: a native MXFP4 checkpoint loaded with the default load_in_4bit=True has bnb disabled later, so the raw flag picks the wrong dtype path.- if not (load_in_4bit and _bnb_compatible_quant):+ if not (load_in_4bit and _bnb_compatible_quant) and transformers_version < Version(+ "5.0.0"+ ):+ # Only upcast MoE biases for MXFP4, not BnB, nor experts transformers 4.x dequantized (dense forward).+ register_custom_dtype(+ "all;None;None;"+ "x = 'gate_up_proj_bias'\n"+ "dense = type(getattr(module, 'gate_up_proj', None)) is torch.nn.Parameter and module.gate_up_proj.is_floating_point()\n"+ "if hasattr(module, x) and not dense: "+ "setattr(module, x, torch.nn.Parameter(getattr(module, x).to(torch.float32)) if isinstance(getattr(module, x), torch.nn.Parameter) else getattr(module, x).to(torch.float32))\n"+ ""+ "x = 'down_proj_bias'\n"+ "if hasattr(module, x) and not dense: "+ "setattr(module, x, torch.nn.Parameter(getattr(module, x).to(torch.float32)) if isinstance(getattr(module, x), torch.nn.Parameter) else getattr(module, x).to(torch.float32))\n"+ ""+ ";"+ )+ elif not (load_in_4bit and _bnb_compatible_quant):# Only upcast MoE biases for MXFP4, not BnB.register_custom_dtype("all;None;None;"