Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
gpt-oss: refuse fused 4-bit with dtype float32 on GPUs without bfloat16 (#15614)
Daniel Hane55893c8c952
diff --git a/tests/test_gpt_oss_bf16_load_in_4bit.py b/tests/test_gpt_oss_bf16_load_in_4bit.pyindex 8780e065b70..8705f5d60e4 100644--- a/tests/test_gpt_oss_bf16_load_in_4bit.py+++ b/tests/test_gpt_oss_bf16_load_in_4bit.py@@ -47,6 +47,17 @@ def test_fused_4bit_refused_in_float16():loader_utils.refuse_gpt_oss_fused_4bit_float16(torch.float32)+def test_fused_4bit_float32_refused_without_bfloat16():+ # gpt-oss is FORCE_FLOAT32: without bfloat16 a float32 request still loads float16 weights, so the+ # fused 4-bit path ran float16 hidden states into a float32 router (Half and Float addmm on T4).+ with pytest.raises(NotImplementedError, match = "dtype = torch.float32 does not help"):+ loader_utils.refuse_gpt_oss_fused_4bit_float16(torch.float32, supports_bfloat16 = False)+ with pytest.raises(NotImplementedError, match = "overflow"):+ loader_utils.refuse_gpt_oss_fused_4bit_float16(torch.float16, supports_bfloat16 = False)+ loader_utils.refuse_gpt_oss_fused_4bit_float16(torch.float32, supports_bfloat16 = True)+ loader_utils.refuse_gpt_oss_fused_4bit_float16(torch.bfloat16, supports_bfloat16 = False)++def test_loader_refuses_before_the_fused_4bit_load():import inspect@@ -54,7 +65,10 @@ def test_loader_refuses_before_the_fused_4bit_load():source = inspect.getsource(loader)branch = source.index("this gpt-oss checkpoint stores fused experts, so it loads")- assert "refuse_gpt_oss_fused_4bit_float16(dtype)" in source[branch - 300 : branch]+ assert (+ "refuse_gpt_oss_fused_4bit_float16(dtype, SUPPORTS_BFLOAT16)"+ in source[branch - 300 : branch]+ )def test_fused_checkpoint_detection():diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 45924eee206..121d2b9dffd 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -2757,11 +2757,14 @@ class FastModel(FastBaseModel):except GptOssUnknownExpertLayoutError:# Only fatal where the layout decides the expert classes or the float16 refusal.if (load_in_4bit and _bnb_compatible_quant) or (- _gpt_oss_bnb_4bit and dtype == torch.float16+ _gpt_oss_bnb_4bit+ and (+ dtype == torch.float16 or (not SUPPORTS_BFLOAT16 and dtype == torch.float32)+ )):raiseif _gpt_oss_fused and _gpt_oss_bnb_4bit:- refuse_gpt_oss_fused_4bit_float16(dtype)+ refuse_gpt_oss_fused_4bit_float16(dtype, SUPPORTS_BFLOAT16)if load_in_4bit and _bnb_compatible_quant and not _gpt_oss_fused:string += "_load_in_4bit_"elif load_in_4bit and _bnb_compatible_quant:diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex d7c7c5df60e..de9a8f16dba 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -4461,13 +4461,16 @@ def quantization_config_is_bnb_4bit(quantization_config):return quantization_config_selects_bnb_4bit(quantization_config)-def refuse_gpt_oss_fused_4bit_float16(dtype):- """Fused 4-bit experts compute in the model dtype and overflow float16 (split classes keep down_projs fp32)."""- if dtype != torch.float16:+def refuse_gpt_oss_fused_4bit_float16(dtype, supports_bfloat16 = True):+ """Fused 4-bit experts compute in the model dtype and overflow float16 (split classes keep down_projs fp32).+ Without bfloat16, dtype = torch.float32 still computes in float16: gpt-oss is a FORCE_FLOAT32 family, whose+ float32 request there becomes the float16 weights + float32 islands recipe."""+ if dtype != torch.float16 and (supports_bfloat16 or dtype != torch.float32):returnraise NotImplementedError("Unsloth: this gpt-oss checkpoint stores fused experts, and in float16 (this GPU has no bfloat16, "- "or dtype = torch.float16) its fused 4-bit experts would overflow. Use the pre-quantized "+ "or dtype = torch.float16) its fused 4-bit experts would overflow. dtype = torch.float32 does not "+ "help on a GPU without bfloat16, since gpt-oss then still computes in float16. Use the pre-quantized ""unsloth/gpt-oss-20b-unsloth-bnb-4bit (or -120b), or a GPU with bfloat16.")