Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

gpt-oss: refuse fused 4-bit with dtype float32 on GPUs without bfloat16 (#15614)

Daniel Hane55893c8c952
diff --git a/tests/test_gpt_oss_bf16_load_in_4bit.py b/tests/test_gpt_oss_bf16_load_in_4bit.py
index 8780e065b70..8705f5d60e4 100644
--- a/tests/test_gpt_oss_bf16_load_in_4bit.py
+++ b/tests/test_gpt_oss_bf16_load_in_4bit.py
@@ -47,6 +47,17 @@ def test_fused_4bit_refused_in_float16():
loader_utils.refuse_gpt_oss_fused_4bit_float16(torch.float32)
+def test_fused_4bit_float32_refused_without_bfloat16():
+ # gpt-oss is FORCE_FLOAT32: without bfloat16 a float32 request still loads float16 weights, so the
+ # fused 4-bit path ran float16 hidden states into a float32 router (Half and Float addmm on T4).
+ with pytest.raises(NotImplementedError, match = "dtype = torch.float32 does not help"):
+ loader_utils.refuse_gpt_oss_fused_4bit_float16(torch.float32, supports_bfloat16 = False)
+ with pytest.raises(NotImplementedError, match = "overflow"):
+ loader_utils.refuse_gpt_oss_fused_4bit_float16(torch.float16, supports_bfloat16 = False)
+ loader_utils.refuse_gpt_oss_fused_4bit_float16(torch.float32, supports_bfloat16 = True)
+ loader_utils.refuse_gpt_oss_fused_4bit_float16(torch.bfloat16, supports_bfloat16 = False)
+
+
def test_loader_refuses_before_the_fused_4bit_load():
import inspect
@@ -54,7 +65,10 @@ def test_loader_refuses_before_the_fused_4bit_load():
source = inspect.getsource(loader)
branch = source.index("this gpt-oss checkpoint stores fused experts, so it loads")
- assert "refuse_gpt_oss_fused_4bit_float16(dtype)" in source[branch - 300 : branch]
+ assert (
+ "refuse_gpt_oss_fused_4bit_float16(dtype, SUPPORTS_BFLOAT16)"
+ in source[branch - 300 : branch]
+ )
def test_fused_checkpoint_detection():
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 45924eee206..121d2b9dffd 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -2757,11 +2757,14 @@ class FastModel(FastBaseModel):
except GptOssUnknownExpertLayoutError:
# Only fatal where the layout decides the expert classes or the float16 refusal.
if (load_in_4bit and _bnb_compatible_quant) or (
- _gpt_oss_bnb_4bit and dtype == torch.float16
+ _gpt_oss_bnb_4bit
+ and (
+ dtype == torch.float16 or (not SUPPORTS_BFLOAT16 and dtype == torch.float32)
+ )
):
raise
if _gpt_oss_fused and _gpt_oss_bnb_4bit:
- refuse_gpt_oss_fused_4bit_float16(dtype)
+ refuse_gpt_oss_fused_4bit_float16(dtype, SUPPORTS_BFLOAT16)
if load_in_4bit and _bnb_compatible_quant and not _gpt_oss_fused:
string += "_load_in_4bit_"
elif load_in_4bit and _bnb_compatible_quant:
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index d7c7c5df60e..de9a8f16dba 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -4461,13 +4461,16 @@ def quantization_config_is_bnb_4bit(quantization_config):
return quantization_config_selects_bnb_4bit(quantization_config)
-def refuse_gpt_oss_fused_4bit_float16(dtype):
- """Fused 4-bit experts compute in the model dtype and overflow float16 (split classes keep down_projs fp32)."""
- if dtype != torch.float16:
+def refuse_gpt_oss_fused_4bit_float16(dtype, supports_bfloat16 = True):
+ """Fused 4-bit experts compute in the model dtype and overflow float16 (split classes keep down_projs fp32).
+ Without bfloat16, dtype = torch.float32 still computes in float16: gpt-oss is a FORCE_FLOAT32 family, whose
+ float32 request there becomes the float16 weights + float32 islands recipe."""
+ if dtype != torch.float16 and (supports_bfloat16 or dtype != torch.float32):
return
raise NotImplementedError(
"Unsloth: this gpt-oss checkpoint stores fused experts, and in float16 (this GPU has no bfloat16, "
- "or dtype = torch.float16) its fused 4-bit experts would overflow. Use the pre-quantized "
+ "or dtype = torch.float16) its fused 4-bit experts would overflow. dtype = torch.float32 does not "
+ "help on a GPU without bfloat16, since gpt-oss then still computes in float16. Use the pre-quantized "
"unsloth/gpt-oss-20b-unsloth-bnb-4bit (or -120b), or a GPU with bfloat16."
)