Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Full finetuning notice: say float32 for forced-float32 models on GPUs without bfloat16 (#16405)

Daniel Hanbb5a3a31302a
diff --git a/tests/test_forced_float32_full_finetune_message.py b/tests/test_forced_float32_full_finetune_message.py
new file mode 100644
index 00000000000..54693a06180
--- /dev/null
+++ b/tests/test_forced_float32_full_finetune_message.py
@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""A FORCE_FLOAT32 family (Qwen3-MoE, Gemma 3, gpt-oss) asked for float16 on a GPU without bfloat16 loads in bfloat16
+and casts, and the trainer then runs float32 without autocast. The full-finetuning notice used to read the bf16 load
+dtype and print "Using bfloat16 full finetuning" on a T4. Source-level: from_pretrained cannot run without a GPU."""
+
+import ast
+from pathlib import Path
+
+VISION = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "vision.py"
+
+
+def _full_finetuning_branch():
+ tree = ast.parse(VISION.read_text(encoding = "utf-8"))
+ for node in ast.walk(tree):
+ if (
+ isinstance(node, ast.If)
+ and isinstance(node.test, ast.Name)
+ and node.test.id == "full_finetuning"
+ ):
+ if any("UNSLOTH_ENABLE_FULL_FINETUNING" in ast.unparse(s) for s in node.body):
+ return node
+ raise AssertionError("full_finetuning notice block not found")
+
+
+def test_forced_float32_without_bf16_is_announced_as_float32_first():
+ block = _full_finetuning_branch()
+ chain = [s for s in block.body if isinstance(s, ast.If)][0]
+ first = ast.unparse(chain.test)
+ assert "do_forced_float32" in first and "SUPPORTS_BFLOAT16" in first, first
+ assert "float32" in ast.unparse(chain.body)
+ assert "Using bfloat16 full finetuning" not in ast.unparse(chain.body)
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 1f2aacd0aa7..0936b0f3698 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -4410,7 +4410,13 @@ class FastBaseModel:
if full_finetuning:
os.environ["UNSLOTH_ENABLE_FULL_FINETUNING"] = "1"
- if dtype == torch.bfloat16:
+ if do_forced_float32 and not SUPPORTS_BFLOAT16:
+ # Forced float32 loads bf16 then casts, but without bf16 the trainer keeps float32 (no autocast).
+ print(
+ f"Unsloth: {model_type_arch} cannot train in float16 and this GPU has no bfloat16, "
+ "so full finetuning runs in float32."
+ )
+ elif dtype == torch.bfloat16:
if float32_mixed_precision != True:
print(
f"Unsloth: Using bfloat16 full finetuning which cuts memory usage by 50%.\n"