Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Full finetuning notice: say float32 for forced-float32 models on GPUs without bfloat16 (#16405)
Daniel Hanbb5a3a31302a
diff --git a/tests/test_forced_float32_full_finetune_message.py b/tests/test_forced_float32_full_finetune_message.pynew file mode 100644index 00000000000..54693a06180--- /dev/null+++ b/tests/test_forced_float32_full_finetune_message.py@@ -0,0 +1,33 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""A FORCE_FLOAT32 family (Qwen3-MoE, Gemma 3, gpt-oss) asked for float16 on a GPU without bfloat16 loads in bfloat16+and casts, and the trainer then runs float32 without autocast. The full-finetuning notice used to read the bf16 load+dtype and print "Using bfloat16 full finetuning" on a T4. Source-level: from_pretrained cannot run without a GPU."""++import ast+from pathlib import Path++VISION = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "vision.py"+++def _full_finetuning_branch():+ tree = ast.parse(VISION.read_text(encoding = "utf-8"))+ for node in ast.walk(tree):+ if (+ isinstance(node, ast.If)+ and isinstance(node.test, ast.Name)+ and node.test.id == "full_finetuning"+ ):+ if any("UNSLOTH_ENABLE_FULL_FINETUNING" in ast.unparse(s) for s in node.body):+ return node+ raise AssertionError("full_finetuning notice block not found")+++def test_forced_float32_without_bf16_is_announced_as_float32_first():+ block = _full_finetuning_branch()+ chain = [s for s in block.body if isinstance(s, ast.If)][0]+ first = ast.unparse(chain.test)+ assert "do_forced_float32" in first and "SUPPORTS_BFLOAT16" in first, first+ assert "float32" in ast.unparse(chain.body)+ assert "Using bfloat16 full finetuning" not in ast.unparse(chain.body)diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 1f2aacd0aa7..0936b0f3698 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -4410,7 +4410,13 @@ class FastBaseModel:if full_finetuning:os.environ["UNSLOTH_ENABLE_FULL_FINETUNING"] = "1"- if dtype == torch.bfloat16:+ if do_forced_float32 and not SUPPORTS_BFLOAT16:+ # Forced float32 loads bf16 then casts, but without bf16 the trainer keeps float32 (no autocast).+ print(+ f"Unsloth: {model_type_arch} cannot train in float16 and this GPU has no bfloat16, "+ "so full finetuning runs in float32."+ )+ elif dtype == torch.bfloat16:if float32_mixed_precision != True:print(f"Unsloth: Using bfloat16 full finetuning which cuts memory usage by 50%.\n"