Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Turn off Granite 3.0-3.2 attention dropout on float16 loads (#13417)
Daniel Hanb4266cb6a939
diff --git a/tests/test_granite_float16_attention_dropout.py b/tests/test_granite_float16_attention_dropout.pynew file mode 100644index 00000000000..02f7a9c5732--- /dev/null+++ b/tests/test_granite_float16_attention_dropout.py@@ -0,0 +1,58 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Granite 3.0-3.2 attention dropout NaNs float16 training, so a float16 load turns it off (#3002)."""++from __future__ import annotations++import pytest+import torch+from transformers import GraniteConfig, GraniteForCausalLM, LlamaConfig, LlamaForCausalLM++from unsloth.models._utils import _drop_float16_overflowing_attention_dropout+++def _tiny(config_class, model_class, attention_dropout):+ config = config_class(+ vocab_size = 16,+ hidden_size = 8,+ intermediate_size = 16,+ num_hidden_layers = 2,+ num_attention_heads = 2,+ num_key_value_heads = 2,+ attention_dropout = attention_dropout,+ )+ return model_class(config)+++def _dropouts(model):+ return [+ m.attention_dropout+ for m in model.modules()+ if hasattr(m, "self_attn") is False and hasattr(m, "attention_dropout")+ ]+++def test_float16_granite_attention_dropout_is_turned_off(capsys):+ model = _tiny(GraniteConfig, GraniteForCausalLM, 0.1)+ assert _dropouts(model) == [0.1, 0.1]+ _drop_float16_overflowing_attention_dropout(model, torch.float16)+ assert _dropouts(model) == [0.0, 0.0]+ # The config is left alone so a saved or merged model keeps the checkpoint's value.+ assert model.config.attention_dropout == 0.1+ assert "attention dropout" in capsys.readouterr().out+ _drop_float16_overflowing_attention_dropout(model, torch.float16)+ assert capsys.readouterr().out == ""++[email protected]("dtype", [torch.bfloat16, torch.float32])+def test_other_dtypes_keep_granite_attention_dropout(dtype):+ model = _tiny(GraniteConfig, GraniteForCausalLM, 0.1)+ _drop_float16_overflowing_attention_dropout(model, dtype)+ assert _dropouts(model) == [0.1, 0.1]+++def test_float16_other_architectures_keep_attention_dropout():+ model = _tiny(LlamaConfig, LlamaForCausalLM, 0.1)+ _drop_float16_overflowing_attention_dropout(model, torch.float16)+ assert _dropouts(model) == [0.1, 0.1]diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex dc33a79911b..a743873288a 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -18,6 +18,7 @@ __all__ = ["_requested_float32","_mark_requested_float32","_mark_forced_float32",+ "_drop_float16_overflowing_attention_dropout","_mark_full_finetuning","is_vLLM_available","prepare_model_for_kbit_training",@@ -4623,6 +4624,28 @@ def _mark_forced_float32(model, forced):return model+def _drop_float16_overflowing_attention_dropout(model, dtype):+ """Granite 3.0-3.2 dense ship attention_dropout = 0.1 (3.3+ ship 0). Dropping the attention sink in training pushes the last layer's SwiGLU past float16's 65504 (85k measured in bfloat16), so float16 training NaNs (#3002)."""+ if dtype != torch.float16:+ return model+ changed = False+ for module in model.modules():+ dropout = getattr(module, "attention_dropout", None)+ if (+ type(module).__name__ == "GraniteAttention"+ and isinstance(dropout, (int, float))+ and dropout > 0+ ):+ module.attention_dropout = 0.0+ changed = True+ if changed:+ print(+ "Unsloth: Granite's attention dropout overflows float16 during training and makes the loss NaN, "+ "so it is turned off for this float16 load. Use bfloat16 to keep it."+ )+ return model++def _mark_full_finetuning(model, full_finetuning):"""Record how this model was loaded. UNSLOTH_ENABLE_FULL_FINETUNING is process wide and every load rewrites it, so a LoRA model loaded before this one trains would answer "no" here and cost it the bfloat16 full finetuning may keep."""try:diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 4eec847ece5..8424aeaa94a 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -123,6 +123,7 @@ from ._utils import (_requested_float32,_mark_requested_float32,_mark_forced_float32,+ _drop_float16_overflowing_attention_dropout,_ignore_block_swap_params_for_ddp,_mark_full_finetuning,_get_text_only_config,@@ -3119,6 +3120,7 @@ class FastModel(FastBaseModel):model = _fix_rope_inv_freq(model)model = _exclude_rope_inv_freq_from_ddp(model)+ model = _drop_float16_overflowing_attention_dropout(model, dtype)# Loaded adapters skip get_peft_model._ignore_block_swap_params_for_ddp(model)model = _mark_forced_float32(model, do_forced_float32)