Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Turn off Granite 3.0-3.2 attention dropout on float16 loads (#13417)

Daniel Hanb4266cb6a939
diff --git a/tests/test_granite_float16_attention_dropout.py b/tests/test_granite_float16_attention_dropout.py
new file mode 100644
index 00000000000..02f7a9c5732
--- /dev/null
+++ b/tests/test_granite_float16_attention_dropout.py
@@ -0,0 +1,58 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Granite 3.0-3.2 attention dropout NaNs float16 training, so a float16 load turns it off (#3002)."""
+
+from __future__ import annotations
+
+import pytest
+import torch
+from transformers import GraniteConfig, GraniteForCausalLM, LlamaConfig, LlamaForCausalLM
+
+from unsloth.models._utils import _drop_float16_overflowing_attention_dropout
+
+
+def _tiny(config_class, model_class, attention_dropout):
+ config = config_class(
+ vocab_size = 16,
+ hidden_size = 8,
+ intermediate_size = 16,
+ num_hidden_layers = 2,
+ num_attention_heads = 2,
+ num_key_value_heads = 2,
+ attention_dropout = attention_dropout,
+ )
+ return model_class(config)
+
+
+def _dropouts(model):
+ return [
+ m.attention_dropout
+ for m in model.modules()
+ if hasattr(m, "self_attn") is False and hasattr(m, "attention_dropout")
+ ]
+
+
+def test_float16_granite_attention_dropout_is_turned_off(capsys):
+ model = _tiny(GraniteConfig, GraniteForCausalLM, 0.1)
+ assert _dropouts(model) == [0.1, 0.1]
+ _drop_float16_overflowing_attention_dropout(model, torch.float16)
+ assert _dropouts(model) == [0.0, 0.0]
+ # The config is left alone so a saved or merged model keeps the checkpoint's value.
+ assert model.config.attention_dropout == 0.1
+ assert "attention dropout" in capsys.readouterr().out
+ _drop_float16_overflowing_attention_dropout(model, torch.float16)
+ assert capsys.readouterr().out == ""
+
+
[email protected]("dtype", [torch.bfloat16, torch.float32])
+def test_other_dtypes_keep_granite_attention_dropout(dtype):
+ model = _tiny(GraniteConfig, GraniteForCausalLM, 0.1)
+ _drop_float16_overflowing_attention_dropout(model, dtype)
+ assert _dropouts(model) == [0.1, 0.1]
+
+
+def test_float16_other_architectures_keep_attention_dropout():
+ model = _tiny(LlamaConfig, LlamaForCausalLM, 0.1)
+ _drop_float16_overflowing_attention_dropout(model, torch.float16)
+ assert _dropouts(model) == [0.1, 0.1]
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index dc33a79911b..a743873288a 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -18,6 +18,7 @@ __all__ = [
"_requested_float32",
"_mark_requested_float32",
"_mark_forced_float32",
+ "_drop_float16_overflowing_attention_dropout",
"_mark_full_finetuning",
"is_vLLM_available",
"prepare_model_for_kbit_training",
@@ -4623,6 +4624,28 @@ def _mark_forced_float32(model, forced):
return model
+def _drop_float16_overflowing_attention_dropout(model, dtype):
+ """Granite 3.0-3.2 dense ship attention_dropout = 0.1 (3.3+ ship 0). Dropping the attention sink in training pushes the last layer's SwiGLU past float16's 65504 (85k measured in bfloat16), so float16 training NaNs (#3002)."""
+ if dtype != torch.float16:
+ return model
+ changed = False
+ for module in model.modules():
+ dropout = getattr(module, "attention_dropout", None)
+ if (
+ type(module).__name__ == "GraniteAttention"
+ and isinstance(dropout, (int, float))
+ and dropout > 0
+ ):
+ module.attention_dropout = 0.0
+ changed = True
+ if changed:
+ print(
+ "Unsloth: Granite's attention dropout overflows float16 during training and makes the loss NaN, "
+ "so it is turned off for this float16 load. Use bfloat16 to keep it."
+ )
+ return model
+
+
def _mark_full_finetuning(model, full_finetuning):
"""Record how this model was loaded. UNSLOTH_ENABLE_FULL_FINETUNING is process wide and every load rewrites it, so a LoRA model loaded before this one trains would answer "no" here and cost it the bfloat16 full finetuning may keep."""
try:
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 4eec847ece5..8424aeaa94a 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -123,6 +123,7 @@ from ._utils import (
_requested_float32,
_mark_requested_float32,
_mark_forced_float32,
+ _drop_float16_overflowing_attention_dropout,
_ignore_block_swap_params_for_ddp,
_mark_full_finetuning,
_get_text_only_config,
@@ -3119,6 +3120,7 @@ class FastModel(FastBaseModel):
model = _fix_rope_inv_freq(model)
model = _exclude_rope_inv_freq_from_ddp(model)
+ model = _drop_float16_overflowing_attention_dropout(model, dtype)
# Loaded adapters skip get_peft_model.
_ignore_block_swap_params_for_ddp(model)
model = _mark_forced_float32(model, do_forced_float32)