Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Let Gemma 4 31B train when it is split across GPUs (#12233)

Nilay70890714a824
diff --git a/tests/test_gemma4_split_vision_mask.py b/tests/test_gemma4_split_vision_mask.py
new file mode 100644
index 00000000000..d64fc177394
--- /dev/null
+++ b/tests/test_gemma4_split_vision_mask.py
@@ -0,0 +1,74 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+import pytest
+import torch
+
+
+pytestmark = pytest.mark.skipif(
+ not torch.cuda.is_available() or torch.cuda.device_count() < 2,
+ reason = "splitting the model needs two CUDA devices",
+)
+
+
+def test_split_gemma4_trains_with_the_vision_attention_mask(tmp_path):
+ from unsloth import FastModel
+ import transformers
+
+ if not hasattr(transformers, "Gemma4ForConditionalGeneration"):
+ pytest.skip(reason = "this transformers has no Gemma 4")
+ config = transformers.Gemma4Config(
+ text_config = dict(
+ hidden_size = 64,
+ intermediate_size = 128,
+ num_hidden_layers = 2,
+ num_attention_heads = 2,
+ num_key_value_heads = 1,
+ head_dim = 32,
+ global_head_dim = 32,
+ hidden_size_per_layer_input = 0,
+ layer_types = ["sliding_attention", "full_attention"],
+ sliding_window = 16,
+ use_bidirectional_attention = "vision",
+ ),
+ vision_config = dict(
+ hidden_size = 32,
+ intermediate_size = 64,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ num_key_value_heads = 2,
+ head_dim = 16,
+ ),
+ audio_config = None,
+ )
+ transformers.Gemma4ForConditionalGeneration(config).save_pretrained(tmp_path)
+ transformers.AutoProcessor.from_pretrained(
+ "unsloth/gemma-4-31B-it-unsloth-bnb-4bit"
+ ).save_pretrained(tmp_path)
+
+ # The head-aware planner's shape: embedding with lm_head on the last card, root hook on the first.
+ device_map = {
+ "model.vision_tower": 0,
+ "model.embed_vision": 0,
+ "model.language_model.layers.0": 0,
+ "model.language_model.rotary_emb": 0,
+ "model.language_model.layers.1": 1,
+ "model.language_model.norm": 1,
+ "model.language_model.embed_tokens": 1,
+ "lm_head": 1,
+ }
+ model, _ = FastModel.from_pretrained(
+ str(tmp_path),
+ load_in_4bit = False,
+ device_map = device_map,
+ max_seq_length = 64,
+ attn_implementation = "sdpa",
+ )
+ model.train()
+ input_ids = torch.randint(10, 1000, (1, 24), device = "cuda:0")
+ loss = model(
+ input_ids = input_ids,
+ labels = input_ids,
+ mm_token_type_ids = torch.zeros_like(input_ids),
+ ).loss
+ assert torch.isfinite(loss)
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 6af0f5e8937..2f780219b9c 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -433,6 +433,28 @@ def _align_root_hook_with_input_embeddings(model):
return target
+def _move_gemma4_token_types_to_input_embeddings(model):
+ inner = getattr(model, "model", None)
+ device_map = getattr(model, "hf_device_map", None)
+ if (
+ getattr(getattr(inner, "config", None), "model_type", None) != "gemma4"
+ or not device_map
+ or len(set(device_map.values())) < 2
+ ):
+ return
+ target = inner.get_input_embeddings().weight.device
+ if target.type in ("cpu", "meta"):
+ return
+
+ # transformers 5.5-5.8 build Gemma 4's vision mask groups on mm_token_type_ids' device (the root hook's) but index them from the embedding's; 5.9 moves them itself.
+ def to_input_embeddings(module, args, kwargs):
+ if kwargs.get("mm_token_type_ids") is not None:
+ kwargs["mm_token_type_ids"] = kwargs["mm_token_type_ids"].to(target)
+ return args, kwargs
+
+ inner.register_forward_pre_hook(to_input_embeddings, with_kwargs = True)
+
+
def _hook_no_placement_ancestors(model):
"""Input-align hooks on single-card ancestors of a CPU-kept no-placement table (split model)."""
from .loader_utils import no_placement_tensor_names
@@ -3139,6 +3161,7 @@ class FastBaseModel:
f"Unsloth: inputs now go straight to {_aligned_root_device}, where the input "
"embedding lives, instead of through the first device in the map."
)
+ _move_gemma4_token_types_to_input_embeddings(model)
# Re-apply block-fp8 weight_scale_inv tensors transformers dropped on load (#6200).
_restore_dropped_fp8_scales(
model,