Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Let Gemma 4 31B train when it is split across GPUs (#12233)
Nilay70890714a824
diff --git a/tests/test_gemma4_split_vision_mask.py b/tests/test_gemma4_split_vision_mask.pynew file mode 100644index 00000000000..d64fc177394--- /dev/null+++ b/tests/test_gemma4_split_vision_mask.py@@ -0,0 +1,74 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++import pytest+import torch+++pytestmark = pytest.mark.skipif(+ not torch.cuda.is_available() or torch.cuda.device_count() < 2,+ reason = "splitting the model needs two CUDA devices",+)+++def test_split_gemma4_trains_with_the_vision_attention_mask(tmp_path):+ from unsloth import FastModel+ import transformers++ if not hasattr(transformers, "Gemma4ForConditionalGeneration"):+ pytest.skip(reason = "this transformers has no Gemma 4")+ config = transformers.Gemma4Config(+ text_config = dict(+ hidden_size = 64,+ intermediate_size = 128,+ num_hidden_layers = 2,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ head_dim = 32,+ global_head_dim = 32,+ hidden_size_per_layer_input = 0,+ layer_types = ["sliding_attention", "full_attention"],+ sliding_window = 16,+ use_bidirectional_attention = "vision",+ ),+ vision_config = dict(+ hidden_size = 32,+ intermediate_size = 64,+ num_hidden_layers = 1,+ num_attention_heads = 2,+ num_key_value_heads = 2,+ head_dim = 16,+ ),+ audio_config = None,+ )+ transformers.Gemma4ForConditionalGeneration(config).save_pretrained(tmp_path)+ transformers.AutoProcessor.from_pretrained(+ "unsloth/gemma-4-31B-it-unsloth-bnb-4bit"+ ).save_pretrained(tmp_path)++ # The head-aware planner's shape: embedding with lm_head on the last card, root hook on the first.+ device_map = {+ "model.vision_tower": 0,+ "model.embed_vision": 0,+ "model.language_model.layers.0": 0,+ "model.language_model.rotary_emb": 0,+ "model.language_model.layers.1": 1,+ "model.language_model.norm": 1,+ "model.language_model.embed_tokens": 1,+ "lm_head": 1,+ }+ model, _ = FastModel.from_pretrained(+ str(tmp_path),+ load_in_4bit = False,+ device_map = device_map,+ max_seq_length = 64,+ attn_implementation = "sdpa",+ )+ model.train()+ input_ids = torch.randint(10, 1000, (1, 24), device = "cuda:0")+ loss = model(+ input_ids = input_ids,+ labels = input_ids,+ mm_token_type_ids = torch.zeros_like(input_ids),+ ).loss+ assert torch.isfinite(loss)diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 6af0f5e8937..2f780219b9c 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -433,6 +433,28 @@ def _align_root_hook_with_input_embeddings(model):return target+def _move_gemma4_token_types_to_input_embeddings(model):+ inner = getattr(model, "model", None)+ device_map = getattr(model, "hf_device_map", None)+ if (+ getattr(getattr(inner, "config", None), "model_type", None) != "gemma4"+ or not device_map+ or len(set(device_map.values())) < 2+ ):+ return+ target = inner.get_input_embeddings().weight.device+ if target.type in ("cpu", "meta"):+ return++ # transformers 5.5-5.8 build Gemma 4's vision mask groups on mm_token_type_ids' device (the root hook's) but index them from the embedding's; 5.9 moves them itself.+ def to_input_embeddings(module, args, kwargs):+ if kwargs.get("mm_token_type_ids") is not None:+ kwargs["mm_token_type_ids"] = kwargs["mm_token_type_ids"].to(target)+ return args, kwargs++ inner.register_forward_pre_hook(to_input_embeddings, with_kwargs = True)++def _hook_no_placement_ancestors(model):"""Input-align hooks on single-card ancestors of a CPU-kept no-placement table (split model)."""from .loader_utils import no_placement_tensor_names@@ -3139,6 +3161,7 @@ class FastBaseModel:f"Unsloth: inputs now go straight to {_aligned_root_device}, where the input ""embedding lives, instead of through the first device in the map.")+ _move_gemma4_token_types_to_input_embeddings(model)# Re-apply block-fp8 weight_scale_inv tensors transformers dropped on load (#6200)._restore_dropped_fp8_scales(model,