Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Context parallel: Ulysses for Gemma 4 text models (#15883)

Immanuel Sun3329d6d2296e
diff --git a/tests/test_cp_model_heads_no_layer_count.py b/tests/test_cp_model_heads_no_layer_count.py
new file mode 100644
index 00000000000..6f3ffca375f
--- /dev/null
+++ b/tests/test_cp_model_heads_no_layer_count.py
@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""A config without num_hidden_layers (remote-code ChatGLM keeps num_layers) still gets its head split: on
+transformers >= 5.15 every config carries a per-layer view whose len() reads num_hidden_layers."""
+
+import types
+
+from transformers import PretrainedConfig
+
+import unsloth # noqa: F401
+from unsloth import context_parallel as cp
+
+
+class _NumLayersConfig(PretrainedConfig):
+ model_type = "unsloth_test_num_layers"
+
+ def __init__(
+ self,
+ num_layers = 4,
+ num_attention_heads = 8,
+ **kwargs,
+ ):
+ self.num_layers = num_layers
+ self.num_attention_heads = num_attention_heads
+ super().__init__(**kwargs)
+
+
+def test_config_without_num_hidden_layers_keeps_its_head_split():
+ model = types.SimpleNamespace(config = _NumLayersConfig())
+ assert cp._model_heads(model) == [(8, 8)]
+ assert cp.resolve_cp_backend("auto", 2, model) == "ulysses"
+ assert cp.resolve_cp_backend("auto", 3, model) == "ring"
diff --git a/tests/test_parallel_capabilities_evidence.py b/tests/test_parallel_capabilities_evidence.py
index feb185f2ab3..e67410cb146 100644
--- a/tests/test_parallel_capabilities_evidence.py
+++ b/tests/test_parallel_capabilities_evidence.py
@@ -410,6 +410,59 @@ def test_deepseek_evidence_ids_are_real_parity_cases():
assert named >= 12
+ "mode, expert_lora",
+ [("full", False), ("lora", False), ("lora", True)],
+ ids = ["full", "lora", "lora+experts"],
+)
+def test_gemma4_moe_under_cp_follows_its_gloo_evidence(mode, expert_lora):
+ kw = dict(
+ trainer = "sft",
+ family = "gemma4",
+ moe = True,
+ windowed = True,
+ mode = mode,
+ expert_lora = expert_lora,
+ cp = 2,
+ cp_backend = "ulysses",
+ torch = (2, 13),
+ attention = "sdpa",
+ )
+ for extra in ({}, {"memory": "gc"}, {"attention": "eager"}, {"dp": 2}, {"cp": 4}):
+ query = caps.Query(**{**kw, **extra})
+ status, cell = caps.cell_status(query)
+ assert status == "supported" and "test_gemma4_cp_matches_fp64_reference" in cell.parity, (
+ extra,
+ cell,
+ )
+ caps.check(query)
+ for extra, match in (
+ ({"memory": "unsloth_offload"}, "Unsloth checkpoint"),
+ ({"fsdp2": True}, "fsdp2_cp"),
+ ({"router_aux_loss": True}, "router auxiliary loss"),
+ ({"compile": "block"}, "no numeric parity"),
+ ({"mode": "qlora"}, "4-bit experts"),
+ ({"multimodal": True}, "bidirectional"),
+ ({"multimodal": True, "memory": "gc"}, "bidirectional"),
+ ):
+ with pytest.raises(NotImplementedError, match = match):
+ caps.check(caps.Query(**{**kw, **extra}))
+
+
+def test_gemma4_evidence_ids_are_real_parity_cases():
+ import re
+
+ parity = pytest.importorskip("tests.test_ulysses_gemma4")
+ named = 0
+ for cell in caps.CELLS:
+ if cell.parity and "test_gemma4_cp_matches_fp64_reference[" in cell.parity:
+ ids = re.search(r"\[(.*)\]", cell.parity).group(1).split("|")
+ assert set(ids) <= set(parity.CASES), (cell.name, ids)
+ assert all(parity.CASES[i]["kind"] == "moe" for i in ids), (cell.name, ids)
+ named += 1
+ assert named >= 3
+
+
def test_unsloth_checkpoint_keeps_an_unverified_moe_row_refused_under_strict(
clean_env, monkeypatch
):
diff --git a/tests/test_parallel_env.py b/tests/test_parallel_env.py
index 83b5f156394..d6ce016b8d8 100644
--- a/tests/test_parallel_env.py
+++ b/tests/test_parallel_env.py
@@ -874,7 +874,7 @@ def test_moe_rows_cover_family_precision_aux_and_mode():
c
for c in caps.CELLS
if c.name.startswith("MoE ")
- and not c.name.endswith((", block compile", ", Unsloth checkpoint"))
+ and not c.name.endswith((", block compile", ", Unsloth checkpoint", ", multimodal"))
]
assert len(moe) == len(caps.MOE_FAMILIES) * 4 * 2 * 6
proven = sorted(c.name for c in moe if not c.missing())
diff --git a/tests/test_ulysses_gemma4.py b/tests/test_ulysses_gemma4.py
new file mode 100644
index 00000000000..478a643744d
--- /dev/null
+++ b/tests/test_ulysses_gemma4.py
@@ -0,0 +1,546 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Ulysses CP for Gemma 4 text models on CPU gloo ranks: fp64 parity and the per-layer head split."""
+
+import copy
+import os
+import tempfile
+import types
+
+import pytest
+import torch
+import torch.distributed as dist
+import torch.multiprocessing as mp
+import torch.nn.functional as F
+
+import unsloth # noqa: F401
+from unsloth import context_parallel as cp
+from unsloth.utils import ulysses_hybrid as uh
+
+transformers = pytest.importorskip("transformers")
+gemma4 = pytest.importorskip("transformers.models.gemma4")
+LAYER_TYPES = ["sliding_attention", "full_attention", "sliding_attention", "full_attention"]
+SEQ = 13
+
+
+def _config(
+ kind,
+ window = 4,
+ **overrides,
+):
+ common = dict(
+ vocab_size = 64,
+ vocab_size_per_layer_input = 64,
+ hidden_size = 32,
+ intermediate_size = 64,
+ num_hidden_layers = 4,
+ num_attention_heads = 4,
+ head_dim = 8,
+ global_head_dim = 16,
+ # A window of 4 inside 13 tokens: windows cross the shard seam.
+ sliding_window = window,
+ final_logit_softcapping = 30.0,
+ layer_types = LAYER_TYPES,
+ )
+ if kind == "e2b":
+ # E2B / E4B: per-layer embeddings, the last two layers reuse earlier KV, one KV head.
+ common.update(
+ hidden_size_per_layer_input = 8,
+ num_kv_shared_layers = 2,
+ num_key_value_heads = 1,
+ use_double_wide_mlp = True,
+ )
+ elif kind == "moe":
+ # 26B-A4B / 31B: global layers share K and V and have fewer KV heads; MoE; no PLE.
+ common.update(
+ hidden_size_per_layer_input = 0,
+ num_kv_shared_layers = 0,
+ num_key_value_heads = 2,
+ num_global_key_value_heads = 1,
+ attention_k_eq_v = True,
+ enable_moe_block = True,
+ num_experts = 4,
+ top_k_experts = 2,
+ moe_intermediate_size = 16,
+ )
+ elif kind == "hd512":
+ # The release global head_dim (512, past flash-attn's 256) with heterogeneous KV heads.
+ common.update(
+ hidden_size_per_layer_input = 0,
+ num_kv_shared_layers = 0,
+ num_key_value_heads = 2,
+ num_global_key_value_heads = 1,
+ attention_k_eq_v = True,
+ global_head_dim = 512,
+ )
+ else:
+ raise ValueError(kind)
+ return gemma4.Gemma4TextConfig(**{**common, **overrides})
+
+
+def _multimodal_config(kind, window = 4):
+ vision = gemma4.Gemma4VisionConfig(
+ hidden_size = 16,
+ intermediate_size = 32,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ num_key_value_heads = 2,
+ head_dim = 8,
+ )
+ return gemma4.Gemma4Config(
+ text_config = _config(kind.removeprefix("mm_"), window).to_dict(),
+ vision_config = vision.to_dict(),
+ audio_config = None,
+ )
+
+
+def _model(
+ kind,
+ *,
+ window = 4,
+ attn = "sdpa",
+ dtype = torch.float64,
+ checkpoint = False,
+ lora = None,
+):
+ os.environ.setdefault("UNSLOTH_MOE_BACKEND", "native_torch")
+ torch.manual_seed(0)
+ if kind.startswith("mm_"):
+ model = gemma4.Gemma4ForConditionalGeneration(_multimodal_config(kind, window))
+ else:
+ model = gemma4.Gemma4ForCausalLM(_config(kind, window))
+ model = model.to(dtype).train()
+ model.config._attn_implementation = attn
+ model.config.use_cache = False
+ for module in model.modules():
+ if hasattr(module, "set_experts_implementation"):
+ module.set_experts_implementation("eager")
+ if checkpoint:
+ model.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": False})
+ if lora:
+ from peft import LoraConfig, get_peft_model
+
+ extra = {}
+ if lora == "experts":
+ extra["target_parameters"] = ["experts.gate_up_proj", "experts.down_proj"]
+ targets = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]
+ model = get_peft_model(
+ model, LoraConfig(r = 4, lora_alpha = 8, target_modules = targets, **extra)
+ )
+ g = torch.Generator().manual_seed(5)
+ with torch.no_grad():
+ for name, p in model.named_parameters():
+ if "lora_B" in name: # nonzero B: the adapter changes the forward
+ p.copy_(0.05 * torch.randn(p.shape, generator = g, dtype = p.dtype))
+ return model
+
+
+def _manager(
+ size,
+ group,
+ local = None,
+):
+ manager = object.__new__(cp.ContextParallelManager)
+ manager.size = size
+ manager.backend = "ulysses"
+ local = dist.get_rank(group) if local is None else local
+ manager.mesh = types.SimpleNamespace(get_group = lambda: group, get_local_rank = lambda: local)
+ return manager
+
+
+def _loss(logits, shift_labels, count):
+ return (
+ F.cross_entropy(
+ logits.to(torch.promote_types(logits.dtype, torch.float32)).flatten(0, 1),
+ shift_labels.flatten(),
+ ignore_index = -100,
+ reduction = "sum",
+ )
+ / count
+ )
+
+
+def _batch(labels_scheme):
+ ids = torch.randint(0, 64, (2, SEQ), generator = torch.Generator().manual_seed(1))
+ mask = torch.ones_like(ids)
+ mask[1, 6:] = 0 # row 1: 6 valid tokens, right padded
+ labels = ids.masked_fill(mask == 0, -100)
+ if labels_scheme == "prompt_masked":
+ # Completion-only labels: the first shard(s) carry no labels at all.
+ labels[:, :9] = -100
+ return ids, mask, labels
+
+
+def _grad_status(grad):
+ return 0 if grad is None else (1 if not grad.any() else 2)
+
+
+def _spawn(fn, world, *args):
+ if "fork" not in mp.get_all_start_methods():
+ pytest.skip("needs the fork start method")
+ with tempfile.TemporaryDirectory() as d:
+ mp.start_processes(
+ _entry,
+ args = (world, os.path.join(d, "init"), fn, args),
+ nprocs = world,
+ join = True,
+ start_method = "fork",
+ )
+
+
+def _entry(rank, world, init_file, fn, args):
+ torch.set_num_threads(1)
+ dist.init_process_group("gloo", init_method = f"file://{init_file}", rank = rank, world_size = world)
+ try:
+ fn(rank, world, *args)
+ finally:
+ dist.destroy_process_group()
+
+
+def _cp_groups(world, size):
+ mine = None
+ for start in range(0, world, size):
+ ranks = list(range(start, start + size))
+ group = dist.new_group(ranks)
+ if dist.get_rank() in ranks:
+ mine = group
+ return mine
+
+
+def _model_case(
+ rank,
+ world,
+ case,
+ control = None,
+):
+ size = case["cp"]
+ dp = world // size
+ group = _cp_groups(world, size)
+ window = case.get("window", 4)
+ model = _model(
+ case["kind"],
+ window = window,
+ attn = case.get("attn", "sdpa"),
+ checkpoint = case.get("checkpoint", False),
+ lora = case.get("lora"),
+ )
+ reference = copy.deepcopy(model)
+ ids, mask, labels = _batch(case.get("labels", "pad"))
+ shift = F.pad(labels, (0, 1), value = -100)[:, 1:]
+ count = shift.ne(-100).sum()
+ _loss(
+ reference(input_ids = ids, attention_mask = mask, use_cache = False).logits, shift, count
+ ).backward()
+
+ rows = list(range(rank // size, ids.shape[0], dp))
+ manager = _manager(size, group)
+ cp._check_cp_model(model, size, "ulysses")
+ manager.attach_attention_hooks(model)
+ attention = [m for n, m in model.named_modules() if n.endswith("self_attn")]
+ if control == "shard_local":
+ for module in attention:
+ module._unsloth_cp_ulysses_attention = False
+ elif control == "global_layers_windowed":
+ for module in attention:
+ module.sliding_window = window
+ elif control == "dropped_window":
+ for module in attention:
+ module.sliding_window = None
+ elif control == "default_scale":
+ for module in attention:
+ module.scaling = None
+ calls = []
+ real = uh._ulysses_hf_attention
+
+ def counted(*args, **kwargs):
+ calls.append((args[1].shape[1], args[2].shape[1], args[1].shape[-1]))
+ return real(*args, **kwargs)
+
+ uh._ulysses_hf_attention = counted
+ try:
+ inputs = {
+ "input_ids": ids[rows].clone(),
+ "attention_mask": mask[rows].clone(),
+ "labels": labels[rows].clone(),
+ }
+ with manager.apply(inputs):
+ positions = inputs["position_ids"]
+ if control == "position_offset":
+ positions = positions - positions[:, :1]
+ out = model(
+ input_ids = inputs["input_ids"],
+ attention_mask = inputs["attention_mask"],
+ position_ids = positions,
+ use_cache = False,
+ )
+ _loss(out.logits, inputs["shift_labels"], count).backward()
+ finally:
+ uh._ulysses_hf_attention = real
+ if control is None:
+ runs = 2 if case.get("checkpoint") else 1
+ text = _config(case["kind"])
+ assert len(calls) == runs * text.num_hidden_layers, (len(calls), runs)
+ expected = {
+ (text.num_attention_heads, layer.num_key_value_heads, layer.head_dim)
+ for layer in text.per_layer_config
+ }
+ assert set(calls) == expected, (set(calls), expected)
+
+ ref_params = dict(reference.named_parameters())
+ for name, param in model.named_parameters():
+ statuses = [None] * world
+ dist.all_gather_object(statuses, _grad_status(param.grad))
+ expected = ref_params[name].grad
+ if expected is None:
+ assert statuses == [0] * world, (name, statuses)
+ continue
+ # Every rank holds a gradient (a rank without labels too), or the reduction desyncs.
+ assert 0 not in statuses, (name, statuses)
+ grad = param.grad
+ dist.all_reduce(grad)
+ torch.testing.assert_close(
+ grad, expected, rtol = 1e-5, atol = 1e-6, msg = lambda m: f"{name}: {m}"
+ )
+ assert _grad_status(grad) == _grad_status(expected), name
+ # Large eps: Gemma 4's RMSNorm runs in float32 even under float64, so gradients carry float32 rounding.
+ for net in (reference, model):
+ trainable = [p for p in net.parameters() if p.requires_grad]
+ torch.optim.AdamW(trainable, lr = 1e-3, eps = 1e-3, weight_decay = 0.1).step()
+ for (name, mine), theirs in zip(model.named_parameters(), reference.parameters()):
+ torch.testing.assert_close(
+ mine, theirs, rtol = 1e-5, atol = 1e-6, msg = lambda m: f"after AdamW {name}: {m}"
+ )
+
+
+def _case_worker(rank, world, case):
+ _model_case(rank, world, case)
+
+
+E2B = dict(kind = "e2b")
+MOE = dict(kind = "moe")
+HD512 = dict(kind = "hd512")
+CASES = {
+ "e2b_cp2": dict(E2B, cp = 2, world = 2, window = 3),
+ "e2b_cp2_window9_prompt_masked": dict(E2B, cp = 2, world = 2, window = 9, labels = "prompt_masked"),
+ "e2b_cp2_window64": dict(E2B, cp = 2, world = 2, window = 64),
+ "e2b_cp4": dict(E2B, cp = 4, world = 4, window = 9, labels = "prompt_masked"),
+ "e2b_dp2cp2": dict(E2B, cp = 2, world = 4, window = 5),
+ "e2b_cp2_eager_checkpoint": dict(E2B, cp = 2, world = 2, window = 3, attn = "eager", checkpoint = True),
+ "e2b_cp2_lora": dict(E2B, cp = 2, world = 2, window = 3, lora = "dense"),
+ "moe_cp2": dict(MOE, cp = 2, world = 2, window = 3),
+ "moe_cp2_window9_prompt_masked": dict(MOE, cp = 2, world = 2, window = 9, labels = "prompt_masked"),
+ "moe_cp4": dict(MOE, cp = 4, world = 4, window = 3),
+ "moe_dp2cp2": dict(MOE, cp = 2, world = 4, window = 3, labels = "prompt_masked"),
+ "moe_cp2_eager_checkpoint": dict(MOE, cp = 2, world = 2, window = 3, attn = "eager", checkpoint = True),
+ "moe_cp2_lora": dict(MOE, cp = 2, world = 2, window = 9, lora = "dense"),
+ "moe_cp2_lora_experts": dict(MOE, cp = 2, world = 2, window = 3, lora = "experts"),
+ "moe_cp2_lora_experts_eager_checkpoint": dict(
+ MOE, cp = 2, world = 2, window = 3, lora = "experts", attn = "eager", checkpoint = True
+ ),
+ "hd512_cp2": dict(HD512, cp = 2, world = 2, window = 3),
+ "hd512_cp2_eager": dict(HD512, cp = 2, world = 2, window = 3, attn = "eager"),
+}
+
+
[email protected]("name", list(CASES))
+def test_gemma4_cp_matches_fp64_reference(name):
+ case = CASES[name]
+ _spawn(_case_worker, case["world"], case)
+
+
+def _controls_worker(rank, world):
+ passed = []
+ for control in (
+ "shard_local",
+ "global_layers_windowed",
+ "dropped_window",
+ "default_scale",
+ "position_offset",
+ ):
+ try:
+ _model_case(rank, world, dict(E2B, cp = 2, window = 3), control = control)
+ passed.append(control)
+ except AssertionError:
+ pass
+ assert not passed, f"negative controls that matched the reference: {passed}"
+
+
+def test_gemma4_negative_controls_fail():
+ _spawn(_controls_worker, 2)
+
+
+def test_gemma4_head_counts_are_read_per_layer():
+ model = _model("moe", dtype = torch.float32)
+ with pytest.raises(RuntimeError, match = "per-layer attribute"):
+ model.config.num_key_value_heads
+ assert cp._model_heads(model) == [(4, 1), (4, 2)]
+ assert cp._model_heads(_model("e2b", dtype = torch.float32)) == [(4, 1)]
+ cp._check_cp_model(model, 2, "ulysses")
+ cp._check_cp_model(model, 4, "ulysses")
+ assert cp.resolve_cp_backend("auto", 2, model) == "ulysses"
+ old = types.SimpleNamespace(
+ config = types.SimpleNamespace(
+ num_attention_heads = 6,
+ num_key_value_heads = 6,
+ num_global_key_value_heads = 3,
+ attention_k_eq_v = True,
+ )
+ )
+ assert cp._model_heads(old) == [(6, 3), (6, 6)]
+ old.config.attention_k_eq_v = False
+ assert cp._model_heads(old) == [(6, 6)]
+ llama = types.SimpleNamespace(config = types.SimpleNamespace(num_attention_heads = 8))
+ assert cp._model_heads(llama) == [(8, 8)]
+ assert cp._model_heads(types.SimpleNamespace(config = None)) == []
+
+
+def test_gemma4_global_layer_heads_that_cannot_split_refuse_before_any_collective():
+ text = _config(
+ "moe", num_attention_heads = 6, num_key_value_heads = 6, num_global_key_value_heads = 3
+ )
+ assert {layer.num_key_value_heads for layer in text.per_layer_config} == {3, 6}
+ model = types.SimpleNamespace(config = text)
+ assert cp._model_heads(model) == [(6, 3), (6, 6)]
+ with pytest.raises(ValueError, match = r"key/value heads \(3\)"):
+ cp._ulysses_head_plans(model, 2)
+ assert cp.resolve_cp_backend("auto", 2, model) == "ring"
+ assert cp.resolve_cp_backend("auto", 3, model) == "ulysses"
+
+
+def test_gemma4_unused_global_kv_count_does_not_refuse():
+ """A global KV count no layer uses (transformers forces Gemma 4's last layer global, so only hand-built
+ or remote configs get here) must not refuse layers that split."""
+ sliding = types.SimpleNamespace(num_attention_heads = 6, num_key_value_heads = 6)
+ per_layer = types.SimpleNamespace(
+ num_attention_heads = 6,
+ num_key_value_heads = 6,
+ num_global_key_value_heads = 3,
+ attention_k_eq_v = True,
+ per_layer_config = [sliding] * 4,
+ )
+ model = types.SimpleNamespace(config = per_layer)
+ assert cp._model_heads(model) == [(6, 6)]
+ cp._ulysses_head_plans(model, 2)
+ assert cp.resolve_cp_backend("auto", 2, model) == "ulysses"
+ old = types.SimpleNamespace(
+ config = types.SimpleNamespace(
+ num_attention_heads = 6,
+ num_key_value_heads = 6,
+ num_global_key_value_heads = 3,
+ attention_k_eq_v = True,
+ layer_types = ["sliding_attention"] * 4,
+ )
+ )
+ assert cp._model_heads(old) == [(6, 6)]
+
+
+def _check(
+ model,
+ backend = "ulysses",
+ allow_hybrids = True,
+):
+ try:
+ cp._check_cp_model(model, 2, backend, allow_hybrids)
+ except NotImplementedError as error:
+ return str(error)
+ return None
+
+
+def test_gemma4_generic_route_gates(monkeypatch):
+ from unsloth.models.llama import LlamaAttention_fast_forward
+
+ model = _model("e2b", dtype = torch.float32)
+ assert cp._generic_hf_attention(model)
+ assert cp._supports_context_parallel(model, "ulysses")
+ assert not cp._supports_context_parallel(model, "ring")
+ assert _check(model) is None
+ assert "sliding" in _check(model, "ring")
+ assert _check(model, allow_hybrids = False) is not None
+ attention = type(model.model.layers[0].self_attn)
+ monkeypatch.setattr(attention, "forward", LlamaAttention_fast_forward)
+ assert not cp._generic_hf_attention(model)
+ assert "sliding" in _check(model)
+ assert "sliding" in _check(model, allow_hybrids = False)
+ monkeypatch.undo()
+ multimodal = _model("mm_e2b", dtype = torch.float32)
+ with pytest.raises(NotImplementedError, match = "Gemma4VisionAttention is bidirectional"):
+ _manager(2, None, local = 0).attach_attention_hooks(multimodal)
+ for model_type in ("gemma3_text", "mistral", "qwen2", "gemma4_text"):
+ other = types.SimpleNamespace(
+ model_type = model_type,
+ sliding_window = 4,
+ layer_types = ["sliding_attention", "full_attention"],
+ )
+ assert cp._unsupported_attention(other, "ulysses") == "sliding_attention layers"
+ text = _config("e2b")
+ text.layer_types = ["chunked_attention"] * 4
+ assert cp._unsupported_attention(text, "ulysses", generic = True) == "chunked_attention layers"
+
+
+# A head_dim flash-attn cannot run fails under CP as in one process (no silent switch to SDPA).
+
+_FLASH_MAX_HEAD_DIM = 256 # flash-attn 2's forward / backward limit
+
+
+def _fake_flash(module, query, key, value, attention_mask, **kwargs):
+ from transformers.integrations.sdpa_attention import sdpa_attention_forward
+
+ if query.shape[-1] > _FLASH_MAX_HEAD_DIM:
+ raise RuntimeError("FlashAttention forward only supports head dimension at most 256")
+ kwargs.pop("sliding_window", None)
+ return sdpa_attention_forward(module, query, key, value, None, **kwargs)
+
+
+def _flash_worker(rank, world, kind, results):
+ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
+
+ ALL_ATTENTION_FUNCTIONS["flash_attention_2"] = _fake_flash
+ model = _model(kind, attn = "flash_attention_2", dtype = torch.float32)
+ ids = torch.randint(0, 64, (1, 16), generator = torch.Generator().manual_seed(1))
+
+ def run(fn):
+ try:
+ fn()
+ except RuntimeError as error:
+ return str(error)
+ return None
+
+ one = run(lambda: model(input_ids = ids, use_cache = False))
+ group = _cp_groups(world, world)
+ manager = _manager(world, group)
+ manager.attach_attention_hooks(model)
+ calls = []
+ real = uh._ulysses_hf_attention
+
+ def counted(*args, **kwargs):
+ calls.append(kwargs["implementation"])
+ return real(*args, **kwargs)
+
+ uh._ulysses_hf_attention = counted
+ inputs = {"input_ids": ids.clone(), "labels": ids.clone()}
+
+ def cp_forward():
+ with manager.apply(inputs):
+ model(
+ input_ids = inputs["input_ids"], position_ids = inputs["position_ids"], use_cache = False
+ )
+
+ many = run(cp_forward)
+ uh._ulysses_hf_attention = real
+ assert calls and set(calls) == {"flash_attention_2"}, calls
+ assert one == many, (one, many)
+ results.append((one, many))
+
+
[email protected]("kind, fails", [("hd512", True), ("moe", False)])
+def test_flash_head_dim_limit_matches_one_process(kind, fails):
+ def worker(rank, world):
+ results = []
+ _flash_worker(rank, world, kind, results)
+ one, _ = results[0]
+ assert (one is not None) == fails, one
+ if fails:
+ assert "head dimension" in one
+
+ _spawn(worker, 2)
diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.py
index 64ba402ad87..3a6e5337f0a 100644
--- a/unsloth/context_parallel.py
+++ b/unsloth/context_parallel.py
@@ -186,13 +186,46 @@ def _keep_layer_out_of_block_compile(model, attn_name: str) -> None:
def _model_heads(model):
+ """Sorted (heads, kv heads) per attention layer kind; per_layer_config first since Gemma 4 global reads raise."""
config = getattr(model, "config", None)
text = getattr(config, "text_config", None) or config
- heads = getattr(text, "num_attention_heads", None)
- if not isinstance(heads, int):
- return None, None
- kv = getattr(text, "num_key_value_heads", None)
- return heads, kv if isinstance(kv, int) and kv > 0 else heads
+ if not isinstance(getattr(text, "num_attention_heads", None), int):
+ return []
+ counts = set()
+ try:
+ # transformers >= 5.15 gives every config a per-layer view; its len() reads num_hidden_layers,
+ # which configs such as remote-code ChatGLM (num_layers) do not define.
+ layers = list(getattr(text, "per_layer_config", None) or ())
+ except (AttributeError, TypeError):
+ layers = []
+ for layer in layers or [text]:
+ heads = getattr(layer, "num_attention_heads", None)
+ heads = heads if isinstance(heads, int) else text.num_attention_heads
+ kv = getattr(layer, "num_key_value_heads", None)
+ counts.add((heads, kv if isinstance(kv, int) and kv > 0 else heads))
+ # Without a per-layer view (older Gemma 4 configs) the global layers' KV count sits beside the sliding
+ # one; a per-layer view already lists what every layer uses, and a config with no global layer uses none.
+ global_kv = getattr(text, "num_global_key_value_heads", None)
+ layer_types = getattr(text, "layer_types", None) or ()
+ if (
+ not layers
+ and isinstance(global_kv, int)
+ and global_kv > 0
+ and getattr(text, "attention_k_eq_v", False)
+ and (not layer_types or "full_attention" in layer_types)
+ ):
+ counts.add((text.num_attention_heads, global_kv))
+ return sorted(counts)
+
+
+def _ulysses_head_plans(
+ model,
+ size: int,
+ tp: int = 1,
+) -> None:
+ # Under tensor parallel each rank holds heads / tp of them (and kv heads / tp, or all when fewer).
+ for heads, kv in _model_heads(model):
+ ulysses_head_plan(heads // tp, kv // tp if kv % tp == 0 else kv, size)
def resolve_cp_backend(
@@ -210,11 +243,8 @@ def resolve_cp_backend(
return backend
if rl:
return "ulysses"
- heads, kv = _model_heads(model)
- if heads is None:
- return "ulysses"
try:
- ulysses_head_plan(heads // tp, kv // tp if kv % tp == 0 else kv, size)
+ _ulysses_head_plans(model, size, tp)
except ValueError:
return "ring"
return "ulysses"
@@ -1582,10 +1612,7 @@ def _check_cp_model(
)
if backend == "ulysses":
# Every rank raises here, before any collective.
- heads, kv_heads = _model_heads(model)
- if heads is not None:
- # Under tensor parallel each rank holds heads / tp of them (and kv heads / tp, or all when fewer).
- ulysses_head_plan(heads // tp, kv_heads // tp if kv_heads % tp == 0 else kv_heads, size)
+ _ulysses_head_plans(model, size, tp)
if _stateful_mixers(model) and not allow_hybrids:
raise NotImplementedError(
"Unsloth: DPO / GRPO context parallelism does not support linear attention hybrids yet."
diff --git a/unsloth/parallel_capabilities.py b/unsloth/parallel_capabilities.py
index 246309d7f03..6dc5b0c4bc1 100644
--- a/unsloth/parallel_capabilities.py
+++ b/unsloth/parallel_capabilities.py
@@ -913,8 +913,20 @@ _DEEPSEEK_CASES = {
"lora+experts": "v4_cp2_lora_experts",
},
}
+# Gemma 4: text-only Gemma4ForCausalLM, HF checkpointing only.
+_GEMMA4 = "tests/test_ulysses_gemma4.py::test_gemma4_cp_matches_fp64_reference"
+_GEMMA4_EAGER = "eager: block compile not measured for this family"
+_GEMMA4_CASES = {
+ "full": "moe_cp2|moe_cp2_window9_prompt_masked|moe_cp4|moe_dp2cp2|moe_cp2_eager_checkpoint",
+ "lora": "moe_cp2_lora",
+ "lora+experts": "moe_cp2_lora_experts|moe_cp2_lora_experts_eager_checkpoint",
+}
# Attention implementations a family's evidence ran (default: _MOE_COMMON's sdpa and eager).
_MOE_ATTENTION = {"deepseek_v4": ("eager",)}
+_MOE_TEXT_ONLY = {
+ "gemma4": "multimodal Gemma 4 refuses under Ulysses (its vision attention is bidirectional); only the "
+ "text-only Gemma4ForCausalLM ran",
+}
# Families whose cp rows also ran Unsloth's checkpoint (path A and B); the others match HF checkpointing only.
_MOE_UNSLOTH_GC_FAMILIES = ("qwen3_moe",)
# (family, precision, aux, mode) -> (parity, engagement, compile) for the rows a test proves.
@@ -990,6 +1002,10 @@ _MOE_EVIDENCE = {
for family, cases in _DEEPSEEK_CASES.items()
for precision, ids in cases.items()
},
+ **{
+ ("gemma4", precision, False, "cp"): (f"{_GEMMA4}[{ids}]",) * 2 + (_GEMMA4_EAGER,)
+ for precision, ids in _GEMMA4_CASES.items()
+ },
}
@@ -1052,6 +1068,8 @@ def _moe_cells():
if evidence:
if family in _MOE_ATTENTION:
match = {**match, "attention": _MOE_ATTENTION[family]}
+ if family in _MOE_TEXT_ONLY:
+ match = {**match, "multimodal": (False,)}
note = compiled_note = ""
if mode in ("cp", "fsdp2_cp") and family in _MOE_UNSLOTH_GC_FAMILIES:
match = {**match, "memory": _WITH_UNSLOTH_GC}
@@ -1064,6 +1082,17 @@ def _moe_cells():
# No evidence run swapped layers; block swap x tp / pp is its own cell.
match = {**match, "block_swap": (False,)}
cells.append(Cell(name, match, *evidence, note = note))
+ if family in _MOE_TEXT_ONLY:
+ cells.append(
+ Cell(
+ name + ", multimodal",
+ {**match, "multimodal": (True,)},
+ None,
+ None,
+ None,
+ note = _MOE_TEXT_ONLY[family],
+ )
+ )
if mode in ("cp", "fsdp2_cp") and family not in _MOE_UNSLOTH_GC_FAMILIES:
cells.append(
Cell(