Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Capability registry: refuse tensor parallel where tp_native cannot shard, and add the zoo MoE families (#16709)

Daniel Hanfd50f8fd8b1f
diff --git a/tests/test_parallel_capabilities_tp_blocked.py b/tests/test_parallel_capabilities_tp_blocked.py
new file mode 100644
index 00000000000..7ae082320d7
--- /dev/null
+++ b/tests/test_parallel_capabilities_tp_blocked.py
@@ -0,0 +1,151 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Registry refuses tp for models tp_native cannot shard, with its reason, before weights load. CPU, meta models."""
+
+import types
+
+import pytest
+import torch
+
+import unsloth # noqa: F401
+from unsloth import parallel_capabilities as caps
+from unsloth.parallel_env import ParallelLayout
+
+
[email protected](autouse = True)
+def clean_env(monkeypatch):
+ for key in ("UNSLOTH_PARALLEL_STRICT", "UNSLOTH_TP_SIZE", "UNSLOTH_PP_SIZE", "WORLD_SIZE"):
+ monkeypatch.delenv(key, raising = False)
+
+
+_SMALL = dict(
+ vocab_size = 128,
+ hidden_size = 64,
+ intermediate_size = 32,
+ num_hidden_layers = 1,
+ num_attention_heads = 4,
+ num_key_value_heads = 4,
+ num_experts_per_tok = 2,
+)
+
+
+def _meta(family):
+ from transformers import AutoModelForCausalLM, DeepseekV3Config, OlmoeConfig, Qwen3MoeConfig
+
+ if family == "olmoe":
+ config = OlmoeConfig(**_SMALL, num_experts = 4)
+ elif family == "deepseek_v3":
+ config = DeepseekV3Config(
+ **_SMALL,
+ n_routed_experts = 4,
+ n_group = 1,
+ topk_group = 1,
+ first_k_dense_replace = 0,
+ moe_intermediate_size = 32,
+ kv_lora_rank = 16,
+ q_lora_rank = 32,
+ qk_rope_head_dim = 8,
+ qk_nope_head_dim = 8,
+ v_head_dim = 16,
+ )
+ else:
+ config = Qwen3MoeConfig(**_SMALL, num_experts = 4, moe_intermediate_size = 32, head_dim = 16)
+ with torch.device("meta"):
+ model = AutoModelForCausalLM.from_config(config)
+ return model.requires_grad_(False)
+
+
+def _check(model, tp = 2):
+ caps.check_layout(
+ ParallelLayout(fsdp2 = False, tp = tp, dp = 1, world = tp),
+ None,
+ model,
+ types.SimpleNamespace(packing = False),
+ )
+
+
+ "family, reason",
+ [
+ ("olmoe", "normalizes 64 features, not one head"),
+ ("deepseek_v3", "multi-head latent attention"),
+ ],
+)
+def test_models_tp_native_cannot_shard_are_refused_by_the_registry(family, reason):
+ from unsloth.models import tp_native
+
+ model = _meta(family)
+ with pytest.raises(ValueError, match = reason):
+ tp_native.derive_tp_plan(model, 2)
+ query = caps.query_from(ParallelLayout(fsdp2 = False, tp = 2, dp = 1, world = 2), None, model)
+ assert reason in query.tp_blocker, query
+ with pytest.raises(NotImplementedError) as info:
+ _check(model)
+ message = str(info.value)
+ assert message.startswith("Unsloth: tp x model: "), message
+ assert reason in message and "Instead: drop tp" in message, message
+ assert all(p.is_meta for p in model.parameters()) # nothing loaded or moved
+
+
+def test_a_model_tp_native_shards_is_not_refused_and_no_tp_asks_nothing():
+ model = _meta("qwen3_moe") # per-head q / k norms: tp_native shards it
+ _check(model)
+ assert (
+ caps.query_from(ParallelLayout(fsdp2 = False, tp = 2, dp = 1, world = 2), None, model).tp_blocker
+ == ""
+ )
+ olmoe = _meta("olmoe")
+ assert (
+ caps.query_from(ParallelLayout(fsdp2 = False, tp = 1, dp = 2, world = 2), None, olmoe).tp_blocker
+ == ""
+ )
+
+
+def test_cells_table_marks_olmoe_and_deepseek_v3_tp_refused():
+ for family, why in (("olmoe", "not one head"), ("deepseek_v3", "MLA")):
+ rows = [
+ c
+ for c in caps.CELLS
+ if c.name.startswith(f"MoE {family} x ") and c.name.endswith(" x tp")
+ ]
+ assert rows, family
+ for c in rows:
+ assert c.missing() and "refused by the tp x model rule" in c.note and why in c.note, (
+ c.name,
+ c.note,
+ )
+
+
+def test_zoo_moe_matrix_families_are_registry_moe_families():
+ for family in (
+ "olmoe",
+ "mixtral",
+ "qwen2_moe",
+ "qwen3_next",
+ "qwen3_vl_moe",
+ "llama4",
+ "glm4_moe",
+ "ernie4_5_moe",
+ "nemotron_h",
+ "jamba",
+ "granitemoehybrid",
+ "gemma4",
+ "lfm2_moe",
+ ):
+ assert family in caps.MOE_FAMILIES, family
+ assert any(c.name.startswith(f"MoE {family} x ") for c in caps.CELLS), family
+
+
+def test_a_multimodal_text_config_names_the_family_without_text():
+ text = types.SimpleNamespace(model_type = "qwen3_5_moe_text", layer_types = [])
+ model = types.SimpleNamespace(
+ config = types.SimpleNamespace(model_type = "qwen3_5_moe", text_config = text)
+ )
+ assert caps._model_traits(model)[0] == "qwen3_5_moe"
+ assert (
+ caps._model_traits(types.SimpleNamespace(config = types.SimpleNamespace(model_type = "olmoe")))[
+ 0
+ ]
+ == "olmoe"
+ )
diff --git a/unsloth/parallel_capabilities.py b/unsloth/parallel_capabilities.py
index 946a71209a6..6456eda0d3f 100644
--- a/unsloth/parallel_capabilities.py
+++ b/unsloth/parallel_capabilities.py
@@ -65,6 +65,8 @@ class Query:
multimodal: bool = False
windowed: bool = False # sliding window, chunked attention or logit softcapping
direct_params: bool = False # an attention / MLP block owns a parameter (attention sinks)
+ # tp_native's refusal at q.tp ranks; "" when it shards or the model is already split (_tp_blocker).
+ tp_blocker: str = ""
mode: str = "lora" # lora, qlora, full, fp8 (float8_training, LoRA or full)
packing: bool = False
cp_backend: Optional[str] = None # ulysses, ring (resolved)
@@ -268,6 +270,14 @@ RULES = (
"projection) has no sharding rule (models/tp_native.py refuses it)",
"drop tp (DDP, or UNSLOTH_FSDP2=1, over the dp axis)",
),
+ Rule(
+ "tp x model",
+ "tensor parallel on models tensor parallel has no sharding rule for (multi-head latent attention, a q / k "
+ "norm over all heads)",
+ lambda q: q.tp > 1 and bool(q.tp_blocker),
+ "models/tp_native.py refuses the model before any weight loads (its reason follows the layout below)",
+ "drop tp (DDP, or UNSLOTH_FSDP2=1, over the dp axis)",
+ ),
Rule(
"pp / tp x vLLM",
"pipeline / tensor parallel with vLLM (fast_inference)",
@@ -666,7 +676,27 @@ CELLS = (
# MoE rows: family x precision x router aux loss x parallel mode. Evidence where a CPU gloo run proves it; every
# other row is refused (under cp / pp / tp) or reported unverified (dp alone), with what it waits for.
-MOE_FAMILIES = ("qwen3_moe", "qwen3_5_moe", "gpt_oss", "deepseek_v3", "kimi_linear")
+# Zoo MoE matrix families (structure checked on transformers 5.17); family drops "_text" (_model_traits).
+MOE_FAMILIES = (
+ "qwen3_moe",
+ "qwen3_5_moe",
+ "gpt_oss",
+ "deepseek_v3",
+ "kimi_linear",
+ "olmoe",
+ "mixtral",
+ "qwen2_moe",
+ "qwen3_next",
+ "qwen3_vl_moe",
+ "llama4",
+ "glm4_moe",
+ "ernie4_5_moe",
+ "nemotron_h",
+ "jamba",
+ "granitemoehybrid",
+ "gemma4",
+ "lfm2_moe",
+)
_MOE_PRECISION = {
"lora": {"mode": ("lora",), "expert_lora": (False,)},
"lora+experts": {"mode": ("lora",), "expert_lora": (True,)},
@@ -739,14 +769,20 @@ _MOE_EVIDENCE = {
}
-# Cells table only; a real model is read by structure (_direct_params).
+# Cells table only; a real model is read by structure (_direct_params, _tp_blocker).
_DIRECT_PARAM_FAMILIES = ("gpt_oss",)
+_TP_BLOCKED_FAMILIES = {
+ "deepseek_v3": "multi-head latent attention (MLA, kv_lora_rank)",
+ "olmoe": "q_norm / k_norm normalize all heads' features, not one head",
+}
def _moe_note(family, precision, aux, mode):
if mode == "tp":
if family in _DIRECT_PARAM_FAMILIES:
return "refused by the tp x model rule (its attention owns the sinks parameter)"
+ if family in _TP_BLOCKED_FAMILIES:
+ return f"refused by the tp x model rule (models/tp_native.py: {_TP_BLOCKED_FAMILIES[family]})"
return "tensor parallel MoE runs (models/tp_native.py, its own gloo tests); no registry parity row yet"
if aux and mode == "cp":
return "refused by the MoE x pp / cp rule (aux-loss injection first)"
@@ -898,6 +934,7 @@ def _describe(q: Query) -> str:
+ (" x ".join(parts) or "single rank")
+ f", trainer {q.trainer}"
+ (f", model {q.family}" if q.family else "")
+ + (f"; models/tp_native.py: {q.tp_blocker}" if q.tp > 1 and q.tp_blocker else "")
)
@@ -950,6 +987,9 @@ def _model_traits(model):
return "", False, False, False
text = getattr(config, "text_config", None) or config
family = str(getattr(text, "model_type", "") or getattr(config, "model_type", "") or "")
+ # A multimodal text config: qwen3_5_moe_text is the qwen3_5_moe family.
+ if family.endswith("_text"):
+ family = family[: -len("_text")]
layer_types = list(getattr(text, "layer_types", None) or [])
hybrid = any(
t not in ("full_attention", "sliding_attention", "chunked_attention") for t in layer_types
@@ -990,6 +1030,29 @@ def _direct_params(model) -> bool:
return False
+def _tp_blocker(model, tp) -> str:
+ """derive_tp_plan's ValueError for `model` at `tp` ranks, "" when it shards. Only for an unsplit model:
+ a sharded model's local shapes are not the model's. Meta-device safe."""
+ if not isinstance(tp, int) or tp <= 1 or not callable(getattr(model, "named_modules", None)):
+ return ""
+ try:
+ from .models import tp_native
+ if tp_native.is_tensor_parallel(model) or any(
+ type(p).__name__ == "DTensor" for p in model.parameters()
+ ):
+ return ""
+ tp_native._decoder_layers(model) # not a decoder model: no answer
+ except Exception:
+ return ""
+ try:
+ tp_native.derive_tp_plan(model, tp)
+ except ValueError as e:
+ return str(e)
+ except Exception:
+ return ""
+ return ""
+
+
def _low_precision(model) -> Optional[str]:
"""Return "fp8" if fp8 LoRA is requested / attached or torchao Float8Linear layers exist, else None."""
stack, seen = [model], set()
@@ -1213,6 +1276,7 @@ def query_from(
multimodal = multimodal,
windowed = windowed,
direct_params = _direct_params(model),
+ tp_blocker = _tp_blocker(model, getattr(layout, "tp", 1)),
mode = mode,
packing = packing,
cp_backend = backend,