Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Capability registry: refuse tensor parallel where tp_native cannot shard, and add the zoo MoE families (#16709)
Daniel Hanfd50f8fd8b1f
diff --git a/tests/test_parallel_capabilities_tp_blocked.py b/tests/test_parallel_capabilities_tp_blocked.pynew file mode 100644index 00000000000..7ae082320d7--- /dev/null+++ b/tests/test_parallel_capabilities_tp_blocked.py@@ -0,0 +1,151 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Registry refuses tp for models tp_native cannot shard, with its reason, before weights load. CPU, meta models."""++import types++import pytest+import torch++import unsloth # noqa: F401+from unsloth import parallel_capabilities as caps+from unsloth.parallel_env import ParallelLayout++[email protected](autouse = True)+def clean_env(monkeypatch):+ for key in ("UNSLOTH_PARALLEL_STRICT", "UNSLOTH_TP_SIZE", "UNSLOTH_PP_SIZE", "WORLD_SIZE"):+ monkeypatch.delenv(key, raising = False)+++_SMALL = dict(+ vocab_size = 128,+ hidden_size = 64,+ intermediate_size = 32,+ num_hidden_layers = 1,+ num_attention_heads = 4,+ num_key_value_heads = 4,+ num_experts_per_tok = 2,+)+++def _meta(family):+ from transformers import AutoModelForCausalLM, DeepseekV3Config, OlmoeConfig, Qwen3MoeConfig++ if family == "olmoe":+ config = OlmoeConfig(**_SMALL, num_experts = 4)+ elif family == "deepseek_v3":+ config = DeepseekV3Config(+ **_SMALL,+ n_routed_experts = 4,+ n_group = 1,+ topk_group = 1,+ first_k_dense_replace = 0,+ moe_intermediate_size = 32,+ kv_lora_rank = 16,+ q_lora_rank = 32,+ qk_rope_head_dim = 8,+ qk_nope_head_dim = 8,+ v_head_dim = 16,+ )+ else:+ config = Qwen3MoeConfig(**_SMALL, num_experts = 4, moe_intermediate_size = 32, head_dim = 16)+ with torch.device("meta"):+ model = AutoModelForCausalLM.from_config(config)+ return model.requires_grad_(False)+++def _check(model, tp = 2):+ caps.check_layout(+ ParallelLayout(fsdp2 = False, tp = tp, dp = 1, world = tp),+ None,+ model,+ types.SimpleNamespace(packing = False),+ )+++ "family, reason",+ [+ ("olmoe", "normalizes 64 features, not one head"),+ ("deepseek_v3", "multi-head latent attention"),+ ],+)+def test_models_tp_native_cannot_shard_are_refused_by_the_registry(family, reason):+ from unsloth.models import tp_native++ model = _meta(family)+ with pytest.raises(ValueError, match = reason):+ tp_native.derive_tp_plan(model, 2)+ query = caps.query_from(ParallelLayout(fsdp2 = False, tp = 2, dp = 1, world = 2), None, model)+ assert reason in query.tp_blocker, query+ with pytest.raises(NotImplementedError) as info:+ _check(model)+ message = str(info.value)+ assert message.startswith("Unsloth: tp x model: "), message+ assert reason in message and "Instead: drop tp" in message, message+ assert all(p.is_meta for p in model.parameters()) # nothing loaded or moved+++def test_a_model_tp_native_shards_is_not_refused_and_no_tp_asks_nothing():+ model = _meta("qwen3_moe") # per-head q / k norms: tp_native shards it+ _check(model)+ assert (+ caps.query_from(ParallelLayout(fsdp2 = False, tp = 2, dp = 1, world = 2), None, model).tp_blocker+ == ""+ )+ olmoe = _meta("olmoe")+ assert (+ caps.query_from(ParallelLayout(fsdp2 = False, tp = 1, dp = 2, world = 2), None, olmoe).tp_blocker+ == ""+ )+++def test_cells_table_marks_olmoe_and_deepseek_v3_tp_refused():+ for family, why in (("olmoe", "not one head"), ("deepseek_v3", "MLA")):+ rows = [+ c+ for c in caps.CELLS+ if c.name.startswith(f"MoE {family} x ") and c.name.endswith(" x tp")+ ]+ assert rows, family+ for c in rows:+ assert c.missing() and "refused by the tp x model rule" in c.note and why in c.note, (+ c.name,+ c.note,+ )+++def test_zoo_moe_matrix_families_are_registry_moe_families():+ for family in (+ "olmoe",+ "mixtral",+ "qwen2_moe",+ "qwen3_next",+ "qwen3_vl_moe",+ "llama4",+ "glm4_moe",+ "ernie4_5_moe",+ "nemotron_h",+ "jamba",+ "granitemoehybrid",+ "gemma4",+ "lfm2_moe",+ ):+ assert family in caps.MOE_FAMILIES, family+ assert any(c.name.startswith(f"MoE {family} x ") for c in caps.CELLS), family+++def test_a_multimodal_text_config_names_the_family_without_text():+ text = types.SimpleNamespace(model_type = "qwen3_5_moe_text", layer_types = [])+ model = types.SimpleNamespace(+ config = types.SimpleNamespace(model_type = "qwen3_5_moe", text_config = text)+ )+ assert caps._model_traits(model)[0] == "qwen3_5_moe"+ assert (+ caps._model_traits(types.SimpleNamespace(config = types.SimpleNamespace(model_type = "olmoe")))[+ 0+ ]+ == "olmoe"+ )diff --git a/unsloth/parallel_capabilities.py b/unsloth/parallel_capabilities.pyindex 946a71209a6..6456eda0d3f 100644--- a/unsloth/parallel_capabilities.py+++ b/unsloth/parallel_capabilities.py@@ -65,6 +65,8 @@ class Query:multimodal: bool = Falsewindowed: bool = False # sliding window, chunked attention or logit softcappingdirect_params: bool = False # an attention / MLP block owns a parameter (attention sinks)+ # tp_native's refusal at q.tp ranks; "" when it shards or the model is already split (_tp_blocker).+ tp_blocker: str = ""mode: str = "lora" # lora, qlora, full, fp8 (float8_training, LoRA or full)packing: bool = Falsecp_backend: Optional[str] = None # ulysses, ring (resolved)@@ -268,6 +270,14 @@ RULES = ("projection) has no sharding rule (models/tp_native.py refuses it)","drop tp (DDP, or UNSLOTH_FSDP2=1, over the dp axis)",),+ Rule(+ "tp x model",+ "tensor parallel on models tensor parallel has no sharding rule for (multi-head latent attention, a q / k "+ "norm over all heads)",+ lambda q: q.tp > 1 and bool(q.tp_blocker),+ "models/tp_native.py refuses the model before any weight loads (its reason follows the layout below)",+ "drop tp (DDP, or UNSLOTH_FSDP2=1, over the dp axis)",+ ),Rule("pp / tp x vLLM","pipeline / tensor parallel with vLLM (fast_inference)",@@ -666,7 +676,27 @@ CELLS = (# MoE rows: family x precision x router aux loss x parallel mode. Evidence where a CPU gloo run proves it; every# other row is refused (under cp / pp / tp) or reported unverified (dp alone), with what it waits for.-MOE_FAMILIES = ("qwen3_moe", "qwen3_5_moe", "gpt_oss", "deepseek_v3", "kimi_linear")+# Zoo MoE matrix families (structure checked on transformers 5.17); family drops "_text" (_model_traits).+MOE_FAMILIES = (+ "qwen3_moe",+ "qwen3_5_moe",+ "gpt_oss",+ "deepseek_v3",+ "kimi_linear",+ "olmoe",+ "mixtral",+ "qwen2_moe",+ "qwen3_next",+ "qwen3_vl_moe",+ "llama4",+ "glm4_moe",+ "ernie4_5_moe",+ "nemotron_h",+ "jamba",+ "granitemoehybrid",+ "gemma4",+ "lfm2_moe",+)_MOE_PRECISION = {"lora": {"mode": ("lora",), "expert_lora": (False,)},"lora+experts": {"mode": ("lora",), "expert_lora": (True,)},@@ -739,14 +769,20 @@ _MOE_EVIDENCE = {}-# Cells table only; a real model is read by structure (_direct_params).+# Cells table only; a real model is read by structure (_direct_params, _tp_blocker)._DIRECT_PARAM_FAMILIES = ("gpt_oss",)+_TP_BLOCKED_FAMILIES = {+ "deepseek_v3": "multi-head latent attention (MLA, kv_lora_rank)",+ "olmoe": "q_norm / k_norm normalize all heads' features, not one head",+}def _moe_note(family, precision, aux, mode):if mode == "tp":if family in _DIRECT_PARAM_FAMILIES:return "refused by the tp x model rule (its attention owns the sinks parameter)"+ if family in _TP_BLOCKED_FAMILIES:+ return f"refused by the tp x model rule (models/tp_native.py: {_TP_BLOCKED_FAMILIES[family]})"return "tensor parallel MoE runs (models/tp_native.py, its own gloo tests); no registry parity row yet"if aux and mode == "cp":return "refused by the MoE x pp / cp rule (aux-loss injection first)"@@ -898,6 +934,7 @@ def _describe(q: Query) -> str:+ (" x ".join(parts) or "single rank")+ f", trainer {q.trainer}"+ (f", model {q.family}" if q.family else "")+ + (f"; models/tp_native.py: {q.tp_blocker}" if q.tp > 1 and q.tp_blocker else ""))@@ -950,6 +987,9 @@ def _model_traits(model):return "", False, False, Falsetext = getattr(config, "text_config", None) or configfamily = str(getattr(text, "model_type", "") or getattr(config, "model_type", "") or "")+ # A multimodal text config: qwen3_5_moe_text is the qwen3_5_moe family.+ if family.endswith("_text"):+ family = family[: -len("_text")]layer_types = list(getattr(text, "layer_types", None) or [])hybrid = any(t not in ("full_attention", "sliding_attention", "chunked_attention") for t in layer_types@@ -990,6 +1030,29 @@ def _direct_params(model) -> bool:return False+def _tp_blocker(model, tp) -> str:+ """derive_tp_plan's ValueError for `model` at `tp` ranks, "" when it shards. Only for an unsplit model:+ a sharded model's local shapes are not the model's. Meta-device safe."""+ if not isinstance(tp, int) or tp <= 1 or not callable(getattr(model, "named_modules", None)):+ return ""+ try:+ from .models import tp_native+ if tp_native.is_tensor_parallel(model) or any(+ type(p).__name__ == "DTensor" for p in model.parameters()+ ):+ return ""+ tp_native._decoder_layers(model) # not a decoder model: no answer+ except Exception:+ return ""+ try:+ tp_native.derive_tp_plan(model, tp)+ except ValueError as e:+ return str(e)+ except Exception:+ return ""+ return ""++def _low_precision(model) -> Optional[str]:"""Return "fp8" if fp8 LoRA is requested / attached or torchao Float8Linear layers exist, else None."""stack, seen = [model], set()@@ -1213,6 +1276,7 @@ def query_from(multimodal = multimodal,windowed = windowed,direct_params = _direct_params(model),+ tp_blocker = _tp_blocker(model, getattr(layout, "tp", 1)),mode = mode,packing = packing,cp_backend = backend,