Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Refuse tensor parallel on blocks that own parameters in the registry (#16638)
Daniel Hane2ffc7405087
diff --git a/tests/test_parallel_capabilities_tp_model.py b/tests/test_parallel_capabilities_tp_model.pynew file mode 100644index 00000000000..37d48e308fd--- /dev/null+++ b/tests/test_parallel_capabilities_tp_model.py@@ -0,0 +1,94 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Tensor parallel on a model whose attention owns a parameter directly (gpt-oss attention sinks) is refused by+the capability registry, with tp_native's reason, before any weight is loaded or sharded. CPU only."""++import types++import pytest+import torch++import unsloth # noqa: F401+from unsloth import parallel_capabilities as caps+from unsloth.parallel_env import ParallelLayout++GPT_OSS = "trl-internal-testing/tiny-GptOssForCausalLM"+QWEN3 = "trl-internal-testing/tiny-Qwen3ForCausalLM"+QWEN3_MOE = "trl-internal-testing/tiny-Qwen3MoeForCausalLM"++[email protected](autouse = True)+def clean_env(monkeypatch):+ for key in ("UNSLOTH_PARALLEL_STRICT", "UNSLOTH_TP_SIZE", "UNSLOTH_PP_SIZE", "WORLD_SIZE"):+ monkeypatch.delenv(key, raising = False)+++def _meta(repo):+ from transformers import AutoConfig, AutoModelForCausalLM+ with torch.device("meta"):+ model = AutoModelForCausalLM.from_config(AutoConfig.from_pretrained(repo))+ return model.requires_grad_(+ False+ ) # a frozen base (LoRA), so the pp x mode rule does not answer first+++def _check(model, **sizes):+ caps.check_layout(+ ParallelLayout(fsdp2 = False, **{"tp": 1, "dp": 1, "world": 2, **sizes}),+ None,+ model,+ types.SimpleNamespace(packing = False),+ )++[email protected]("tp, pp", [(2, 1), (4, 1), (2, 2)])+def test_gpt_oss_under_tp_is_refused_by_the_registry_before_weights_load(tp, pp):+ model = _meta(GPT_OSS)+ assert all(p.is_meta for p in model.parameters())+ with pytest.raises(NotImplementedError) as info:+ _check(model, tp = tp, pp = pp, world = tp * pp)+ message = str(info.value)+ assert message.startswith("Unsloth: tp x model: "), message+ assert "own a parameter directly" in message and "Instead: drop tp" in message, message+ query = caps.query_from(None, None, model)+ assert query.family == "gpt_oss" and query.direct_params+++def test_the_registry_and_tp_native_refuse_the_same_model():+ from transformers import AutoModelForCausalLM++ from unsloth.models import tp_native++ model = AutoModelForCausalLM.from_pretrained(GPT_OSS, dtype = torch.float32)+ with pytest.raises(NotImplementedError, match = r"^Unsloth: tp x model: "):+ _check(model, tp = 2)+ with pytest.raises(ValueError, match = "owns the parameter 'sinks'"):+ tp_native.derive_tp_plan(model, 2)+++def test_gpt_oss_without_tp_is_not_refused_by_the_rule():+ model = _meta(GPT_OSS)+ _check(model, dp = 2)++[email protected]("repo", [QWEN3, QWEN3_MOE])+def test_models_whose_blocks_own_no_parameter_keep_running_under_tp(repo):+ from unsloth.models import tp_native++ model = _meta(repo)+ _check(model, tp = 2)+ assert not getattr(caps.query_from(None, None, model), "direct_params", False)+ tp_native.derive_tp_plan(model, 2)+++def test_cells_table_says_gpt_oss_tp_is_refused():+ rows = [c for c in caps.CELLS if c.name.startswith("MoE gpt_oss") and c.name.endswith(" x tp")]+ assert rows and all("refused by the tp x model rule" in c.note for c in rows), [+ c.note for c in rows+ ]+ others = [+ c for c in caps.CELLS if c.name.startswith("MoE qwen3_moe") and c.name.endswith(" x tp")+ ]+ assert others and all("tensor parallel MoE runs" in c.note for c in others)+ assert "| tp x model |" in caps.markdown_table()diff --git a/unsloth/parallel_capabilities.py b/unsloth/parallel_capabilities.pyindex 57a8d1afa8d..946a71209a6 100644--- a/unsloth/parallel_capabilities.py+++ b/unsloth/parallel_capabilities.py@@ -64,6 +64,7 @@ class Query:hybrid: bool = False # linear attention / state space layers beside attentionmultimodal: bool = Falsewindowed: bool = False # sliding window, chunked attention or logit softcapping+ direct_params: bool = False # an attention / MLP block owns a parameter (attention sinks)mode: str = "lora" # lora, qlora, full, fp8 (float8_training, LoRA or full)packing: bool = Falsecp_backend: Optional[str] = None # ulysses, ring (resolved)@@ -259,6 +260,14 @@ RULES = ("torch.optim.Muon (it steps DTensor shards as whole matrices), adamw_torch / adamw_8bit, or DDP ""(dp without fsdp) for another Muon",),+ Rule(+ "tp x model",+ "tensor parallel on attention / MLP blocks that own a parameter directly (gpt-oss attention sinks)",+ lambda q: q.tp > 1 and q.direct_params,+ "tensor parallel shards the projections inside a block; a parameter the block owns itself (not inside a "+ "projection) has no sharding rule (models/tp_native.py refuses it)",+ "drop tp (DDP, or UNSLOTH_FSDP2=1, over the dp axis)",+ ),Rule("pp / tp x vLLM","pipeline / tensor parallel with vLLM (fast_inference)",@@ -730,8 +739,14 @@ _MOE_EVIDENCE = {}+# Cells table only; a real model is read by structure (_direct_params).+_DIRECT_PARAM_FAMILIES = ("gpt_oss",)++def _moe_note(family, precision, aux, mode):if mode == "tp":+ if family in _DIRECT_PARAM_FAMILIES:+ return "refused by the tp x model rule (its attention owns the sinks parameter)"return "tensor parallel MoE runs (models/tp_native.py, its own gloo tests); no registry parity row yet"if aux and mode == "cp":return "refused by the MoE x pp / cp rule (aux-loss injection first)"@@ -951,6 +966,30 @@ def _model_traits(model):return family, hybrid, multimodal, windowed+def _direct_params(model) -> bool:+ """An attention / MLP block owns a parameter itself (attention sinks), as tp_native.derive_tp_plan refuses.+ MoE blocks are skipped as tp_native skips them (replicated). Meta-device safe."""+ if not callable(getattr(model, "named_modules", None)):+ return False+ try:+ from .models import tp_native+ _, layers = tp_native._decoder_layers(model)+ except Exception:+ return False+ for layer in layers:+ for child_name, child in layer.named_children():+ attention = child_name in tp_native._ATTN_NAMES+ if not attention and child_name not in tp_native._MLP_NAMES:+ continue+ if not tp_native._has_params(child) or (+ not attention and tp_native._is_moe_block(child)+ ):+ continue+ if next(child.named_parameters(recurse = False), None) is not None:+ return True+ return False++def _low_precision(model) -> Optional[str]:"""Return "fp8" if fp8 LoRA is requested / attached or torchao Float8Linear layers exist, else None."""stack, seen = [model], set()@@ -1173,6 +1212,7 @@ def query_from(hybrid = hybrid,multimodal = multimodal,windowed = windowed,+ direct_params = _direct_params(model),mode = mode,packing = packing,cp_backend = backend,