Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Refuse tensor parallel on blocks that own parameters in the registry (#16638)

Daniel Hane2ffc7405087
diff --git a/tests/test_parallel_capabilities_tp_model.py b/tests/test_parallel_capabilities_tp_model.py
new file mode 100644
index 00000000000..37d48e308fd
--- /dev/null
+++ b/tests/test_parallel_capabilities_tp_model.py
@@ -0,0 +1,94 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Tensor parallel on a model whose attention owns a parameter directly (gpt-oss attention sinks) is refused by
+the capability registry, with tp_native's reason, before any weight is loaded or sharded. CPU only."""
+
+import types
+
+import pytest
+import torch
+
+import unsloth # noqa: F401
+from unsloth import parallel_capabilities as caps
+from unsloth.parallel_env import ParallelLayout
+
+GPT_OSS = "trl-internal-testing/tiny-GptOssForCausalLM"
+QWEN3 = "trl-internal-testing/tiny-Qwen3ForCausalLM"
+QWEN3_MOE = "trl-internal-testing/tiny-Qwen3MoeForCausalLM"
+
+
[email protected](autouse = True)
+def clean_env(monkeypatch):
+ for key in ("UNSLOTH_PARALLEL_STRICT", "UNSLOTH_TP_SIZE", "UNSLOTH_PP_SIZE", "WORLD_SIZE"):
+ monkeypatch.delenv(key, raising = False)
+
+
+def _meta(repo):
+ from transformers import AutoConfig, AutoModelForCausalLM
+ with torch.device("meta"):
+ model = AutoModelForCausalLM.from_config(AutoConfig.from_pretrained(repo))
+ return model.requires_grad_(
+ False
+ ) # a frozen base (LoRA), so the pp x mode rule does not answer first
+
+
+def _check(model, **sizes):
+ caps.check_layout(
+ ParallelLayout(fsdp2 = False, **{"tp": 1, "dp": 1, "world": 2, **sizes}),
+ None,
+ model,
+ types.SimpleNamespace(packing = False),
+ )
+
+
[email protected]("tp, pp", [(2, 1), (4, 1), (2, 2)])
+def test_gpt_oss_under_tp_is_refused_by_the_registry_before_weights_load(tp, pp):
+ model = _meta(GPT_OSS)
+ assert all(p.is_meta for p in model.parameters())
+ with pytest.raises(NotImplementedError) as info:
+ _check(model, tp = tp, pp = pp, world = tp * pp)
+ message = str(info.value)
+ assert message.startswith("Unsloth: tp x model: "), message
+ assert "own a parameter directly" in message and "Instead: drop tp" in message, message
+ query = caps.query_from(None, None, model)
+ assert query.family == "gpt_oss" and query.direct_params
+
+
+def test_the_registry_and_tp_native_refuse_the_same_model():
+ from transformers import AutoModelForCausalLM
+
+ from unsloth.models import tp_native
+
+ model = AutoModelForCausalLM.from_pretrained(GPT_OSS, dtype = torch.float32)
+ with pytest.raises(NotImplementedError, match = r"^Unsloth: tp x model: "):
+ _check(model, tp = 2)
+ with pytest.raises(ValueError, match = "owns the parameter 'sinks'"):
+ tp_native.derive_tp_plan(model, 2)
+
+
+def test_gpt_oss_without_tp_is_not_refused_by_the_rule():
+ model = _meta(GPT_OSS)
+ _check(model, dp = 2)
+
+
[email protected]("repo", [QWEN3, QWEN3_MOE])
+def test_models_whose_blocks_own_no_parameter_keep_running_under_tp(repo):
+ from unsloth.models import tp_native
+
+ model = _meta(repo)
+ _check(model, tp = 2)
+ assert not getattr(caps.query_from(None, None, model), "direct_params", False)
+ tp_native.derive_tp_plan(model, 2)
+
+
+def test_cells_table_says_gpt_oss_tp_is_refused():
+ rows = [c for c in caps.CELLS if c.name.startswith("MoE gpt_oss") and c.name.endswith(" x tp")]
+ assert rows and all("refused by the tp x model rule" in c.note for c in rows), [
+ c.note for c in rows
+ ]
+ others = [
+ c for c in caps.CELLS if c.name.startswith("MoE qwen3_moe") and c.name.endswith(" x tp")
+ ]
+ assert others and all("tensor parallel MoE runs" in c.note for c in others)
+ assert "| tp x model |" in caps.markdown_table()
diff --git a/unsloth/parallel_capabilities.py b/unsloth/parallel_capabilities.py
index 57a8d1afa8d..946a71209a6 100644
--- a/unsloth/parallel_capabilities.py
+++ b/unsloth/parallel_capabilities.py
@@ -64,6 +64,7 @@ class Query:
hybrid: bool = False # linear attention / state space layers beside attention
multimodal: bool = False
windowed: bool = False # sliding window, chunked attention or logit softcapping
+ direct_params: bool = False # an attention / MLP block owns a parameter (attention sinks)
mode: str = "lora" # lora, qlora, full, fp8 (float8_training, LoRA or full)
packing: bool = False
cp_backend: Optional[str] = None # ulysses, ring (resolved)
@@ -259,6 +260,14 @@ RULES = (
"torch.optim.Muon (it steps DTensor shards as whole matrices), adamw_torch / adamw_8bit, or DDP "
"(dp without fsdp) for another Muon",
),
+ Rule(
+ "tp x model",
+ "tensor parallel on attention / MLP blocks that own a parameter directly (gpt-oss attention sinks)",
+ lambda q: q.tp > 1 and q.direct_params,
+ "tensor parallel shards the projections inside a block; a parameter the block owns itself (not inside a "
+ "projection) has no sharding rule (models/tp_native.py refuses it)",
+ "drop tp (DDP, or UNSLOTH_FSDP2=1, over the dp axis)",
+ ),
Rule(
"pp / tp x vLLM",
"pipeline / tensor parallel with vLLM (fast_inference)",
@@ -730,8 +739,14 @@ _MOE_EVIDENCE = {
}
+# Cells table only; a real model is read by structure (_direct_params).
+_DIRECT_PARAM_FAMILIES = ("gpt_oss",)
+
+
def _moe_note(family, precision, aux, mode):
if mode == "tp":
+ if family in _DIRECT_PARAM_FAMILIES:
+ return "refused by the tp x model rule (its attention owns the sinks parameter)"
return "tensor parallel MoE runs (models/tp_native.py, its own gloo tests); no registry parity row yet"
if aux and mode == "cp":
return "refused by the MoE x pp / cp rule (aux-loss injection first)"
@@ -951,6 +966,30 @@ def _model_traits(model):
return family, hybrid, multimodal, windowed
+def _direct_params(model) -> bool:
+ """An attention / MLP block owns a parameter itself (attention sinks), as tp_native.derive_tp_plan refuses.
+ MoE blocks are skipped as tp_native skips them (replicated). Meta-device safe."""
+ if not callable(getattr(model, "named_modules", None)):
+ return False
+ try:
+ from .models import tp_native
+ _, layers = tp_native._decoder_layers(model)
+ except Exception:
+ return False
+ for layer in layers:
+ for child_name, child in layer.named_children():
+ attention = child_name in tp_native._ATTN_NAMES
+ if not attention and child_name not in tp_native._MLP_NAMES:
+ continue
+ if not tp_native._has_params(child) or (
+ not attention and tp_native._is_moe_block(child)
+ ):
+ continue
+ if next(child.named_parameters(recurse = False), None) is not None:
+ return True
+ return False
+
+
def _low_precision(model) -> Optional[str]:
"""Return "fp8" if fp8 LoRA is requested / attached or torchao Float8Linear layers exist, else None."""
stack, seen = [model], set()
@@ -1173,6 +1212,7 @@ def query_from(
hybrid = hybrid,
multimodal = multimodal,
windowed = windowed,
+ direct_params = _direct_params(model),
mode = mode,
packing = packing,
cp_backend = backend,