Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""The registry follows the measured composition evidence. CPU only."""
import dataclasses
import types
import pytest
import torch
import unsloth # noqa: F401
from unsloth import parallel_capabilities as caps
from unsloth.parallel_env import ParallelLayout
TINY = "trl-internal-testing/tiny-Qwen3ForCausalLM"
_ENV = (
"UNSLOTH_PARALLEL",
"UNSLOTH_TP_SIZE",
"UNSLOTH_PP_SIZE",
"UNSLOTH_CP_SIZE",
"UNSLOTH_FSDP2",
"UNSLOTH_PARALLEL_STRICT",
"WORLD_SIZE",
)
@pytest.fixture(autouse = True)
def clean_env(monkeypatch):
for key in _ENV:
monkeypatch.delenv(key, raising = False)
yield monkeypatch
def _tiny():
from transformers import AutoConfig, AutoModelForCausalLM
try:
config = AutoConfig.from_pretrained(TINY)
except Exception as e: # offline host without the tiny repo cached
pytest.skip(f"{TINY}: {e}")
config.num_hidden_layers = 2
torch.manual_seed(0)
return AutoModelForCausalLM.from_config(config)
def _packed(**kw):
base = dict(
trainer = "sft",
family = "qwen3",
packing = True,
cp = 2,
dp = 1,
cp_backend = "ulysses",
attention = "sdpa",
compile = "eager",
mode = "lora",
torch = (2, 13),
)
base.update(kw)
return caps.Query(**base)
# Hybrid (Gated DeltaNet + attention) cp x native FSDP2.
def _hybrid(**kw):
base = dict(
trainer = "sft",
family = "qwen3_5",
hybrid = True,
cp = 2,
dp = 2,
fsdp2 = True,
cp_backend = "ulysses",
attention = "sdpa",
compile = "eager",
mode = "lora",
torch = (2, 13),
)
base.update(kw)
return caps.Query(**base)
def test_hybrid_qwen3_5_cp_x_native_fsdp2_is_supported_under_strict(clean_env):
status, cell = caps.cell_status(_hybrid())
assert status == "supported", cell
assert "hybrid" in cell.name and not cell.missing()
assert "test_hybrid_qwen3_5_cp_with_native_fsdp2_matches_one_process" in cell.parity
clean_env.setenv("UNSLOTH_PARALLEL_STRICT", "1")
caps.check(_hybrid())
@pytest.mark.parametrize(
"kw",
[
{"family": "qwen3_next"},
{"attention": "eager"},
{"dp": 4},
{"mode": "full"},
{"memory": "gc"},
{"compile": "block"},
],
)
def test_hybrid_cp_x_native_fsdp2_outside_the_measured_cell_stays_unverified(kw, clean_env):
assert caps.cell_status(_hybrid(**kw))[0] == "unverified"
clean_env.setenv("UNSLOTH_PARALLEL_STRICT", "1")
with pytest.raises(NotImplementedError, match = "no numeric parity"):
caps.check(_hybrid(**kw))
# Packed / padding-free rows x cp Ulysses.
@pytest.mark.parametrize(
"kw",
[
{},
{"family": "gemma3", "windowed": True, "attention": "eager"},
{"family": "mistral", "windowed": True},
{"family": "gemma2", "windowed": True, "attention": "eager"},
{"mode": "full"},
],
)
def test_packed_rows_under_generic_ulysses_eager_are_supported(kw):
status, cell = caps.cell_status(_packed(**kw))
assert status == "supported", cell
assert "packed rows x cp Ulysses" in cell.name and not cell.missing()
assert "test_generic_ulysses_matches_one_process" in cell.parity
caps.check(_packed(**kw))
def test_packed_rows_through_query_from_reach_the_eager_cell():
model = _tiny()
layout = types.SimpleNamespace(dp = 1, tp = 1, pp = 1, cp = 2, fsdp2 = False, cp_backend = "ulysses")
args = types.SimpleNamespace(packing = False, padding_free = True, gradient_checkpointing = False)
trl = pytest.importorskip("trl")
query = caps.query_from(
layout, trl.SFTTrainer, model, args, cp_backend = "ulysses", peft_config = object()
)
assert query.packing and query.family == "qwen3" and query.memory == "none"
status, cell = caps.cell_status(query)
assert status == "supported" and "packed rows" in cell.name, (status, cell)
@pytest.mark.parametrize("family", ["qwen3", "llama"])
def test_packed_rows_under_block_compile_are_supported_on_torch_2_13(family, clean_env):
query = _packed(compile = "block", family = family)
status, cell = caps.cell_status(query)
assert (
status == "supported" and cell.name.endswith("Unsloth block compile") and not cell.missing()
)
for kind in (cell.parity, cell.engagement, cell.compile):
assert "test_block_compile_packed_rows_under_cp_match_one_process" in kind
clean_env.setenv("UNSLOTH_PARALLEL_STRICT", "1")
caps.check(query)
@pytest.mark.parametrize(
"kw",
[
{"torch": (2, 12)}, # the compile test passes on torch >= 2.13 only
{"family": "gemma3", "windowed": True, "attention": "eager"},
{"family": "llama", "torch": (2, 12)},
{"mode": "full"},
{"attention": "eager"},
],
)
def test_packed_rows_under_block_compile_outside_the_measured_cell_stay_unverified(kw, clean_env):
query = _packed(compile = "block", **kw)
status, detail = caps.cell_status(query)
assert status == "unverified", detail
assert "lacks engagement, compile evidence" in detail and "torch >= 2.13 only" in detail, detail
clean_env.setenv("UNSLOTH_PARALLEL_STRICT", "1")
with pytest.raises(NotImplementedError):
caps.check(query)
@pytest.mark.parametrize(
"kw",
[
{"dp": 2}, # dp 2 x cp 2 was not measured
{"fsdp2": True},
{"family": "llama"},
{"kernels": True, "loss": "fused_ce"}, # Unsloth's fast attention is not the generic path
{"memory": "unsloth_offload"},
],
)
def test_packed_rows_outside_the_measured_cell_are_not_supported(kw):
assert caps.cell_status(_packed(**kw))[0] != "supported"
def test_kept_refusals_flex_and_ring_with_packing_or_window():
with pytest.raises(NotImplementedError, match = "flex attention block masks"):
caps.check(_packed(attention = "flex"))
with pytest.raises(NotImplementedError, match = "ring context parallel with packed rows"):
caps.check(_packed(cp_backend = "ring", sm = 9))
with pytest.raises(NotImplementedError, match = "sliding window"):
caps.check(_packed(cp_backend = "ring", packing = False, windowed = True, sm = 9))
# Memory: Unsloth's checkpoint and tiling.
@pytest.mark.parametrize(
"fn_name, offloads, want",
[
("unsloth_checkpoint", None, "unsloth_offload"), # path A (UnslothCheckpointFunction)
("_nonre_checkpoint", True, "unsloth_offload"), # path B with offload hooks
# path B without offload: a plain non-reentrant checkpoint
("_nonre_checkpoint", False, "gc"),
("unsloth_offloaded_gradient_checkpoint", None, "unsloth_offload_legacy"), # not measured
("unsloth_gradient_checkpoint", None, "unsloth_gc"),
("checkpoint", None, "gc"),
],
)
def test_memory_reports_unsloth_checkpoints(monkeypatch, fn_name, offloads, want):
import torch.utils.checkpoint as checkpoint
def fn(function, *args, **kwargs):
return function(*args, **kwargs)
fn.__name__ = fn_name
if offloads is not None:
fn._unsloth_offloads = offloads
monkeypatch.setattr(checkpoint, "checkpoint", fn)
args = types.SimpleNamespace(gradient_checkpointing = True)
assert caps._memory(None, args) == want
assert caps._memory(None, types.SimpleNamespace(gradient_checkpointing = False)) == "none"
def test_memory_reports_the_zoo_unsloth_checkpoint(monkeypatch):
zoo = pytest.importorskip("unsloth_zoo.gradient_checkpointing")
import torch.utils.checkpoint as checkpoint
monkeypatch.setattr(checkpoint, "checkpoint", zoo.unsloth_checkpoint)
assert (
caps._memory(None, types.SimpleNamespace(gradient_checkpointing = True)) == "unsloth_offload"
)
def test_unsloth_checkpoint_is_seen_and_covered_by_the_cells_that_measured_it(monkeypatch):
zoo = pytest.importorskip("unsloth_zoo.gradient_checkpointing")
import torch.utils.checkpoint as checkpoint
monkeypatch.setattr(checkpoint, "checkpoint", zoo.unsloth_checkpoint)
memory = caps._memory(None, types.SimpleNamespace(gradient_checkpointing = True))
assert memory == "unsloth_offload"
for kw, name in (
(dict(family = "llama", tp = 2), "cp x tp (DDP dp), eager"),
(dict(family = "llama", tp = 2, dp = 2, fsdp2 = True), "cp x tp (FSDP2 dp), eager"),
(dict(family = "llama", tp = 2, compile = "block"), "cp x tp, Unsloth block compile"),
(dict(family = "llama", fsdp2 = True), "cp x native FSDP2 (dp_shard x cp), eager"),
(
dict(family = "qwen3_moe", moe = True, expert_lora = True),
"MoE qwen3_moe x lora+experts x aux off x cp",
),
(
dict(family = "qwen3_moe", moe = True, mode = "full", compile = "block"),
"MoE qwen3_moe x full x aux off x cp, block compile",
),
):
query = caps.Query(
trainer = "sft", cp = 2, cp_backend = "ulysses", memory = memory, torch = (2, 13), **kw
)
status, cell = caps.cell_status(query)
assert status == "supported" and cell.name == name, (kw, status, cell)
assert "unsloth_checkpointing" in cell.note, cell.note
caps.check(query)
@pytest.mark.parametrize(
"mode, expert_lora",
[("full", False), ("lora", False), ("lora", True)],
ids = ["full", "lora", "lora+experts"],
)
def test_kimi_linear_moe_under_cp_follows_its_gloo_evidence(mode, expert_lora):
# tests/test_ulysses_kimi.py::test_kimi_moe_under_ulysses_matches_one_process: sdpa / eager, none / HF
# checkpointing, cp 2 and dp 2 x cp 2. Unsloth's checkpoint, FSDP2 x cp and the router loss stay refused.
kw = dict(
trainer = "sft",
family = "kimi_linear",
moe = True,
mode = mode,
expert_lora = expert_lora,
cp = 2,
cp_backend = "ulysses",
torch = (2, 13),
)
for extra in ({}, {"memory": "gc"}, {"attention": "eager"}, {"dp": 2}):
query = caps.Query(**kw, **extra)
status, cell = caps.cell_status(query)
assert status == "supported" and "test_kimi_moe_under_ulysses" in cell.parity, (extra, cell)
caps.check(query)
for extra, match in (
({"memory": "unsloth_offload"}, "Unsloth checkpoint"),
({"fsdp2": True}, "fsdp2_cp"),
({"router_aux_loss": True}, "router auxiliary loss"),
({"compile": "block"}, "no numeric parity"),
):
with pytest.raises(NotImplementedError, match = match):
caps.check(caps.Query(**kw, **extra))
@pytest.mark.parametrize("family", ["minimax", "minimax_m2"])
@pytest.mark.parametrize(
"mode, expert_lora",
[("full", False), ("lora", False), ("lora", True)],
ids = ["full", "lora", "lora+experts"],
)
def test_minimax_moe_under_cp_follows_its_gloo_evidence(family, mode, expert_lora):
# tests/test_ulysses_minimax_parity.py::test_minimax_model_matches_upstream: sdpa / eager, none / HF
# checkpointing, cp 2 and dp 2 x cp 2. Unsloth's checkpoint, FSDP2 x cp and the router loss stay refused.
kw = dict(
trainer = "sft",
family = family,
moe = True,
mode = mode,
expert_lora = expert_lora,
cp = 2,
cp_backend = "ulysses",
torch = (2, 13),
)
for extra in ({}, {"memory": "gc"}, {"attention": "eager"}, {"dp": 2}):
query = caps.Query(**kw, **extra)
status, cell = caps.cell_status(query)
assert status == "supported" and "test_minimax_model_matches_upstream" in cell.parity, (
extra,
cell,
)
caps.check(query)
for extra, match in (
({"memory": "unsloth_offload"}, "Unsloth checkpoint"),
({"fsdp2": True}, "fsdp2_cp"),
({"router_aux_loss": True}, "router auxiliary loss"),
({"compile": "block"}, "no numeric parity"),
({"mode": "qlora"}, "4-bit experts"),
):
with pytest.raises(NotImplementedError, match = match):
caps.check(caps.Query(**{**kw, **extra}))
def test_minimax_evidence_ids_are_real_parity_cases():
# Every case a MiniMax cell names exists in the parity test, so a renamed case cannot leave a cell
# pointing at nothing.
import re
from tests import test_ulysses_minimax_parity as parity
for cell in caps.CELLS:
if cell.parity and "test_minimax_model_matches_upstream[" in cell.parity:
ids = re.search(r"\[(.*)\]", cell.parity).group(1).split("|")
assert set(ids) <= set(parity.MODEL_CASES), (cell.name, ids)
@pytest.mark.parametrize("family", ["deepseek_v2", "deepseek_v3", "deepseek_v32", "deepseek_v4"])
@pytest.mark.parametrize(
"mode, expert_lora",
[("full", False), ("lora", False), ("lora", True)],
ids = ["full", "lora", "lora+experts"],
)
def test_deepseek_moe_under_cp_follows_its_gloo_evidence(family, mode, expert_lora):
# tests/test_ulysses_deepseek_reference.py::test_deepseek_cp_matches_fp64_reference: cp 2 / 4, dp 2 x cp 2,
# HF checkpointing, sdpa and eager (V4: eager). Unsloth's checkpoint, FSDP2 x cp and the router loss stay refused.
kw = dict(
trainer = "sft",
family = family,
moe = True,
mode = mode,
expert_lora = expert_lora,
cp = 2,
cp_backend = "ulysses",
torch = (2, 13),
attention = "eager",
)
for extra in ({}, {"memory": "gc"}, {"attention": "sdpa"}, {"dp": 2}, {"cp": 4}):
if family == "deepseek_v4" and extra.get("attention") == "sdpa":
continue
query = caps.Query(**{**kw, **extra})
status, cell = caps.cell_status(query)
assert status == "supported" and "test_deepseek_cp_matches_fp64_reference" in cell.parity, (
extra,
cell,
)
caps.check(query)
refused = [
({"memory": "unsloth_offload"}, "Unsloth checkpoint"),
({"fsdp2": True}, "fsdp2_cp"),
({"router_aux_loss": True}, "router auxiliary loss"),
({"compile": "block"}, "no numeric parity"),
({"mode": "qlora"}, "4-bit experts"),
]
if family == "deepseek_v4":
refused.append(({"attention": "sdpa"}, "no numeric parity"))
for extra, match in refused:
with pytest.raises(NotImplementedError, match = match):
caps.check(caps.Query(**{**kw, **extra}))
def test_deepseek_evidence_ids_are_real_parity_cases():
import re
parity = pytest.importorskip("tests.test_ulysses_deepseek_reference")
named = 0
for cell in caps.CELLS:
if cell.parity and "test_deepseek_cp_matches_fp64_reference[" in cell.parity:
ids = re.search(r"\[(.*)\]", cell.parity).group(1).split("|")
assert set(ids) <= set(parity.CASES), (cell.name, ids)
named += 1
assert named >= 12
@pytest.mark.parametrize(
"mode, expert_lora",
[("full", False), ("lora", False), ("lora", True)],
ids = ["full", "lora", "lora+experts"],
)
def test_gemma4_moe_under_cp_follows_its_gloo_evidence(mode, expert_lora):
kw = dict(
trainer = "sft",
family = "gemma4",
moe = True,
windowed = True,
mode = mode,
expert_lora = expert_lora,
cp = 2,
cp_backend = "ulysses",
torch = (2, 13),
attention = "sdpa",
)
for extra in ({}, {"memory": "gc"}, {"attention": "eager"}, {"dp": 2}, {"cp": 4}):
query = caps.Query(**{**kw, **extra})
status, cell = caps.cell_status(query)
assert status == "supported" and "test_gemma4_cp_matches_fp64_reference" in cell.parity, (
extra,
cell,
)
caps.check(query)
for extra, match in (
({"memory": "unsloth_offload"}, "Unsloth checkpoint"),
({"fsdp2": True}, "fsdp2_cp"),
({"router_aux_loss": True}, "router auxiliary loss"),
({"compile": "block"}, "no numeric parity"),
({"mode": "qlora"}, "4-bit experts"),
({"multimodal": True}, "bidirectional"),
({"multimodal": True, "memory": "gc"}, "bidirectional"),
):
with pytest.raises(NotImplementedError, match = match):
caps.check(caps.Query(**{**kw, **extra}))
def test_gemma4_evidence_ids_are_real_parity_cases():
import re
parity = pytest.importorskip("tests.test_ulysses_gemma4")
named = 0
for cell in caps.CELLS:
if cell.parity and "test_gemma4_cp_matches_fp64_reference[" in cell.parity:
ids = re.search(r"\[(.*)\]", cell.parity).group(1).split("|")
assert set(ids) <= set(parity.CASES), (cell.name, ids)
assert all(parity.CASES[i]["kind"] == "moe" for i in ids), (cell.name, ids)
named += 1
assert named >= 3
@pytest.mark.parametrize(
"mode, expert_lora",
[("full", False), ("lora", False), ("lora", True)],
ids = ["full", "lora", "lora+experts"],
)
def test_gpt_oss_moe_under_cp_follows_its_gloo_evidence(mode, expert_lora):
kw = dict(
trainer = "sft",
family = "gpt_oss",
moe = True,
windowed = True,
direct_params = True,
mode = mode,
expert_lora = expert_lora,
cp = 2,
cp_backend = "ulysses",
torch = (2, 13),
attention = "eager",
)
for extra in ({}, {"memory": "gc"}, {"dp": 2}, {"cp": 4}):
query = caps.Query(**{**kw, **extra})
status, cell = caps.cell_status(query)
assert status == "supported" and "test_gpt_oss_cp_matches_fp64_reference" in cell.parity, (
extra,
cell,
)
caps.check(query)
for extra, match in (
({"memory": "unsloth_offload"}, "Unsloth checkpoint"),
({"fsdp2": True}, "fsdp2_cp"),
({"router_aux_loss": True}, "router auxiliary loss"),
({"compile": "block"}, "no numeric parity"),
({"attention": "sdpa"}, "no numeric parity"),
({"mode": "qlora"}, "4-bit experts"),
({"cp": 1, "tp": 2}, "sinks"),
):
with pytest.raises(NotImplementedError, match = match):
caps.check(caps.Query(**{**kw, **extra}))
def test_gpt_oss_evidence_ids_are_real_parity_cases():
import re
parity = pytest.importorskip("tests.test_ulysses_gpt_oss")
named = 0
for cell in caps.CELLS:
if cell.parity and "test_gpt_oss_cp_matches_fp64_reference[" in cell.parity:
ids = re.search(r"\[(.*)\]", cell.parity).group(1).split("|")
assert set(ids) <= set(parity.CASES), (cell.name, ids)
lora = {parity.CASES[i].get("lora") for i in ids}
assert len(lora) == 1, (cell.name, ids)
assert {"lora": "dense", "lora+experts": "experts"}.get(
cell.name.split(" x ")[1]
) in lora
named += 1
assert named == 3
def test_unsloth_checkpoint_keeps_an_unverified_moe_row_refused_under_strict(
clean_env, monkeypatch
):
zoo = pytest.importorskip("unsloth_zoo.gradient_checkpointing")
import torch.utils.checkpoint as checkpoint
monkeypatch.setattr(checkpoint, "checkpoint", zoo.unsloth_checkpoint)
memory = caps._memory(None, types.SimpleNamespace(gradient_checkpointing = True))
query = caps.Query(trainer = "sft", family = "qwen3_moe", moe = True, dp = 2, memory = memory)
assert caps.cell_status(query)[0] == "unverified"
clean_env.setenv("UNSLOTH_PARALLEL_STRICT", "1")
with pytest.raises(NotImplementedError, match = "MoE qwen3_moe x lora x aux off x ddp"):
caps.check(query)
def test_tiling_is_its_own_flag():
pytest.importorskip("unsloth_zoo.tiling_registry")
from unsloth.tiling import tile_mlp
model = _tiny()
args = types.SimpleNamespace(gradient_checkpointing = True)
assert not caps.query_from(None, None, model, args).tiled
tile_mlp(model)
query = caps.query_from(None, None, model, args)
assert query.tiled and query.memory != "tiled_mlp"
assert {"tiled_mlp", "gradient_checkpointing"} <= set(caps.check_features(query))
def test_every_feature_module_path_exists():
import os
import re
root = os.path.dirname(caps.__file__)
for feature in caps.FEATURES:
local = feature.module.split("unsloth_zoo")[0]
for path in re.findall(r"[\w/]+\.py", local):
assert os.path.exists(os.path.join(root, path)), (feature.name, path)
# Block swap.
def _swap(model, variant):
layers = model.model.layers
if variant == "count":
swapper = types.SimpleNamespace(indices = [1])
layers._unsloth_block_swap = swapper
model._unsloth_block_swap = swapper
elif variant == "empty": # a swapper that swapped nothing (not auto): nothing to refuse
layers._unsloth_block_swap = types.SimpleNamespace(indices = [])
elif variant == "auto":
model._unsloth_offload_layers_auto = "auto"
elif variant == "auto_ddp":
model._unsloth_offload_layers_auto_ddp = True
return model
@pytest.mark.parametrize("variant", ["none", "count", "empty", "auto", "auto_ddp"])
def test_block_swap_flag_agrees_with_the_fsdp2_shard_refusal(variant):
from unsloth.models import fsdp2_native
model = _swap(_tiny(), variant)
try:
fsdp2_native._refuse_block_swap(model)
refused = False
except ValueError:
refused = True
assert caps._block_swap(model) == refused == (variant in ("count", "auto", "auto_ddp"))
@pytest.mark.parametrize("variant", ["count", "auto"])
def test_block_swap_under_fsdp2_is_refused_by_check_layout_before_weights_move(variant):
model = _swap(_tiny(), variant)
before = {n: (p.data_ptr(), p.detach().clone()) for n, p in model.named_parameters()}
layout = ParallelLayout(tp = 2, dp = 2, world = 4, fsdp2 = True)
with pytest.raises(NotImplementedError, match = "block swap"):
caps.check_layout(layout, None, model, None)
for name, param in model.named_parameters():
assert param.data_ptr() == before[name][0] and torch.equal(param.detach(), before[name][1])
assert "block_swap" in caps.markdown_table("features")
assert "block swap (offload_layers) under native FSDP2" in caps.markdown_table("rules")
def test_block_swap_without_fsdp2_passes_the_rule():
query = caps.Query(dp = 2, block_swap = True)
assert "block_swap" in caps.check_features(query)
caps.check(query)
@pytest.mark.parametrize("size", [2, 3, 16])
@pytest.mark.parametrize("axis", ["tp", "pp"])
def test_block_swap_under_tp_or_pp_is_unverified(clean_env, axis, size):
query = caps.Query(trainer = "sft", block_swap = True, **{axis: size})
status, detail = caps.cell_status(query)
assert status == "unverified" and f"block swap x {axis}" in detail
caps.check(query) # a single axis: runs outside strict mode
clean_env.setenv("UNSLOTH_PARALLEL_STRICT", "1")
with pytest.raises(NotImplementedError, match = "block swap check"):
caps.check(query)
@pytest.mark.parametrize(
"layout",
[
dict(family = "llama", cp = 2, tp = 2, cp_backend = "ulysses"),
dict(family = "qwen3_moe", moe = True, expert_lora = True, pp = 2, pp_schedule = "gpipe"),
],
)
def test_block_swap_refusal_on_a_gated_layout_names_offload_layers(clean_env, layout):
query = caps.Query(trainer = "sft", torch = (2, 13), **layout)
assert caps.cell_status(query)[0] == "supported"
query = dataclasses.replace(query, block_swap = True)
with pytest.raises(NotImplementedError) as e:
caps.check(query)
assert "block swap x" in str(e.value) and "offload_layers = 0" in str(e.value)
# MoE qwen3_moe x qlora x pp: measured with one micro-batch only, which the registry cannot see.
def test_qwen3_moe_qlora_pp_keeps_its_refusal_with_the_micro_batch_evidence():
query = caps.Query(
trainer = "sft",
family = "qwen3_moe",
moe = True,
router_aux_loss = True,
mode = "qlora",
pp = 2,
pp_schedule = "gpipe",
torch = (2, 13),
)
with pytest.raises(NotImplementedError, match = "UNSLOTH_PP_MICROBATCHES=1") as e:
caps.check(query)
assert "qwen3_moe x qlora x aux on x pp" in str(e.value)