Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Add unsloth.precision: one registry for low-precision recipes, refusals and high-precision layers (#16387)

Daniel Han0bfae0173801
diff --git a/tests/test_precision_registry.py b/tests/test_precision_registry.py
new file mode 100644
index 00000000000..31dca5933a0
--- /dev/null
+++ b/tests/test_precision_registry.py
@@ -0,0 +1,773 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""unsloth.precision: recipe table, refusals before model mutation, structural high-precision layers,
+engagement, and the existing float8_training / qat_scheme paths wired through it (CPU only)."""
+
+from __future__ import annotations
+
+import ast
+import importlib.util
+import inspect
+import warnings
+
+import types
+
+import pytest
+import torch
+
+from unsloth import precision as P
+from unsloth.models import float8_training as f8t
+
+needs_torchao = pytest.mark.skipif(
+ importlib.util.find_spec("torchao") is None, reason = "require torchao"
+)
+
+
+def _tiny_qwen3(
+ layers = 2,
+ tie = False,
+ dtype = torch.float32,
+):
+ from transformers import Qwen3Config, Qwen3ForCausalLM
+
+ torch.manual_seed(0)
+ config = Qwen3Config(
+ vocab_size = 256,
+ hidden_size = 128,
+ intermediate_size = 256,
+ num_hidden_layers = layers,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ head_dim = 32,
+ tie_word_embeddings = tie,
+ )
+ return Qwen3ForCausalLM(config).to(dtype)
+
+
+def _tiny_qwen3_moe():
+ from transformers import Qwen3MoeConfig, Qwen3MoeForCausalLM
+
+ torch.manual_seed(0)
+ config = Qwen3MoeConfig(
+ vocab_size = 256,
+ hidden_size = 128,
+ intermediate_size = 256,
+ moe_intermediate_size = 64,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ head_dim = 32,
+ num_experts = 8,
+ num_experts_per_tok = 2,
+ decoder_sparse_step = 1,
+ mlp_only_layers = [],
+ )
+ return Qwen3MoeForCausalLM(config)
+
+
+class _Expert(torch.nn.Module):
+ def __init__(self, h, i):
+ super().__init__()
+ self.up = torch.nn.Linear(h, i, bias = False)
+ self.down = torch.nn.Linear(i, h, bias = False)
+
+ def forward(self, x):
+ return self.down(torch.relu(self.up(x)))
+
+
+class _RemoteMoE(torch.nn.Module):
+ """Remote-code style MoE block: the router is called `wg`, which no name rule matches."""
+
+ def __init__(
+ self,
+ h = 2048,
+ e = 64,
+ ):
+ super().__init__()
+ self.wg = torch.nn.Linear(h, e, bias = False)
+ self.experts = torch.nn.ModuleList([_Expert(h, 32) for _ in range(e)])
+ self.shared_experts = _Expert(h, 32)
+
+
+class _RemoteModel(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.layers = torch.nn.ModuleList([_RemoteMoE() for _ in range(2)])
+ self.lm_head = torch.nn.Linear(2048, 64, bias = False)
+
+ def get_output_embeddings(self):
+ return self.lm_head
+
+
+# ---- the table ----------------------------------------------------------------------------------------
+
+
+def test_float8_names_and_gates_unchanged():
+ # The literals main had before the registry; byte-identical behaviour for the shipped recipe.
+ assert f8t.FLOAT8_TRAINING_RECIPES == ("tensorwise",)
+ assert f8t._MEASURED_FASTER == {"tensorwise": ((12, 0), (12, 1))}
+
+
+def test_every_recipe_is_well_formed():
+ gpu_classes = {"sm75", "sm80", "sm89", "sm90", "sm100", "sm120", "rocm"}
+ for name, r in P.RECIPES.items():
+ assert r.kind in ("dtype", "float8_training", "qat", "frozen_base"), name
+ assert r.status in ("shipped", "planned", "refused"), name
+ assert set(r.gates) <= gpu_classes, name
+ for g in r.gates.values():
+ assert g.status in ("named", "explicit", "refused", "unprobed"), name
+ assert g.status != "refused" or g.reason, name
+ assert r.status != "refused" or r.refusal, name
+ assert set(r.input_dtypes) <= {"bf16", "fp16", "fp32"}, name
+ assert r.grad_scaler in ("none", "dynamic", "per_dtype"), name
+ if r.alias_of:
+ assert r.alias_of in P.RECIPES, name
+ for short, full in P.FLOAT8_TRAINING_NAMES.items():
+ assert P.RECIPES[full].kind == "float8_training", short
+
+
+def test_paper_recipes_encoded():
+ mx = P.get_recipe("mxfp8")
+ for gemm in (
+ mx.fwd,
+ mx.dgrad,
+ mx.wgrad,
+ ): # arXiv 2506.08027: e4m3 everywhere, 1x32, e8m0, round up
+ for op in (gemm.a, gemm.b):
+ assert (op.dtype, op.granularity, op.scale_dtype, op.rounding) == (
+ "e4m3",
+ "block1x32",
+ "e8m0",
+ "rceil",
+ )
+ nv = P.get_recipe("nvfp4") # arXiv 2509.25149
+ assert nv.fwd.b.granularity == "block16x16" and nv.dgrad.b.granularity == "block16x16"
+ assert nv.fwd.a.rounding == "rn" and nv.dgrad.a.rounding == "sr" and nv.wgrad.a.rounding == "sr"
+ assert nv.wgrad.a.rht and nv.wgrad.b.rht and not nv.fwd.a.rht and not nv.dgrad.a.rht
+ assert all(op.scale_dtype == "e4m3" for g in (nv.fwd, nv.dgrad, nv.wgrad) for op in (g.a, g.b))
+ assert nv.hp_layers.last_frac == pytest.approx(0.15)
+ tw = P.get_recipe("fp8_tensorwise")
+ assert tw.fwd.a.dtype == "e4m3" and tw.dgrad.a.dtype == "e5m2"
+ assert P.get_recipe("fp8_rowwise_gw_hp").wgrad.low_precision is False
+
+
+def test_qat_schemes_match_prepare_model_for_qat():
+ # Every string scheme _prepare_model_for_qat branches on has a registry row, and no row is stale.
+ from unsloth.models import _utils
+
+ tree = ast.parse(inspect.getsource(_utils._prepare_model_for_qat))
+ names = set()
+ for node in ast.walk(tree):
+ if (
+ isinstance(node, ast.Compare)
+ and isinstance(node.left, ast.Name)
+ and node.left.id == "qat_scheme"
+ ):
+ for c in node.comparators:
+ if isinstance(c, ast.Constant) and isinstance(c.value, str):
+ names.add(c.value)
+ assert names, "no scheme comparisons found"
+ rows = {n[len("qat:") :] for n in P.recipes_of_kind("qat")}
+ assert names == rows
+
+
+# ---- refusals -----------------------------------------------------------------------------------------
+
+
+def test_resolve_shipped_values_unchanged():
+ assert f8t.resolve_float8_training(None) is None
+ assert f8t.resolve_float8_training(False) is None
+ assert f8t.resolve_float8_training(True) == "tensorwise"
+ assert f8t.resolve_float8_training("TensorWise") == "tensorwise"
+ assert f8t.resolve_float8_training(True, dtype = torch.float16) == "tensorwise"
+ with pytest.raises(ValueError, match = "slower than bfloat16"):
+ f8t.resolve_float8_training("rowwise")
+ with pytest.raises(ValueError, match = "is not supported. Use True or one of"):
+ f8t.resolve_float8_training("int3")
+
+
+def test_refuse_mxfp6_training():
+ with pytest.raises(P.PrecisionRefused, match = "MXFP6 has no training kernel"):
+ f8t.resolve_float8_training("mxfp6")
+
+
[email protected]("name", ["mxfp8", "nvfp4", "blockwise"])
+def test_refuse_fp16_for_formats_whose_quantizer_rejects_it(name):
+ with pytest.raises(P.PrecisionRefused, match = "float16|fp16"):
+ f8t.resolve_float8_training(name, dtype = torch.float16)
+
+
+def test_refuse_blockwise_off_sm90(monkeypatch):
+ monkeypatch.setattr(f8t._precision, "current_gpu_class", lambda device = None: "sm120")
+ with pytest.raises(P.PrecisionRefused, match = "SM90 only"):
+ f8t.resolve_float8_training("blockwise")
+
+
[email protected]("name", ["mxfp8", "nvfp4"])
+def test_planned_formats_refuse_with_their_gates(name):
+ with pytest.raises(P.PrecisionRefused, match = "not available in Unsloth yet"):
+ f8t.resolve_float8_training(name, dtype = torch.bfloat16)
+
+
+def test_gw_hp_refused_by_name_with_the_config_hint():
+ with pytest.raises(ValueError, match = "from_recipe_name\\('rowwise_with_gw_hp'\\)"):
+ f8t.resolve_float8_training("rowwise_with_gw_hp")
+
+
+def test_refusal_happens_before_any_load():
+ # The loader resolves float8_training before it looks the (nonexistent) repo up or builds anything.
+ from unsloth.models import loader
+ with pytest.raises(P.PrecisionRefused, match = "MXFP6"):
+ loader.FastModel.from_pretrained(
+ model_name = "unsloth/does-not-exist", full_finetuning = True, float8_training = "mxfp6"
+ )
+ with pytest.raises(P.PrecisionRefused, match = "float16"):
+ loader.FastModel.from_pretrained(
+ model_name = "unsloth/does-not-exist",
+ full_finetuning = True,
+ float8_training = "mxfp8",
+ dtype = torch.float16,
+ )
+
+
+def test_gate_evidence_names_no_local_paths():
+ # Evidence is printed in refusals; a scratch path (outputs/, plans/) means nothing to a user.
+ for name, r in P.RECIPES.items():
+ for gpu, gate in r.gates.items():
+ assert not any(p in gate.evidence for p in ("outputs/", "plans/", ".md")), (
+ name,
+ gpu,
+ gate.evidence,
+ )
+
+
+def test_check_recipe_gate_table():
+ with pytest.raises(P.PrecisionRefused, match = "fp8 tensor cores"):
+ P.check_recipe("fp8_tensorwise", gpu = "sm75")
+ with pytest.raises(P.PrecisionRefused, match = "NaN"):
+ P.check_recipe("fp8_tensorwise", gpu = "sm89")
+ assert P.check_recipe("fp8_tensorwise", gpu = "sm120").status == "named"
+ assert P.check_recipe("fp8_rowwise", gpu = "sm89").status == "explicit"
+ with pytest.raises(P.PrecisionRefused, match = "block-scaled"):
+ P.check_recipe("mxfp8", gpu = "sm89", allow_planned = True)
+ with pytest.raises(P.PrecisionRefused, match = "no bfloat16"):
+ P.check_recipe("bf16", gpu = "sm75")
+ assert P.check_recipe("fp16", gpu = "sm75").status == "named"
+ with pytest.raises(P.PrecisionRefused, match = "training modes"):
+ P.check_recipe("fp8_tensorwise", mode = "lora")
+
+
+def test_gate_table_agrees_with_float8_hardware_check():
+ # float8_training._check_hardware refuses tensorwise below sm90 and everything below 8.9.
+ for gpu in ("sm75", "sm80", "sm89"):
+ assert P.get_recipe("fp8_tensorwise").gate(gpu).status == "refused"
+ for gpu in ("sm75", "sm80"):
+ assert P.get_recipe("fp8_rowwise").gate(gpu).status == "refused"
+
+
+def test_gpu_class_and_scaler_kind():
+ assert (
+ P.gpu_class((7, 5)) == "sm75"
+ and P.gpu_class((8, 6)) == "sm80"
+ and P.gpu_class((8, 9)) == "sm89"
+ )
+ assert (
+ P.gpu_class((9, 0)) == "sm90"
+ and P.gpu_class((10, 0)) == "sm100"
+ and P.gpu_class((12, 0)) == "sm120"
+ )
+ assert P.gpu_class(None) is None and P.gpu_class((9, 0), hip = True) == "rocm"
+ assert P.grad_scaler_kind("fp16", torch.float16) == "GradScaler"
+ assert P.grad_scaler_kind("fp16", torch.float16, fsdp2 = True) == "ShardedGradScaler"
+ assert P.grad_scaler_kind("bf16", torch.bfloat16) == "none"
+ assert P.grad_scaler_kind("fp8_tensorwise", torch.float16) == "GradScaler"
+ assert P.grad_scaler_kind("fp8_tensorwise", torch.bfloat16) == "none"
+ assert P.grad_scaler_kind("mxfp8", "bfloat16") == "none"
+
+
+# ---- structural high-precision layers ------------------------------------------------------------------
+
+
+def test_routers_found_by_structure():
+ moe = _tiny_qwen3_moe()
+ assert set(P.find_routers(moe)) == {"model.layers.0.mlp.gate", "model.layers.1.mlp.gate"}
+ remote = _RemoteModel()
+ assert set(P.find_routers(remote)) == {"layers.0.wg", "layers.1.wg"}
+ assert P.find_routers(_tiny_qwen3()) == {}
+
+
+def _tiny_moe_expert_width_collision():
+ # Qwen3-Next-80B-A3B has num_experts == moe_intermediate_size == shared_expert_intermediate_size == 512:
+ # an expert / shared expert / dense MLP gate_proj is as wide as the router but is not one.
+ from transformers import Qwen2MoeConfig, Qwen2MoeForCausalLM
+
+ torch.manual_seed(0)
+ config = Qwen2MoeConfig(
+ vocab_size = 256,
+ hidden_size = 128,
+ intermediate_size = 16,
+ moe_intermediate_size = 16,
+ shared_expert_intermediate_size = 16,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ num_experts = 16,
+ num_experts_per_tok = 2,
+ decoder_sparse_step = 1,
+ mlp_only_layers = [0],
+ )
+ return Qwen2MoeForCausalLM(config)
+
+
+def test_routers_ignore_mlps_as_wide_as_the_expert_count():
+ m = _tiny_moe_expert_width_collision()
+ assert set(P.find_routers(m)) == {"model.layers.1.mlp.gate"}
+
+
+def test_hp_layers_policy():
+ m = _tiny_qwen3(layers = 8, tie = True)
+ keep = P.high_precision_layers(m, P.get_recipe("nvfp4").hp_layers)
+ assert "lm_head" in keep and "model.embed_tokens" in keep
+ assert {k for k in keep if k.startswith("model.layers.")} == {
+ "model.layers.6",
+ "model.layers.7",
+ } # ceil(0.15 * 8)
+ is_hp = P.hp_filter(m, P.HPLayerPolicy(first_n = 1, last_n = 1))
+ assert is_hp(None, "model.layers.0.mlp.up_proj") and is_hp(
+ None, "model.layers.7.self_attn.q_proj"
+ )
+ assert not is_hp(None, "model.layers.3.mlp.up_proj")
+ assert P.decoder_layers(m) == ("model.layers", 8)
+
+
+def test_num_experts_nested_config():
+ assert P.num_experts_of({"text_config": {"num_local_experts": 16}}) == 16
+ assert P.num_experts_of({"hidden_size": 8}) is None
+
+
+@needs_torchao
+def test_float8_filter_skips_structural_router():
+ # Failing on main: the name rules (gate / router / score / classifier) convert a router called `wg`.
+ remote = _RemoteModel()
+ fn = f8t.float8_training_filter(remote, "tensorwise", size_filter = False)
+ assert not fn(remote.layers[0].wg, "layers.0.wg")
+ assert fn(remote.layers[0].experts[0].up, "layers.0.experts.0.up")
+ assert not fn(remote.lm_head, "lm_head")
+
+
+@needs_torchao
[email protected]("build", [_tiny_qwen3, _tiny_qwen3_moe, _tiny_moe_expert_width_collision])
+def test_float8_filter_unchanged_on_standard_models(build, monkeypatch):
+ m = build()
+ for size_filter in (True, False):
+ new = f8t.float8_training_filter(m, "tensorwise", size_filter = size_filter)
+ new_set = {n for n, mod in m.named_modules() if new(mod, n)}
+ with monkeypatch.context() as mp:
+ mp.setattr(f8t, "_find_routers", lambda model: {})
+ old = f8t.float8_training_filter(m, "tensorwise", size_filter = size_filter)
+ old_set = {n for n, mod in m.named_modules() if old(mod, n)}
+ assert new_set == old_set
+
+
+# ---- engagement ------------------------------------------------------------------------------------------
+
+
+def _emulated_config(recipe = "tensorwise"):
+ from torchao.float8 import Float8LinearConfig
+
+ config = Float8LinearConfig.from_recipe_name(recipe)
+ object.__setattr__(config, "emulate", True)
+ return config
+
+
+@needs_torchao
+def test_engagement_recorded_and_strict(monkeypatch):
+ m = _tiny_qwen3()
+ converted = f8t.convert_model_to_float8_training(m, _emulated_config(), size_filter = False)
+ assert converted and P.engaged(m)["float8_training:Float8LinearConfig(tensorwise)"][
+ "modules"
+ ] == len(converted)
+
+ tiny = _tiny_qwen3() # 128 / 256 wide: the size filter keeps every layer out
+ monkeypatch.delenv("UNSLOTH_STRICT_ENGAGE", raising = False)
+ with warnings.catch_warnings(record = True) as w:
+ warnings.simplefilter("always")
+ assert f8t.convert_model_to_float8_training(tiny, _emulated_config()) == []
+ assert any("no eligible Linear" in str(x.message) for x in w)
+ assert P.engaged(tiny)["float8_training:Float8LinearConfig(tensorwise)"]["modules"] == 0
+ monkeypatch.setenv("UNSLOTH_STRICT_ENGAGE", "1")
+ with pytest.raises(P.EngagementError, match = "engaged on 0 modules"):
+ f8t.convert_model_to_float8_training(_tiny_qwen3(), _emulated_config())
+
+
+@needs_torchao
+def test_qat_engagement_recorded():
+ from unsloth.models._utils import _prepare_model_for_qat
+
+ m = _prepare_model_for_qat(_tiny_qwen3(dtype = torch.bfloat16), "int4")
+ rec = P.engaged(m)["qat:int4"]
+ assert rec["modules"] == sum(1 for x in m.modules() if "FakeQuantized" in type(x).__name__) > 0
+
+
+def test_require_and_counter(monkeypatch):
+ monkeypatch.delenv("UNSLOTH_STRICT_ENGAGE", raising = False)
+ assert P.require("x", 3) and not P.require("x", 0)
+ monkeypatch.setenv("UNSLOTH_STRICT_ENGAGE", "1")
+ with pytest.raises(P.EngagementError):
+ P.require("x", 0)
+ with P.count_lowp_gemms() as c:
+ torch.ones(4, 4) @ torch.ones(4, 4)
+ assert all(v == 0 for v in c.values())
+ a = torch.ones(16, 16).to(torch.float8_e4m3fn)
+ b = torch.ones(16, 16).to(torch.float8_e4m3fn).t()
+ one = torch.ones((), dtype = torch.float32)
+ try:
+ with P.count_lowp_gemms() as c:
+ torch._scaled_mm(a, b, scale_a = one, scale_b = one, out_dtype = torch.bfloat16)
+ except (RuntimeError, NotImplementedError):
+ pytest.skip("no CPU _scaled_mm in this torch")
+ assert c["_scaled_mm"] == 1
+
+
+def test_counter_runs_fullgraph_compiled_calls():
+ # Failing before: a plain TorchDispatchMode makes Dynamo refuse fullgraph frames ("non-infra torch dispatch
+ # mode present"), so the engagement check crashed around compiled models.
+ a = torch.ones(16, 16).to(torch.float8_e4m3fn)
+ b = torch.ones(16, 16).to(torch.float8_e4m3fn).t()
+ one = torch.ones((), dtype = torch.float32)
+
+ def f(a, b):
+ return torch._scaled_mm(a, b, scale_a = one, scale_b = one, out_dtype = torch.bfloat16)
+
+ try:
+ expected = f(a, b)
+ except (RuntimeError, NotImplementedError):
+ pytest.skip("no CPU _scaled_mm in this torch")
+ torch._dynamo.reset()
+ compiled = torch.compile(f, backend = "aot_eager", fullgraph = True)
+ compiled(a, b)
+ with P.count_lowp_gemms() as c:
+ out = compiled(a, b)
+ assert torch.equal(out, expected)
+ assert c["_scaled_mm"] == 1
+
+
+def test_counter_warns_when_torch_cannot_keep_compiled_calls(monkeypatch):
+ # Failing before: torch < 2.9 has no ignore_compile_internals hook, so Dynamo ran compiled callables
+ # eagerly inside the counter without saying so.
+ from torch.utils._python_dispatch import TorchDispatchMode
+ from unsloth.precision import engage
+
+ monkeypatch.delattr(TorchDispatchMode, "ignore_compile_internals", raising = False)
+ monkeypatch.setattr(engage, "_EAGER_UNDER_COUNTER_WARNED", False)
+ with pytest.warns(UserWarning, match = "eagerly"):
+ with P.count_lowp_gemms() as c:
+ torch.ones(4, 4) @ torch.ones(4, 4)
+ assert all(v == 0 for v in c.values())
+
+
+def test_decoder_layers_is_the_block_stack_not_an_experts_list():
+ # Failing before: the longest ModuleList won, so a model with more experts than layers returned
+ # model.layers.0.mlp.experts and first_n / last_n kept experts in high precision instead of blocks.
+ from transformers import Qwen2MoeConfig, Qwen2MoeForCausalLM
+
+ config = Qwen2MoeConfig(
+ vocab_size = 256,
+ hidden_size = 64,
+ intermediate_size = 128,
+ moe_intermediate_size = 32,
+ shared_expert_intermediate_size = 64,
+ num_hidden_layers = 4,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ num_experts = 12,
+ num_experts_per_tok = 2,
+ )
+ with torch.device("meta"):
+ m = Qwen2MoeForCausalLM(config)
+ assert P.decoder_layers(m) == ("model.layers", 4)
+ is_hp = P.hp_filter(m, P.HPLayerPolicy(first_n = 1, last_n = 1))
+ assert is_hp(None, "model.layers.0.self_attn.q_proj")
+ assert is_hp(None, "model.layers.3.mlp.experts.5.up_proj")
+ assert not is_hp(None, "model.layers.1.self_attn.q_proj")
+ assert not is_hp(None, "model.layers.1.mlp.experts.0.up_proj")
+
+
+def test_decoder_layers_is_the_language_decoder_not_a_longer_encoder():
+ # Failing before: the longest outer list won anywhere, so a vision tower or speech encoder deeper than
+ # the language decoder got the first / last block protection.
+ from transformers import (
+ CLIPVisionConfig,
+ LlamaConfig,
+ LlavaConfig,
+ LlavaForConditionalGeneration,
+ WhisperConfig,
+ WhisperForConditionalGeneration,
+ )
+
+ text = LlamaConfig(
+ vocab_size = 256,
+ hidden_size = 64,
+ intermediate_size = 128,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ )
+ vision = CLIPVisionConfig(
+ hidden_size = 32,
+ intermediate_size = 64,
+ num_hidden_layers = 4,
+ num_attention_heads = 4,
+ image_size = 32,
+ patch_size = 16,
+ )
+ with torch.device("meta"):
+ llava = LlavaForConditionalGeneration(LlavaConfig(vision_config = vision, text_config = text))
+ name, n = P.decoder_layers(llava)
+ assert n == 2 and "vision" not in name and name.endswith("layers"), (name, n)
+ is_hp = P.hp_filter(llava, P.HPLayerPolicy(first_n = 1, last_n = 1))
+ assert is_hp(None, f"{name}.0.self_attn.q_proj") and is_hp(None, f"{name}.1.mlp.down_proj")
+
+ whisper = WhisperConfig(
+ vocab_size = 256,
+ d_model = 32,
+ encoder_layers = 4,
+ decoder_layers = 2,
+ encoder_attention_heads = 4,
+ decoder_attention_heads = 4,
+ encoder_ffn_dim = 64,
+ decoder_ffn_dim = 64,
+ pad_token_id = 0,
+ bos_token_id = 1,
+ eos_token_id = 2,
+ decoder_start_token_id = 1,
+ )
+ with torch.device("meta"):
+ w = WhisperForConditionalGeneration(whisper)
+ assert P.decoder_layers(w) == ("model.decoder.layers", 2)
+
+
+def test_decoder_layers_accepts_a_hybrid_stack_of_mixed_block_classes():
+ # Failing before: a stack had to be one block class, so Jamba (Mamba + attention blocks) returned
+ # (None, 0) and first_n / last_n protected nothing.
+ from transformers import JambaConfig, JambaForCausalLM
+
+ config = JambaConfig(
+ vocab_size = 256,
+ hidden_size = 64,
+ intermediate_size = 128,
+ num_hidden_layers = 4,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ num_experts = 1,
+ attn_layer_period = 2,
+ attn_layer_offset = 1,
+ mamba_d_state = 8,
+ mamba_d_conv = 4,
+ mamba_expand = 2,
+ use_mamba_kernels = False,
+ )
+ with torch.device("meta"):
+ m = JambaForCausalLM(config)
+ assert len({type(b) for b in m.model.layers}) == 2
+ assert P.decoder_layers(m) == ("model.layers", 4)
+ is_hp = P.hp_filter(m, P.HPLayerPolicy(first_n = 1, last_n = 1))
+ assert is_hp(None, "model.layers.0.mamba.in_proj") and is_hp(
+ None, "model.layers.3.self_attn.q_proj"
+ )
+ assert not is_hp(None, "model.layers.1.self_attn.q_proj")
+
+
+def test_counter_counts_opaque_custom_ops_once_at_their_boundary(monkeypatch):
+ # Failing before: only torch.ops.aten targets were resolved, so a low-precision GEMM inside an opaque
+ # custom op (unsloth::nvfp4_qat_linear, fbgemm::f8f8bf16_rowwise) counted zero.
+ from unsloth.precision import engage
+
+ a = torch.ones(16, 16).to(torch.float8_e4m3fn)
+ b = torch.ones(16, 16).to(torch.float8_e4m3fn).t()
+ one = torch.ones((), dtype = torch.float32)
+ try:
+ torch._scaled_mm(a, b, scale_a = one, scale_b = one, out_dtype = torch.bfloat16)
+ except (RuntimeError, NotImplementedError):
+ pytest.skip("no CPU _scaled_mm in this torch")
+
+ name = "unsloth_precision_test::opaque_fp8_mm"
+ if not hasattr(torch.ops.unsloth_precision_test, "opaque_fp8_mm"):
+
+ @torch.library.custom_op(name, mutates_args = ())
+ def _opaque(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
+ one = torch.ones((), dtype = torch.float32)
+ return torch._scaled_mm(a, b, scale_a = one, scale_b = one, out_dtype = torch.bfloat16)
+
+ monkeypatch.setattr(engage, "LOWP_GEMM_CUSTOM_OPS", engage.LOWP_GEMM_CUSTOM_OPS + (name,))
+ with P.count_lowp_gemms() as c:
+ torch.ops.unsloth_precision_test.opaque_fp8_mm(a, b)
+ assert c[name] == 1 and c["_scaled_mm"] == 0 # the boundary, not the hidden inner call
+ assert "fbgemm::f8f8bf16_rowwise" in c # absent namespace: listed, never resolved
+
+
+@needs_torchao
+def test_counter_counts_the_real_nvfp4_qat_linear_op():
+ from unsloth.models._utils import _nvfp4_qat_linear_op
+
+ try:
+ op = _nvfp4_qat_linear_op()
+ except Exception as e:
+ pytest.skip(f"torchao NVFP4 internals unavailable: {e!r}")
+ x = torch.empty(16, 64, device = "meta", dtype = torch.bfloat16)
+ w = torch.empty(32, 64, device = "meta", dtype = torch.bfloat16)
+ with P.count_lowp_gemms() as c:
+ op(x, w, None, False, False, False, False, None)
+ assert c["unsloth::nvfp4_qat_linear"] == 1
+
+
+def test_counter_hears_eager_fp8_triton_launchers_and_not_the_fp16_gemv():
+ # Failing before: eager fp8 block launchers run Triton without dispatching their op (counted 0), and
+ # fp8_block_gemv, which upcasts the weight before its dot, was counted as a low-precision GEMM.
+ from unsloth.precision import engage
+
+ assert "unsloth::fp8_block_gemv" not in engage.LOWP_GEMM_CUSTOM_OPS
+ with P.count_lowp_gemms() as c:
+ engage.release_lowp_gemm(engage.note_lowp_gemm("unsloth::fp8_block_small_m_linear"))
+ assert c["unsloth::fp8_block_small_m_linear"] == 1
+ engage.note_lowp_gemm("unsloth::fp8_block_small_m_linear") # no counter open: a no-op
+ assert c["unsloth::fp8_block_small_m_linear"] == 1 and not engage._active()
+
+ fp8 = pytest.importorskip("unsloth.kernels.fp8")
+ names = fp8._EAGER_COUNTED_LOWP_GEMMS
+ assert "fp8_block_gemv" not in names
+ for name in ("fp8_block_matmul_triton", "fp8_block_small_m_linear", "fp8_block_matmul_torchao"):
+ assert name in names and f"unsloth::{name}" in engage.LOWP_GEMM_CUSTOM_OPS
+
+
+def test_mxfp8_qat_registry_admits_only_bfloat16_like_its_backend():
+ # Failing before: the qat:mxfp8 row allowed fp32 while _prepare_model_for_qat quantizes bf16 layers only.
+ assert P.check_recipe("qat:mxfp8", gpu = "sm100", input_dtype = torch.bfloat16, mode = "full")
+ with pytest.raises(ValueError):
+ P.check_recipe("qat:mxfp8", gpu = "sm100", input_dtype = torch.float32, mode = "full")
+
+
+def test_nested_counters_close_by_identity_and_delegated_gemms_count_once():
+ # Failing before: list.remove matched the outer (equal, all-zero) dict when the inner scope closed, so
+ # the outer cleanup raised; and a launcher that delegates to another counted launcher counted twice.
+ from unsloth.precision import engage
+
+ with P.count_lowp_gemms() as outer:
+ with P.count_lowp_gemms() as inner:
+ pass
+ assert any(a is outer for a in engage._active()) and not any(
+ a is inner for a in engage._active()
+ )
+ entered = engage.note_lowp_gemm("unsloth::fp8_block_matmul_torchao")
+ try: # the delegated Triton launch inside the torchao one
+ engage.release_lowp_gemm(engage.note_lowp_gemm("unsloth::fp8_block_matmul_triton"))
+ finally:
+ engage.release_lowp_gemm(entered)
+ assert not engage._active()
+ assert outer["unsloth::fp8_block_matmul_torchao"] == 1
+ assert outer.get("unsloth::fp8_block_matmul_triton", 0) == 0
+ assert inner.get("unsloth::fp8_block_matmul_torchao", 0) == 0
+
+
+def test_float8_filter_keeps_a_wrapped_router_and_its_inner_linear_in_high_precision():
+ # Failing before: find_routers returns the gating wrapper `wg`, and only that exact fqn was excluded, so
+ # its inner `wg.proj` Linear was converted to fp8 and could flip expert choices.
+ class Gate(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(64, 16, bias = False)
+
+ def forward(self, x):
+ return self.proj(x)
+
+ class Moe(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.wg = Gate()
+ self.experts = torch.nn.ModuleList(
+ torch.nn.Linear(64, 64, bias = False) for _ in range(16)
+ )
+
+ class Block(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.moe = Moe()
+
+ class Model(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.config = types.SimpleNamespace(num_experts = 16, num_local_experts = 16)
+ self.layers = torch.nn.ModuleList([Block(), Block()])
+
+ m = Model()
+ routers = P.find_routers(m)
+ assert any(r.endswith("moe.wg") for r in routers), routers
+ fn = f8t.float8_training_filter(m, "tensorwise", size_filter = False)
+ assert not fn(m.layers[0].moe.wg.proj, "layers.0.moe.wg.proj")
+ assert fn(m.layers[0].moe.experts[0], "layers.0.moe.experts.0")
+
+
+def test_float8_filter_keeps_a_router_beside_moduledict_experts_in_high_precision():
+ # Failing before: only a ModuleList counted as an experts collection, so with 16 experts in a ModuleDict
+ # (SwitchTransformers' layout) the wrapped router `wg` was not found and `wg.proj` was converted to fp8.
+ class Gate(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(64, 16, bias = False)
+
+ def forward(self, x):
+ return self.proj(x)
+
+ class Moe(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.wg = Gate()
+ self.experts = torch.nn.ModuleDict(
+ {f"expert_{i}": torch.nn.Linear(64, 64, bias = False) for i in range(16)}
+ )
+
+ class Model(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.config = types.SimpleNamespace(num_experts = 16)
+ self.layers = torch.nn.ModuleList([Moe(), Moe()])
+
+ m = Model()
+ assert set(P.find_routers(m)) == {"layers.0.wg", "layers.1.wg"}
+ fn = f8t.float8_training_filter(m, "tensorwise", size_filter = False)
+ assert not fn(m.layers[0].wg.proj, "layers.0.wg.proj")
+ assert fn(m.layers[0].experts.expert_0, "layers.0.experts.expert_0")
+
+
+def test_gemm_counters_are_per_thread():
+ # Failing before: the active-counter list was process-global, so an eager launch on one thread was
+ # counted by a counter open on another.
+ import threading
+
+ from unsloth.precision import engage
+
+ seen = {}
+ ready, go = threading.Barrier(2), threading.Event()
+
+ def worker(tag, launches):
+ with P.count_lowp_gemms() as c:
+ ready.wait()
+ for _ in range(launches):
+ engage.release_lowp_gemm(engage.note_lowp_gemm("unsloth::fp8_block_small_m_linear"))
+ go.wait(5)
+ seen[tag] = c.get("unsloth::fp8_block_small_m_linear", 0)
+
+ threads = [
+ threading.Thread(target = worker, args = ("busy", 1)),
+ threading.Thread(target = worker, args = ("idle", 0)),
+ ]
+ for t in threads:
+ t.start()
+ import time
+
+ time.sleep(0.2)
+ go.set()
+ for t in threads:
+ t.join()
+ assert seen == {"busy": 1, "idle": 0}, seen
diff --git a/tests/test_precision_selectors_families.py b/tests/test_precision_selectors_families.py
new file mode 100644
index 00000000000..92bbcc28fc2
--- /dev/null
+++ b/tests/test_precision_selectors_families.py
@@ -0,0 +1,767 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""unsloth.precision selectors over every model family transformers ships (tiny configs on the meta device, CPU
+only): decoder_layers is the language decoder's block list, find_output_heads / find_embeddings / find_routers
+return the expected modules (routers only for MoE families, never an expert / shared expert / dense MLP), and
+hp_filter(first_n = 1, last_n = 1) protects exactly the first and last language blocks.
+
+unsloth.precision is loaded standalone (it imports only torch), so this runs in any venv with transformers."""
+
+from __future__ import annotations
+
+import importlib.util
+import os
+import sys
+
+import pytest
+import torch
+import transformers
+
+_PKG = os.path.join(
+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "unsloth", "precision"
+)
+
+
+def _load_precision():
+ name = "_unsloth_precision_standalone"
+ if name in sys.modules:
+ return sys.modules[name]
+ spec = importlib.util.spec_from_file_location(
+ name, os.path.join(_PKG, "__init__.py"), submodule_search_locations = [_PKG]
+ )
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+P = _load_precision()
+
+TXT = dict(
+ vocab_size = 128,
+ hidden_size = 64,
+ intermediate_size = 128,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ head_dim = 16,
+ pad_token_id = 0,
+ bos_token_id = 1,
+ eos_token_id = 2,
+)
+MAMBA = dict(mamba_d_state = 8, mamba_d_conv = 4, mamba_expand = 2)
+
+
+def _cls(name):
+ c = getattr(transformers, name, None)
+ if c is None:
+ pytest.skip(f"{name} not in transformers {transformers.__version__}")
+ return c
+
+
+def _build(
+ model_cls,
+ config_cls,
+ meta = True,
+ **kwargs,
+):
+ config = _cls(config_cls)(**kwargs)
+ if meta:
+ with torch.device("meta"):
+ return _cls(model_cls)(config)
+ torch.manual_seed(0)
+ return _cls(model_cls)(config)
+
+
+def _causal(
+ arch,
+ n = 3,
+ **kw,
+):
+ return lambda: _build(
+ f"{arch}ForCausalLM", f"{arch}Config", **{**TXT, "num_hidden_layers": n, **kw}
+ )
+
+
+def _siglip(n = 5):
+ return dict(
+ hidden_size = 32,
+ intermediate_size = 64,
+ num_hidden_layers = n,
+ num_attention_heads = 4,
+ image_size = 32,
+ patch_size = 8,
+ )
+
+
+def _text(n = 3, **kw):
+ return {**TXT, "num_hidden_layers": n, **kw}
+
+
+# name -> (builder, decoder list paths (first one present wins), decoder length, router fqn suffixes, heads)
+# heads: None = {the module get_output_embeddings returns}; otherwise the exact expected set.
+FAMILIES = {
+ "llama": (_causal("Llama"), ["model.layers"], 3, (), None),
+ "mistral": (_causal("Mistral"), ["model.layers"], 3, (), None),
+ "qwen2": (_causal("Qwen2"), ["model.layers"], 3, (), None),
+ "qwen3_1layer": (_causal("Qwen3", 1), ["model.layers"], 1, (), None),
+ "qwen3_4layer": (_causal("Qwen3", 4), ["model.layers"], 4, (), None),
+ "gemma2": (_causal("Gemma2"), ["model.layers"], 3, (), None),
+ "gemma3_text": (
+ lambda: _build("Gemma3ForCausalLM", "Gemma3TextConfig", **_text()),
+ ["model.layers"],
+ 3,
+ (),
+ None,
+ ),
+ "gemma3_mm": (
+ lambda: _build(
+ "Gemma3ForConditionalGeneration",
+ "Gemma3Config",
+ text_config = _text(),
+ vision_config = _siglip(),
+ mm_tokens_per_image = 4,
+ ),
+ ["model.language_model.layers", "language_model.model.layers"],
+ 3,
+ (),
+ None,
+ ),
+ "gemma3n_text": (
+ lambda: _build(
+ "Gemma3nForCausalLM",
+ "Gemma3nTextConfig",
+ **_text(
+ 4,
+ laurel_rank = 8,
+ hidden_size_per_layer_input = 8,
+ altup_num_inputs = 4,
+ num_kv_shared_layers = 0,
+ layer_types = ["sliding_attention", "full_attention"] * 2,
+ activation_sparsity_pattern = [0.0] * 4,
+ ),
+ ),
+ ["model.layers"],
+ 4,
+ (),
+ None,
+ ),
+ "phi3": (_causal("Phi3"), ["model.layers"], 3, (), None),
+ "granite": (_causal("Granite"), ["model.layers"], 3, (), None),
+ "granitemoe": (
+ _causal("GraniteMoe", num_local_experts = 4, num_experts_per_tok = 2),
+ ["model.layers"],
+ 3,
+ (".block_sparse_moe.router",),
+ None,
+ ),
+ "granitemoehybrid": (
+ _causal(
+ "GraniteMoeHybrid",
+ 4,
+ num_local_experts = 4,
+ num_experts_per_tok = 2,
+ shared_intermediate_size = 64,
+ layer_types = ["mamba", "attention", "mamba", "attention"],
+ mamba_n_heads = 8,
+ mamba_d_head = 16,
+ mamba_n_groups = 1,
+ mamba_d_state = 8,
+ mamba_d_conv = 4,
+ mamba_expand = 2,
+ mamba_chunk_size = 16,
+ ),
+ ["model.layers"],
+ 4,
+ (".block_sparse_moe.router",),
+ None,
+ ),
+ "qwen2_moe": (
+ _causal(
+ "Qwen2Moe",
+ num_experts = 8,
+ num_experts_per_tok = 2,
+ moe_intermediate_size = 32,
+ shared_expert_intermediate_size = 64,
+ ),
+ ["model.layers"],
+ 3,
+ (".mlp.gate",),
+ None,
+ ),
+ "qwen3_moe": (
+ _causal("Qwen3Moe", num_experts = 8, num_experts_per_tok = 2, moe_intermediate_size = 32),
+ ["model.layers"],
+ 3,
+ (".mlp.gate",),
+ None,
+ ),
+ "qwen3_next": (
+ _causal(
+ "Qwen3Next",
+ 4,
+ num_experts = 8,
+ num_experts_per_tok = 2,
+ moe_intermediate_size = 32,
+ shared_expert_intermediate_size = 64,
+ layer_types = ["linear_attention", "full_attention"] * 2,
+ linear_num_value_heads = 4,
+ linear_num_key_heads = 2,
+ linear_key_head_dim = 16,
+ linear_value_head_dim = 16,
+ linear_conv_kernel_dim = 4,
+ ),
+ ["model.layers"],
+ 4,
+ (".mlp.gate",),
+ None,
+ ),
+ "mixtral": (
+ _causal("Mixtral", num_local_experts = 4, num_experts_per_tok = 2),
+ ["model.layers"],
+ 3,
+ (".block_sparse_moe.gate", ".mlp.gate"),
+ None,
+ ),
+ "olmoe": (
+ _causal("Olmoe", num_experts = 8, num_experts_per_tok = 2),
+ ["model.layers"],
+ 3,
+ (".mlp.gate",),
+ None,
+ ),
+ "gpt_oss": (
+ _causal(
+ "GptOss",
+ num_local_experts = 4,
+ num_experts_per_tok = 2,
+ layer_types = ["sliding_attention", "full_attention", "sliding_attention"],
+ ),
+ ["model.layers"],
+ 3,
+ (".mlp.router",),
+ None,
+ ),
+ "deepseek_v3": (
+ _causal(
+ "DeepseekV3",
+ n_routed_experts = 8,
+ num_experts_per_tok = 2,
+ n_shared_experts = 1,
+ first_k_dense_replace = 1,
+ moe_intermediate_size = 32,
+ n_group = 1,
+ topk_group = 1,
+ kv_lora_rank = 16,
+ q_lora_rank = 16,
+ qk_rope_head_dim = 8,
+ qk_nope_head_dim = 8,
+ v_head_dim = 16,
+ ),
+ ["model.layers"],
+ 3,
+ (".mlp.gate",),
+ None,
+ ),
+ "glm4_moe": (
+ _causal(
+ "Glm4Moe",
+ n_routed_experts = 8,
+ num_experts_per_tok = 2,
+ n_shared_experts = 1,
+ first_k_dense_replace = 1,
+ moe_intermediate_size = 32,
+ n_group = 1,
+ topk_group = 1,
+ ),
+ ["model.layers"],
+ 3,
+ (".mlp.gate",),
+ None,
+ ),
+ "llama4_text": (
+ lambda: _build(
+ "Llama4ForCausalLM",
+ "Llama4TextConfig",
+ **_text(
+ num_local_experts = 4,
+ num_experts_per_tok = 1,
+ interleave_moe_layer_step = 1,
+ intermediate_size_mlp = 128,
+ ),
+ ),
+ ["model.layers"],
+ 3,
+ (".feed_forward.router",),
+ None,
+ ),
+ "llama4_mm": (
+ lambda: _build(
+ "Llama4ForConditionalGeneration",
+ "Llama4Config",
+ text_config = _text(
+ num_local_experts = 4,
+ num_experts_per_tok = 1,
+ interleave_moe_layer_step = 1,
+ intermediate_size_mlp = 128,
+ ),
+ vision_config = dict(
+ hidden_size = 32,
+ intermediate_size = 64,
+ num_hidden_layers = 5,
+ num_attention_heads = 4,
+ image_size = 32,
+ patch_size = 8,
+ vision_output_dim = 64,
+ projector_input_dim = 64,
+ projector_output_dim = 64,
+ pixel_shuffle_ratio = 0.5,
+ ),
+ ),
+ ["language_model.model.layers", "model.language_model.layers"],
+ 3,
+ (".feed_forward.router",),
+ None,
+ ),
+ "jamba": (
+ _causal(
+ "Jamba",
+ 4,
+ num_experts = 4,
+ num_experts_per_tok = 2,
+ expert_layer_period = 2,
+ expert_layer_offset = 1,
+ attn_layer_period = 2,
+ attn_layer_offset = 1,
+ use_mamba_kernels = False,
+ **MAMBA,
+ ),
+ ["model.layers"],
+ 4,
+ (".feed_forward.router",),
+ None,
+ ),
+ "zamba": (
+ _causal(
+ "Zamba",
+ 6,
+ attn_layer_period = 2,
+ attn_layer_offset = 1,
+ mamba_dt_rank = 8,
+ n_mamba_heads = 2,
+ use_mamba_kernels = False,
+ **MAMBA,
+ ),
+ ["model.layers"],
+ 6,
+ (),
+ None,
+ ),
+ "zamba2": (
+ _causal(
+ "Zamba2",
+ 4,
+ layers_block_type = ["mamba", "hybrid", "mamba", "hybrid"],
+ mamba_headdim = 16,
+ mamba_ngroups = 1,
+ n_mamba_heads = 8,
+ num_mem_blocks = 1,
+ use_mamba_kernels = False,
+ **MAMBA,
+ ),
+ ["model.layers"],
+ 4,
+ (),
+ None,
+ ),
+ "falcon_h1": (
+ _causal(
+ "FalconH1",
+ mamba_d_ssm = 128,
+ mamba_n_heads = 8,
+ mamba_d_head = 16,
+ mamba_n_groups = 1,
+ mamba_chunk_size = 16,
+ **MAMBA,
+ ),
+ ["model.layers"],
+ 3,
+ (),
+ None,
+ ),
+ "nemotron_h": (
+ _causal(
+ "NemotronH",
+ hybrid_override_pattern = "M*-",
+ mamba_num_heads = 8,
+ mamba_head_dim = 16,
+ n_groups = 1,
+ ssm_state_size = 8,
+ conv_kernel = 4,
+ expand = 2,
+ chunk_size = 16,
+ ),
+ ["model.layers", "backbone.layers"],
+ 3,
+ (),
+ None,
+ ),
+ "nemotron_h_moe": (
+ _causal(
+ "NemotronH",
+ 4,
+ layers_block_type = ["linear_attention", "moe", "full_attention", "moe"],
+ n_routed_experts = 8,
+ num_experts_per_tok = 2,
+ moe_intermediate_size = 32,
+ moe_shared_expert_intermediate_size = 64,
+ mamba_num_heads = 8,
+ mamba_head_dim = 16,
+ n_groups = 1,
+ ssm_state_size = 8,
+ conv_kernel = 4,
+ expand = 2,
+ chunk_size = 16,
+ ),
+ ["model.layers"],
+ 4,
+ (".mixer.gate",),
+ None,
+ ),
+ "mamba": (
+ lambda: _build(
+ "MambaForCausalLM",
+ "MambaConfig",
+ vocab_size = 128,
+ hidden_size = 64,
+ state_size = 8,
+ num_hidden_layers = 3,
+ use_mambapy = False,
+ ),
+ ["backbone.layers"],
+ 3,
+ (),
+ None,
+ ),
+ "llava": (
+ lambda: _build(
+ "LlavaForConditionalGeneration",
+ "LlavaConfig",
+ text_config = dict(model_type = "llama", **_text()),
+ vision_config = dict(model_type = "clip_vision_model", **_siglip()),
+ ),
+ ["model.language_model.layers", "language_model.model.layers"],
+ 3,
+ (),
+ None,
+ ),
+ "qwen2_vl": (
+ lambda: _build(
+ "Qwen2VLForConditionalGeneration",
+ "Qwen2VLConfig",
+ text_config = _text(rope_scaling = {"type": "mrope", "mrope_section": [2, 3, 3]}),
+ vision_config = dict(depth = 5, embed_dim = 32, hidden_size = 64, num_heads = 4, mlp_ratio = 2),
+ **_text(rope_scaling = {"type": "mrope", "mrope_section": [2, 3, 3]}),
+ ),
+ ["model.language_model.layers", "model.layers"],
+ 3,
+ (),
+ None,
+ ),
+ "qwen2_5_vl": (
+ lambda: _build(
+ "Qwen2_5_VLForConditionalGeneration",
+ "Qwen2_5_VLConfig",
+ text_config = _text(rope_scaling = {"type": "mrope", "mrope_section": [2, 3, 3]}),
+ vision_config = dict(
+ depth = 5,
+ hidden_size = 32,
+ intermediate_size = 64,
+ out_hidden_size = 64,
+ num_heads = 4,
+ fullatt_block_indexes = [4],
+ ),
+ **_text(rope_scaling = {"type": "mrope", "mrope_section": [2, 3, 3]}),
+ ),
+ ["model.language_model.layers", "model.layers"],
+ 3,
+ (),
+ None,
+ ),
+ "mllama": (
+ lambda: _build(
+ "MllamaForConditionalGeneration",
+ "MllamaConfig",
+ text_config = dict(
+ vocab_size = 128,
+ hidden_size = 64,
+ intermediate_size = 128,
+ num_hidden_layers = 4,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ cross_attention_layers = [1],
+ pad_token_id = 0,
+ rope_scaling = {"rope_type": "default"},
+ ),
+ vision_config = dict(
+ hidden_size = 32,
+ intermediate_size = 64,
+ num_hidden_layers = 5,
+ num_global_layers = 2,
+ attention_heads = 4,
+ image_size = 32,
+ patch_size = 8,
+ vision_output_dim = 64,
+ intermediate_layers_indices = [1],
+ ),
+ ),
+ ["model.language_model.layers", "language_model.model.layers"],
+ 4,
+ (),
+ None,
+ ),
+ "idefics3": (
+ lambda: _build(
+ "Idefics3ForConditionalGeneration",
+ "Idefics3Config",
+ text_config = dict(model_type = "llama", **_text()),
+ vision_config = dict(
+ hidden_size = 32,
+ intermediate_size = 64,
+ num_hidden_layers = 5,
+ num_attention_heads = 4,
+ image_size = 32,
+ patch_size = 8,
+ ),
+ scale_factor = 2,
+ ),
+ ["model.text_model.layers"],
+ 3,
+ (),
+ None,
+ ),
+ "smolvlm": (
+ lambda: _build(
+ "SmolVLMForConditionalGeneration",
+ "SmolVLMConfig",
+ text_config = dict(model_type = "llama", **_text()),
+ vision_config = dict(
+ hidden_size = 32,
+ intermediate_size = 64,
+ num_hidden_layers = 5,
+ num_attention_heads = 4,
+ image_size = 32,
+ patch_size = 8,
+ ),
+ scale_factor = 2,
+ ),
+ ["model.text_model.layers"],
+ 3,
+ (),
+ None,
+ ),
+ "paligemma": (
+ lambda: _build(
+ "PaliGemmaForConditionalGeneration",
+ "PaliGemmaConfig",
+ text_config = dict(model_type = "gemma", **_text()),
+ vision_config = dict(model_type = "siglip_vision_model", **_siglip()),
+ projection_dim = 64,
+ hidden_size = 64,
+ ),
+ ["model.language_model.layers", "language_model.model.layers"],
+ 3,
+ (),
+ None,
+ ),
+ "whisper": (
+ lambda: _build(
+ "WhisperForConditionalGeneration",
+ "WhisperConfig",
+ vocab_size = 128,
+ d_model = 32,
+ encoder_layers = 5,
+ decoder_layers = 3,
+ encoder_attention_heads = 4,
+ decoder_attention_heads = 4,
+ encoder_ffn_dim = 64,
+ decoder_ffn_dim = 64,
+ pad_token_id = 0,
+ bos_token_id = 1,
+ eos_token_id = 2,
+ decoder_start_token_id = 1,
+ ),
+ ["model.decoder.layers"],
+ 3,
+ (),
+ None,
+ ),
+ "t5": (
+ lambda: _build(
+ "T5ForConditionalGeneration",
+ "T5Config",
+ vocab_size = 128,
+ d_model = 32,
+ d_kv = 8,
+ d_ff = 64,
+ num_layers = 5,
+ num_decoder_layers = 3,
+ num_heads = 4,
+ decoder_start_token_id = 0,
+ ),
+ ["decoder.block"],
+ 3,
+ (),
+ None,
+ ),
+ "bert_encoder": (
+ lambda: _build(
+ "BertModel",
+ "BertConfig",
+ vocab_size = 128,
+ hidden_size = 32,
+ num_hidden_layers = 3,
+ num_attention_heads = 4,
+ intermediate_size = 64,
+ ),
+ ["encoder.layer"],
+ 3,
+ (),
+ set(),
+ ),
+ "bert_seq_cls": (
+ lambda: _build(
+ "BertForSequenceClassification",
+ "BertConfig",
+ vocab_size = 128,
+ hidden_size = 32,
+ num_hidden_layers = 3,
+ num_attention_heads = 4,
+ intermediate_size = 64,
+ num_labels = 3,
+ ),
+ ["bert.encoder.layer"],
+ 3,
+ (),
+ {"classifier"},
+ ),
+ "llama_seq_cls": (
+ lambda: _build("LlamaForSequenceClassification", "LlamaConfig", **_text(), num_labels = 3),
+ ["model.layers"],
+ 3,
+ (),
+ {"score"},
+ ),
+ # Experts in a ModuleDict (`experts.expert_<i>`); one sparse layer each in the encoder and the decoder.
+ "switch_transformers": (
+ lambda: _build(
+ "SwitchTransformersForConditionalGeneration",
+ "SwitchTransformersConfig",
+ vocab_size = 128,
+ d_model = 32,
+ d_kv = 8,
+ d_ff = 64,
+ expert_capacity = 8,
+ num_layers = 3,
+ num_sparse_encoder_layers = 1,
+ num_decoder_layers = 3,
+ num_sparse_decoder_layers = 1,
+ num_heads = 4,
+ num_experts = 4,
+ decoder_start_token_id = 0,
+ ),
+ ["decoder.block"],
+ 3,
+ {"encoder.block.1.layer.1.mlp.router", "decoder.block.1.layer.2.mlp.router"},
+ None,
+ ),
+ # Attention MoE too: `self_attention.experts` (JetMoeMoA) owns stacked expert weights and its own router.
+ "jetmoe": (
+ _causal(
+ "JetMoe",
+ kv_channels = 16,
+ intermediate_size = 64,
+ num_local_experts = 4,
+ num_experts_per_tok = 2,
+ ),
+ ["model.layers"],
+ 3,
+ {
+ f"model.layers.{i}.{m}.router"
+ for i in range(3)
+ for m in ("mlp", "self_attention.experts")
+ },
+ None,
+ ),
+ "qwen3_tied_real_weights": (
+ lambda: _build(
+ "Qwen3ForCausalLM", "Qwen3Config", meta = False, **_text(tie_word_embeddings = True)
+ ),
+ ["model.layers"],
+ 3,
+ (),
+ {"lm_head"},
+ ),
+}
+
+
+def _lists(model):
+ return {n: m for n, m in model.named_modules() if isinstance(m, torch.nn.ModuleList)}
+
+
[email protected]("family", list(FAMILIES))
+def test_selectors_on_family(family):
+ build, paths, n, router_suffixes, heads = FAMILIES[family]
+ try:
+ model = build()
+ except pytest.skip.Exception:
+ raise
+ except Exception as e: # a config this transformers cannot build is not a selector result
+ pytest.skip(f"{family} does not build in transformers {transformers.__version__}: {e!r}")
+ names = dict(model.named_modules())
+ lists = _lists(model)
+ path = next((p for p in paths if p in lists), None)
+ assert path is not None, (family, paths, sorted(lists)[:20])
+ assert len(lists[path]) == n
+
+ # decoder_layers: the language decoder's block list.
+ assert P.decoder_layers(model) == (path, n), family
+
+ # hp_filter(first_n = 1, last_n = 1): exactly the first and last language blocks.
+ is_hp = P.hp_filter(
+ model,
+ P.HPLayerPolicy(output_head = False, embeddings = False, routers = False, first_n = 1, last_n = 1),
+ )
+ assert set(is_hp.keep) == {f"{path}.0", f"{path}.{n - 1}"}, (family, is_hp.keep)
+ for i in range(n):
+ for sub, _ in lists[path][i].named_modules():
+ fqn = f"{path}.{i}.{sub}" if sub else f"{path}.{i}"
+ assert is_hp(None, fqn) == (i in (0, n - 1)), (family, fqn)
+ for fqn in names:
+ if fqn and not (fqn == path or fqn.startswith(path + ".")):
+ assert not is_hp(None, fqn), (family, fqn)
+
+ # Output heads.
+ out = model.get_output_embeddings()
+ expected_heads = heads if heads is not None else {k for k, v in names.items() if v is out}
+ assert set(P.find_output_heads(model)) == expected_heads, family
+
+ # Embeddings: the input embedding is found, and nothing found is a Linear.
+ emb = P.find_embeddings(model)
+ inp = model.get_input_embeddings()
+ assert any(names[k] is inp for k in emb), family
+ assert not any(
+ isinstance(names[k], torch.nn.Linear) and names[k] is not inp for k in emb
+ ), family
+
+ # Routers: exactly the router modules, never an expert, shared expert or dense MLP.
+ # A set is the exact expected fqns; a tuple is suffixes matched outside any expert subtree.
+ if isinstance(router_suffixes, set):
+ expected_routers = router_suffixes
+ else:
+ expected_routers = {
+ k
+ for k in names
+ if router_suffixes and k.endswith(router_suffixes) and "expert" not in k
+ }
+ assert set(P.find_routers(model)) == expected_routers, family
diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index fd7bbd16329..340e9521b13 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -78,11 +78,33 @@ def _opaque_under_compile(name, fake):
ValueError,
): # Already registered by an earlier import of this module.
pass
- return fn
+ if name not in _EAGER_COUNTED_LOWP_GEMMS:
+ return fn
+ from ..precision import engage
+
+ # Eager calls launch Triton directly and never dispatch the op, so count_lowp_gemms() is told here.
+ @functools.wraps(fn)
+ def counted(*args, **kwargs):
+ if torch.compiler.is_compiling() or not engage._active():
+ return fn(*args, **kwargs)
+ entered = engage.note_lowp_gemm(f"unsloth::{name}")
+ try:
+ return fn(*args, **kwargs)
+ finally:
+ engage.release_lowp_gemm(entered)
+
+ return counted
return decorator
+_EAGER_COUNTED_LOWP_GEMMS = (
+ "fp8_block_matmul_triton",
+ "fp8_block_small_m_linear",
+ "fp8_block_matmul_torchao",
+)
+
+
try:
from transformers.integrations.finegrained_fp8 import FP8Linear
except:
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 0698721a41c..4184264fdde 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -10001,10 +10001,23 @@ def _prepare_model_for_qat(
_reinstall_input_require_grads(require_grads_owners)
_reregister_unowned_require_grads_hooks(model, all_embeddings, unowned_require_grads)
_patch_microscaling_qat_forwards(model)
+ _record_qat_engagement(model, torchao_config)
return model
+def _record_qat_engagement(model, torchao_config):
+ """Store how many layers QAT fake-quantizes on the model (unsloth.precision.engaged); under
+ UNSLOTH_STRICT_ENGAGE=1 a scheme that wrapped nothing raises instead of training unquantized."""
+ from ..precision import record, require
+
+ scheme = getattr(torchao_config, "qat_scheme", None)
+ name = "qat:" + (scheme if isinstance(scheme, str) else "custom")
+ count = sum(1 for m in model.modules() if "FakeQuantized" in type(m).__name__)
+ record(model, name, count)
+ require(name, count, "no Linear / Embedding layer matched the scheme's filters")
+
+
def patch_hf_quantizer():
def make_trainable(self):
return True
diff --git a/unsloth/models/float8_training.py b/unsloth/models/float8_training.py
index fab1bd63f1a..1cf9916d977 100644
--- a/unsloth/models/float8_training.py
+++ b/unsloth/models/float8_training.py
@@ -8,6 +8,42 @@ from __future__ import annotations
import torch
+
+def _load_precision():
+ """unsloth.precision; also when this file is loaded on its own (gloo test workers import it by path
+ without the unsloth package), from the sibling directory."""
+ try:
+ from .. import precision
+ return precision
+ except ImportError:
+ import importlib.util
+ import os
+ import sys
+
+ name = "_unsloth_precision_standalone"
+ if name in sys.modules:
+ return sys.modules[name]
+ root = os.path.join(
+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "precision"
+ )
+ spec = importlib.util.spec_from_file_location(
+ name, os.path.join(root, "__init__.py"), submodule_search_locations = [root]
+ )
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+_precision = _load_precision()
+_PRECISION_NAMES = _precision.FLOAT8_TRAINING_NAMES
+_check_precision_recipe = _precision.check_recipe
+_find_routers = _precision.find_routers
+_precision_measured_faster = _precision.measured_faster
+_precision_named_recipes = _precision.named_recipes
+_record_engagement = _precision.record
+_require_engagement = _precision.require
+
__all__ = [
"FLOAT8_TRAINING_RECIPES",
"resolve_float8_training",
@@ -15,21 +51,25 @@ __all__ = [
"convert_model_to_float8_training",
]
-FLOAT8_TRAINING_RECIPES = ("tensorwise",)
+# Names and gates come from the precision registry (unsloth/precision/recipes.py).
+FLOAT8_TRAINING_RECIPES = _precision_named_recipes("float8_training")
# Named recipes run only where they were timed faster than bfloat16 at equal loss (Qwen3-1.7B / 4B full
# finetuning). tensorwise: RTX PRO 6000 (12.0) 15% faster; L4 (8.9) NaN gradients with gradient
# checkpointing. rowwise: 3% slower on the RTX PRO 6000 and 2% slower on the L4, so it is not offered by
# name. Any other GPU needs an explicit torchao Float8LinearConfig.
-_MEASURED_FASTER = {"tensorwise": ((12, 0), (12, 1))}
+_MEASURED_FASTER = _precision_measured_faster("float8_training")
# Never fp8: output heads (Unsloth's fused CE reads lm_head.weight directly), MoE routers (tiny, precision sensitive).
_SKIP_NAME_PARTS = ("lm_head", "embed_out", "output_projection", "router")
_SKIP_LEAF_NAMES = ("gate", "router", "score", "classifier")
-def resolve_float8_training(float8_training):
- """`float8_training` kwarg -> recipe name or a torchao Float8LinearConfig; None when off."""
+def resolve_float8_training(float8_training, dtype = None):
+ """`float8_training` kwarg -> recipe name or a torchao Float8LinearConfig; None when off. A format the
+ precision registry knows but Unsloth cannot train (MXFP6, or one not built yet) raises with its reason,
+ before the model is loaded; `dtype` (the load dtype, None = auto) refuses float16 for formats whose
+ quantizer rejects it."""
if float8_training is None or float8_training is False:
return None
if float8_training is True:
@@ -42,6 +82,16 @@ def resolve_float8_training(float8_training):
"timed on (L4, RTX PRO 6000). Use float8_training = True (tensorwise), or pass "
"torchao.float8.Float8LinearConfig.from_recipe_name('rowwise') to force it."
)
+ if recipe not in FLOAT8_TRAINING_RECIPES and recipe in _PRECISION_NAMES:
+ # Raises PrecisionRefused (a ValueError) with the format's own reason.
+ _check_precision_recipe(
+ _PRECISION_NAMES[recipe], gpu = _precision.current_gpu_class(), input_dtype = dtype
+ )
+ raise ValueError(
+ f"Unsloth: float8_training = '{float8_training}' has not been measured faster than bfloat16 "
+ "on any GPU, so it is not offered by name. Pass "
+ f"torchao.float8.Float8LinearConfig.from_recipe_name({recipe!r}) to force it."
+ )
if recipe not in FLOAT8_TRAINING_RECIPES:
raise ValueError(
f"Unsloth: float8_training = '{float8_training}' is not supported. "
@@ -80,6 +130,12 @@ def float8_training_filter(
except Exception:
pass
size_recipe = _recipe_scaling(recipe)
+ # Routers found by shape beside an experts container (a router named `wg` or in remote code), on top of the
+ # name rules below.
+ try:
+ routers = frozenset(_find_routers(model))
+ except Exception:
+ routers = frozenset()
def module_filter_fn(module, fqn):
# Exact type: bitsandbytes / QAT / fp8 / PEFT linears subclass nn.Linear and must stay as they are.
@@ -87,7 +143,10 @@ def float8_training_filter(
return False
if id(module) in out_heads or any(p in fqn for p in _SKIP_NAME_PARTS):
return False
- if fqn.rsplit(".", 1)[-1] in _SKIP_LEAF_NAMES:
+ # A router found structurally can be a gating wrapper: its inner Linear is the router too.
+ if fqn.rsplit(".", 1)[-1] in _SKIP_LEAF_NAMES or any(
+ fqn == r or fqn.startswith(r + ".") for r in routers
+ ):
return False
weight = module.weight
if not weight.is_floating_point() or weight.element_size() < 2 or not weight.requires_grad:
@@ -327,6 +386,16 @@ def convert_model_to_float8_training(
converted = [name for name, m in model.named_modules() if isinstance(m, Float8Linear)]
if dispatch_hooks:
_reattach_dispatch_hooks(model, converted, dispatch_hooks)
+ name = (
+ recipe if isinstance(recipe, str) else "Float8LinearConfig(" + _recipe_scaling(recipe) + ")"
+ )
+ _record_engagement(model, "float8_training:" + name, len(converted))
+ # UNSLOTH_STRICT_ENGAGE=1 raises here instead of training in the original precision.
+ _require_engagement(
+ "float8_training",
+ len(converted),
+ "no eligible Linear layer: dims divisible by 16 and large enough",
+ )
if not converted:
import warnings
warnings.warn(
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 5a85254c007..6d25df63c2d 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -2263,7 +2263,7 @@ class FastModel(FastBaseModel):
):
rope_scaling = _fold_rope_parameters(rope_scaling, kwargs)
user_config = kwargs.pop("config", None)
- float8_training = resolve_float8_training(kwargs.pop("float8_training", None))
+ float8_training = resolve_float8_training(kwargs.pop("float8_training", None), dtype = dtype)
if float8_training is not None and not full_finetuning:
raise ValueError(
"Unsloth: `float8_training` is for `full_finetuning = True` only; LoRA / QLoRA keep "
diff --git a/unsloth/precision/__init__.py b/unsloth/precision/__init__.py
new file mode 100644
index 00000000000..e11634c5f7b
--- /dev/null
+++ b/unsloth/precision/__init__.py
@@ -0,0 +1,72 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Low-precision training policy: one recipe per format (per-GEMM dtypes and scaling, high-precision
+layers, GradScaler kind, per-GPU gates), refusals that run before the model is touched, structural
+high-precision layer selection and engagement checks. Data and checks only: the kernels stay in torchao and
+Unsloth's kernels, reached through the existing entry points (`float8_training`, `qat_scheme`, `load_in_fp8`)."""
+
+from .recipes import (
+ FLOAT8_TRAINING_NAMES,
+ RECIPES,
+ Gate,
+ GemmPolicy,
+ HPLayerPolicy,
+ Operand,
+ Recipe,
+ get_recipe,
+ measured_faster,
+ named_recipes,
+ recipes_of_kind,
+)
+from .hardware import (
+ PrecisionRefused,
+ check_recipe,
+ current_gpu_class,
+ dtype_tag,
+ gpu_class,
+ grad_scaler_kind,
+)
+from .selectors import (
+ decoder_layers,
+ find_embeddings,
+ find_output_heads,
+ find_routers,
+ high_precision_layers,
+ hp_filter,
+ num_experts_of,
+)
+from .engage import EngagementError, count_lowp_gemms, engaged, record, require, strict_engage
+
+__all__ = [
+ "FLOAT8_TRAINING_NAMES",
+ "RECIPES",
+ "Gate",
+ "GemmPolicy",
+ "HPLayerPolicy",
+ "Operand",
+ "Recipe",
+ "get_recipe",
+ "measured_faster",
+ "named_recipes",
+ "recipes_of_kind",
+ "PrecisionRefused",
+ "check_recipe",
+ "current_gpu_class",
+ "dtype_tag",
+ "gpu_class",
+ "grad_scaler_kind",
+ "decoder_layers",
+ "find_embeddings",
+ "find_output_heads",
+ "find_routers",
+ "high_precision_layers",
+ "hp_filter",
+ "num_experts_of",
+ "EngagementError",
+ "count_lowp_gemms",
+ "engaged",
+ "record",
+ "require",
+ "strict_engage",
+]
diff --git a/unsloth/precision/engage.py b/unsloth/precision/engage.py
new file mode 100644
index 00000000000..ea891206763
--- /dev/null
+++ b/unsloth/precision/engage.py
@@ -0,0 +1,205 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Engagement: proof that a requested low-precision path really ran (strict mode: UNSLOTH_STRICT_ENGAGE=1)."""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import threading
+import warnings
+from typing import Dict
+
+__all__ = [
+ "EngagementError",
+ "strict_engage",
+ "record",
+ "engaged",
+ "require",
+ "LOWP_GEMM_OPS",
+ "LOWP_GEMM_CUSTOM_OPS",
+ "count_lowp_gemms",
+]
+
+_ENV = "UNSLOTH_STRICT_ENGAGE"
+
+
+class EngagementError(RuntimeError):
+ """A requested feature silently fell back (strict mode)."""
+
+
+def strict_engage() -> bool:
+ return os.environ.get(_ENV, "0").strip().lower() in ("1", "true", "yes", "on")
+
+
+def record(
+ model,
+ recipe: str,
+ count: int,
+ detail = None,
+) -> None:
+ try:
+ table = getattr(model, "_unsloth_precision_engaged", None)
+ if not isinstance(table, dict):
+ table = {}
+ table[recipe] = {
+ "modules": int(count),
+ **({"detail": detail} if detail is not None else {}),
+ }
+ model._unsloth_precision_engaged = table
+ except Exception:
+ pass
+
+
+def engaged(model) -> Dict[str, dict]:
+ return dict(getattr(model, "_unsloth_precision_engaged", None) or {})
+
+
+def require(
+ recipe: str,
+ count: int,
+ why: str = "",
+) -> bool:
+ """True when engaged. Count 0 under strict mode raises EngagementError; otherwise returns False."""
+ if count > 0:
+ return True
+ if strict_engage():
+ raise EngagementError(
+ f"Unsloth: {recipe} was requested but engaged on 0 modules"
+ + (f" ({why})" if why else "")
+ + f". {_ENV}=1 refuses silent fallbacks; unset it to train in the original precision."
+ )
+ return False
+
+
+# aten ops that run a low-precision GEMM (names resolved lazily: absent ops are skipped).
+LOWP_GEMM_OPS = ("_scaled_mm", "_scaled_mm_v2", "_scaled_grouped_mm", "_scaled_grouped_mm_v2")
+
+# Opaque custom ops that run a low-precision GEMM inside: dispatch sees only the outer op, so each call counts
+# once at its boundary. Matched by qualified name, so a namespace that never loaded (no fbgemm) costs nothing.
+# Dequantize-then-matmul ops (mxfp4_bounded_dense_matmul, fp8 weight dequant, fp8_block_gemv, which upcasts
+# the weight before its dot) are not low-precision GEMMs.
+LOWP_GEMM_CUSTOM_OPS = (
+ "unsloth::nvfp4_qat_linear",
+ "unsloth::fp8_block_matmul_triton",
+ "unsloth::fp8_block_matmul_torchao",
+ "unsloth::fp8_block_small_m_linear",
+ "unsloth_zoo::mxfp4_ogs_moe",
+ "fbgemm::f8f8bf16",
+ "fbgemm::f8f8bf16_tensorwise",
+ "fbgemm::f8f8bf16_rowwise",
+ "fbgemm::f8f8bf16_blockwise",
+ "fbgemm::f8f8bf16_cublas",
+)
+
+
+# Per thread, like torch's dispatch-mode stack: the counters open on this thread, and the ids of those already
+# inside a counted GEMM (a GEMM that delegates to another counted launcher or op, torchao block matmul ->
+# Triton, counts once at its outer boundary). Eager launchers that bypass dispatch report via note_lowp_gemm.
+_STATE = threading.local()
+
+
+def _active():
+ active = getattr(_STATE, "active", None)
+ if active is None:
+ active = _STATE.active = []
+ return active
+
+
+def _inside():
+ ids = getattr(_STATE, "inside", None)
+ if ids is None:
+ ids = _STATE.inside = set()
+ return ids
+
+
+_EAGER_UNDER_COUNTER_WARNED = False
+
+
+def _warn_eager_under_counter(torch):
+ # torch < 2.9 has no TorchDispatchMode.ignore_compile_internals: Dynamo skips compiled code while any
+ # user dispatch mode is active, so counting would silently swap the compiled path for eager.
+ global _EAGER_UNDER_COUNTER_WARNED
+ if _EAGER_UNDER_COUNTER_WARNED:
+ return
+ _EAGER_UNDER_COUNTER_WARNED = True
+ warnings.warn(
+ f"Unsloth: count_lowp_gemms() on torch {torch.__version__} runs torch.compile'd callables eagerly "
+ "while counting (needs torch >= 2.9 to count compiled calls in place). Counts are exact for the "
+ "eager path, not for the compiled kernels.",
+ stacklevel = 3,
+ )
+
+
+def note_lowp_gemm(name):
+ """Count one GEMM in every open counter not already inside a counted GEMM; returns the ids to release
+ with `release_lowp_gemm` once the call returns."""
+ inside = _inside()
+ entered = [id(counts) for counts in _active() if id(counts) not in inside]
+ for counts in _active():
+ if id(counts) in entered:
+ counts[name] = counts.get(name, 0) + 1
+ inside.update(entered)
+ return entered
+
+
+def release_lowp_gemm(entered):
+ _inside().difference_update(entered)
+
+
+def count_lowp_gemms():
+ """Count low-precision GEMM dispatches: `with count_lowp_gemms() as c: ...; c['_scaled_mm']`, and
+ `c['unsloth::nvfp4_qat_linear']` for opaque custom ops (LOWP_GEMM_CUSTOM_OPS). On torch < 2.9, compiled
+ callables run eagerly inside the counter (warned once)."""
+ import torch
+ from torch.utils._python_dispatch import TorchDispatchMode
+
+ if not hasattr(TorchDispatchMode, "ignore_compile_internals"):
+ _warn_eager_under_counter(torch)
+
+ names = {}
+ for name in LOWP_GEMM_OPS:
+ if getattr(torch.ops.aten, name, None) is not None:
+ names[f"aten::{name}"] = name
+ for name in LOWP_GEMM_CUSTOM_OPS:
+ names[name] = name
+ counts = {key: 0 for key in names.values()}
+
+ class _Counter(TorchDispatchMode):
+ # Skip Dynamo tracing (a plain mode breaks fullgraph); torch < 2.9 ignores this hook (warned above).
+ @classmethod
+ def ignore_compile_internals(cls):
+ return True
+
+ def __torch_dispatch__(
+ self,
+ func,
+ types,
+ args = (),
+ kwargs = None,
+ ):
+ key = names.get(getattr(getattr(func, "_schema", None), "name", None))
+ inside = _inside()
+ if key is None or id(counts) in inside:
+ return func(*args, **(kwargs or {}))
+ counts[key] += 1
+ inside.add(id(counts))
+ try:
+ return func(*args, **(kwargs or {}))
+ finally:
+ inside.discard(id(counts))
+
+ _active().append(counts)
+ try:
+ with _Counter():
+ yield counts
+ finally:
+ # By identity: nested counters start as equal dicts, and list.remove compares by value.
+ active = _active()
+ for i, entry in enumerate(active):
+ if entry is counts:
+ del active[i]
+ break
+ _inside().discard(id(counts))
diff --git a/unsloth/precision/hardware.py b/unsloth/precision/hardware.py
new file mode 100644
index 00000000000..5afaf5f600b
--- /dev/null
+++ b/unsloth/precision/hardware.py
@@ -0,0 +1,134 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""GPU class of a device and the refusal check for a recipe on it. Everything here runs before the model is
+mutated: a refused request leaves the model as it was loaded."""
+
+from __future__ import annotations
+
+from typing import Optional, Tuple
+
+from .recipes import Gate, Recipe, get_recipe
+
+__all__ = [
+ "PrecisionRefused",
+ "gpu_class",
+ "current_gpu_class",
+ "dtype_tag",
+ "check_recipe",
+ "grad_scaler_kind",
+]
+
+
+class PrecisionRefused(ValueError):
+ """A precision request that cannot run here; raised before any layer is swapped."""
+
+
+def gpu_class(cc: Optional[Tuple[int, int]], hip: bool = False) -> Optional[str]:
+ """Compute capability -> the gate table's GPU class (None when unknown / not CUDA)."""
+ if hip:
+ return "rocm"
+ if cc is None:
+ return None
+ major, minor = int(cc[0]), int(cc[1])
+ if major >= 12:
+ return "sm120"
+ if major == 11 or major == 10:
+ return "sm100"
+ if major == 9:
+ return "sm90"
+ if (major, minor) >= (8, 9):
+ return "sm89"
+ if major == 8:
+ return "sm80"
+ return "sm75"
+
+
+def current_gpu_class(device = None) -> Optional[str]:
+ import torch
+ try:
+ if not torch.cuda.is_available():
+ return None
+ if torch.version.hip is not None:
+ return "rocm"
+ return gpu_class(torch.cuda.get_device_capability(device))
+ except Exception:
+ return None
+
+
+def dtype_tag(dtype) -> Optional[str]:
+ """torch.dtype / string -> 'bf16' / 'fp16' / 'fp32' (None when unknown or auto)."""
+ if dtype is None:
+ return None
+ s = str(dtype).replace("torch.", "").lower()
+ return {
+ "bfloat16": "bf16",
+ "bf16": "bf16",
+ "float16": "fp16",
+ "half": "fp16",
+ "fp16": "fp16",
+ "float32": "fp32",
+ "float": "fp32",
+ "fp32": "fp32",
+ }.get(s)
+
+
+_DTYPE_WHY = {
+ "fp16": "float16 activations are rejected by its quantizer (torchao to_mx / NVFP4 training accept "
+ "bfloat16 / float32 only); load with dtype = torch.bfloat16",
+ "fp32": "float32 activations are not supported by this recipe; load with dtype = torch.bfloat16",
+ "bf16": "bfloat16 activations are not supported by this recipe",
+}
+
+
+def check_recipe(
+ recipe,
+ *,
+ gpu: Optional[str] = None,
+ input_dtype = None,
+ mode: Optional[str] = None,
+ allow_planned: bool = False,
+) -> Gate:
+ """Refuse `recipe` (a Recipe or a registry name) for this GPU class / activation dtype / training mode.
+
+ Order: a format with no training path at all, then the dtype, then the mode, then the GPU gate, then
+ "not built yet". Unknown GPU or dtype (None) skips that check. Returns the gate that applies."""
+ r: Recipe = get_recipe(recipe) if isinstance(recipe, str) else recipe
+ if r.status == "refused":
+ raise PrecisionRefused(f"Unsloth: {r.name} training is refused: {r.refusal}")
+ tag = dtype_tag(input_dtype)
+ if tag is not None and tag not in r.input_dtypes:
+ raise PrecisionRefused(
+ f"Unsloth: {r.name} cannot train with {tag} activations: {_DTYPE_WHY[tag]}."
+ )
+ if mode is not None and mode not in r.modes:
+ raise PrecisionRefused(
+ f"Unsloth: {r.name} supports training modes {r.modes}, not {mode!r}."
+ )
+ gate = r.gate(gpu) if gpu is not None else Gate("unprobed")
+ if gate.status == "refused":
+ raise PrecisionRefused(
+ f"Unsloth: {r.name} is refused on {gpu}: {gate.reason}."
+ + (f" (evidence: {gate.evidence})" if gate.evidence else "")
+ )
+ if r.status == "planned" and not allow_planned:
+ where = ", ".join(f"{g}: {v.status}" for g, v in sorted(r.gates.items()))
+ raise PrecisionRefused(
+ f"Unsloth: {r.name} training is not available in Unsloth yet (planned; per GPU: {where}). "
+ "Train in bfloat16, or with float8_training = True on a supported GPU."
+ )
+ return gate
+
+
+def grad_scaler_kind(
+ recipe,
+ input_dtype,
+ fsdp2: bool = False,
+) -> str:
+ """'none', 'GradScaler' or 'ShardedGradScaler': the loss scaler this recipe needs with these activations."""
+ r: Recipe = get_recipe(recipe) if isinstance(recipe, str) else recipe
+ tag = dtype_tag(input_dtype)
+ needs = r.grad_scaler == "dynamic" or (r.grad_scaler == "per_dtype" and tag == "fp16")
+ if not needs:
+ return "none"
+ return "ShardedGradScaler" if fsdp2 else "GradScaler"
diff --git a/unsloth/precision/recipes.py b/unsloth/precision/recipes.py
new file mode 100644
index 00000000000..07d99d80fe6
--- /dev/null
+++ b/unsloth/precision/recipes.py
@@ -0,0 +1,577 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""One table entry per low-precision training format: what each GEMM runs in, how it is scaled, which
+layers stay in high precision, which GradScaler it needs, and where it runs.
+
+A recipe is data only. `hardware.check_recipe` turns the gate table into a refusal before the model is
+touched, `selectors.high_precision_layers` applies the HP-layer policy, and `engage` checks that the
+low-precision path really ran. The existing entry points (`float8_training`, `qat_scheme`, `load_in_fp8`) look
+their recipe up here, so the table is the one place that says what is supported.
+
+GEMM names: fwd = y = x W^T, dgrad = dx = dy W, wgrad = dW = dy^T x.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+from typing import Mapping, Optional, Tuple
+
+__all__ = [
+ "Operand",
+ "GemmPolicy",
+ "HPLayerPolicy",
+ "Gate",
+ "Recipe",
+ "RECIPES",
+ "get_recipe",
+ "recipes_of_kind",
+ "named_recipes",
+ "measured_faster",
+]
+
+
[email protected](frozen = True)
+class Operand:
+ """One GEMM operand: element dtype and how its scale is computed."""
+
+ dtype: str # e4m3, e5m2, e2m1, bf16, fp16, fp32, int4, int8, input (= the activation dtype)
+ granularity: str = "none" # none, tensor, row (axiswise), block1x128, block128x128, block1x32, block1x16, block16x16, group32, group128
+ scale_dtype: str = "none" # fp32, e8m0, e4m3, none
+ rounding: str = (
+ "rn" # rn, sr (stochastic), rceil (MX: scale rounded up), pow2 (power-of-two scales)
+ )
+ rht: bool = False # random Hadamard transform before quantizing (NVFP4 wgrad)
+
+
[email protected](frozen = True)
+class GemmPolicy:
+ """(a, b) operands of one GEMM; None = this GEMM runs in the activation dtype (high precision)."""
+
+ a: Optional[Operand] = None
+ b: Optional[Operand] = None
+
+ @property
+ def low_precision(self) -> bool:
+ return self.a is not None or self.b is not None
+
+
+HP = GemmPolicy()
+
+
[email protected](frozen = True)
+class HPLayerPolicy:
+ """Which linear layers stay in high precision. Detected by structure (selectors.py), not by exact names."""
+
+ output_head: bool = True # lm_head / get_output_embeddings and any weight tied to it
+ embeddings: bool = True # nn.Embedding and get_input_embeddings
+ routers: bool = (
+ True # MoE routers: out_features == number of experts beside an experts container
+ )
+ first_n: int = 0 # first N decoder layers
+ last_n: int = 0 # last N decoder layers
+ last_frac: float = 0.0 # or the last fraction of decoder layers (NVFP4 ~0.15), rounded up
+ shared_experts: bool = False # MoE shared expert (humans& NVFP4-RL keeps it bf16)
+
+
[email protected](frozen = True)
+class Gate:
+ """Status of a recipe on one GPU class.
+
+ status: named = runs by name (measured faster at equal loss),
+ explicit = runs only with an explicit torchao config (works, not measured faster),
+ refused = raises with `reason` before the model is touched,
+ unprobed = never run on this class; treated as explicit with a warning.
+ """
+
+ status: str
+ reason: str = ""
+ evidence: str = ""
+
+
[email protected](frozen = True)
+class Recipe:
+ name: str
+ kind: str # dtype, float8_training, qat, frozen_base
+ fwd: GemmPolicy = HP
+ dgrad: GemmPolicy = HP
+ wgrad: GemmPolicy = HP
+ hp_layers: HPLayerPolicy = HPLayerPolicy()
+ input_dtypes: Tuple[str, ...] = ("bf16", "fp16", "fp32")
+ modes: Tuple[str, ...] = ("full",) # full, lora, qlora
+ grad_scaler: str = "none" # none, dynamic (fp16 loss scaling), per_dtype (dynamic only when the activations are fp16)
+ compile: str = (
+ "ok" # ok, opaque (wrapped in a custom op inside compiled regions), fails, unknown
+ )
+ backend: str = "" # entry point that implements it
+ status: str = "shipped" # shipped, planned, refused
+ refusal: str = "" # reason when status == refused (for every GPU)
+ gates: Mapping[str, Gate] = dataclasses.field(default_factory = dict) # gpu class -> Gate
+ alias_of: str = ""
+ notes: str = ""
+
+ def gate(self, gpu_class: str) -> Gate:
+ if self.status == "refused":
+ return Gate("refused", self.refusal)
+ return self.gates.get(gpu_class, Gate("unprobed"))
+
+
+E4M3 = "e4m3"
+E5M2 = "e5m2"
+
+
+def _op(
+ dtype,
+ gran,
+ scale = "fp32",
+ rounding = "rn",
+ rht = False,
+):
+ return Operand(dtype, gran, scale, rounding, rht)
+
+
+_NO_FP8_HW = Gate("refused", "needs fp8 tensor cores (compute capability 8.9+)")
+_NO_MX_HW = Gate("refused", "needs block-scaled tensor cores (compute capability 10.0+ / 12.0)")
+_HP_FP8 = HPLayerPolicy(output_head = True, embeddings = True, routers = True)
+
+# GPU classes: sm75 (T4), sm80 (A100), sm89 (L4 / RTX 40), sm90 (H100), sm100 (B200 / GB200),
+# sm120 (RTX PRO 6000 / RTX 50), rocm. Evidence strings name the probe that set the gate.
+_PROBE_G4 = "probed on an RTX PRO 6000 (sm120), torch 2.13.0, torchao 0.18.0"
+_PROBE_L4 = "probed on an L4 (sm89), torch 2.13.0, torchao 0.18.0"
+_PROBE_B200 = "probed on a B200 (sm100), torch 2.13, torchao 0.18"
+
+RECIPES = {}
+
+
+def _add(recipe: Recipe):
+ assert recipe.name not in RECIPES, recipe.name
+ RECIPES[recipe.name] = recipe
+ return recipe
+
+
+# ---- plain dtypes -----------------------------------------------------------------------------------
+_add(
+ Recipe(
+ name = "fp16",
+ kind = "dtype",
+ input_dtypes = ("fp16",),
+ modes = ("full", "lora", "qlora"),
+ grad_scaler = "dynamic",
+ backend = "torch.amp.GradScaler (fsdp2_native._ShardedGradScaler under FSDP2)",
+ gates = {
+ g: Gate("named") for g in ("sm75", "sm80", "sm89", "sm90", "sm100", "sm120", "rocm")
+ },
+ notes = "fp32 masters for trainable params; Megatron default scaler init 2^32, window 1000, hysteresis 2",
+ )
+)
+_add(
+ Recipe(
+ name = "bf16",
+ kind = "dtype",
+ input_dtypes = ("bf16",),
+ modes = ("full", "lora", "qlora"),
+ gates = {
+ "sm75": Gate("refused", "no bfloat16 tensor cores; Unsloth downgrades to float16"),
+ **{g: Gate("named") for g in ("sm80", "sm89", "sm90", "sm100", "sm120", "rocm")},
+ },
+ )
+)
+
+# ---- torchao float8 training (full finetuning) ------------------------------------------------------
+_add(
+ Recipe(
+ name = "fp8_tensorwise",
+ kind = "float8_training",
+ fwd = GemmPolicy(_op(E4M3, "tensor"), _op(E4M3, "tensor")),
+ dgrad = GemmPolicy(_op(E5M2, "tensor"), _op(E4M3, "tensor")),
+ wgrad = GemmPolicy(_op(E5M2, "tensor"), _op(E4M3, "tensor")),
+ hp_layers = _HP_FP8,
+ grad_scaler = "per_dtype",
+ backend = "torchao.float8.Float8LinearConfig.from_recipe_name('tensorwise')",
+ gates = {
+ "sm75": _NO_FP8_HW,
+ "sm80": _NO_FP8_HW,
+ "sm89": Gate(
+ "refused",
+ "NaN gradients on an L4: the compiled bfloat16 weight gradient is NaN at "
+ "2048 x 4096 x 4096 (3 of 3 seeds) and was 0.50 relative off at 512 x 1024 x 2048 once, while "
+ "eager and the compiled forward / input gradient are correct",
+ _PROBE_L4,
+ ),
+ "sm90": Gate("explicit"),
+ "sm100": Gate("explicit", evidence = _PROBE_B200),
+ "sm120": Gate(
+ "named",
+ evidence = "Qwen3 full FT 15% faster on RTX PRO 6000 (float8_training.py); "
+ "compiled linear fwd+bwd 1.56-1.61x bf16 at 4096 x 14336 (" + _PROBE_G4 + ")",
+ ),
+ },
+ alias_of = "",
+ notes = "float8_training = True",
+ )
+)
+_add(
+ Recipe(
+ name = "fp8_rowwise",
+ kind = "float8_training",
+ fwd = GemmPolicy(_op(E4M3, "row", rounding = "pow2"), _op(E4M3, "row", rounding = "pow2")),
+ dgrad = GemmPolicy(_op(E4M3, "row", rounding = "pow2"), _op(E4M3, "row", rounding = "pow2")),
+ wgrad = GemmPolicy(_op(E4M3, "row", rounding = "pow2"), _op(E4M3, "row", rounding = "pow2")),
+ hp_layers = _HP_FP8,
+ grad_scaler = "per_dtype",
+ backend = "torchao.float8.Float8LinearConfig.from_recipe_name('rowwise')",
+ gates = {
+ "sm75": _NO_FP8_HW,
+ "sm80": _NO_FP8_HW,
+ "sm89": Gate(
+ "explicit",
+ evidence = "2% slower than bf16 on an L4 (float8_training.py); compiled linear "
+ "0.69-1.13x bf16 (" + _PROBE_L4 + ")",
+ ),
+ "sm90": Gate("explicit"),
+ "sm100": Gate("explicit", evidence = _PROBE_B200),
+ "sm120": Gate(
+ "explicit",
+ evidence = "3% slower than bf16 on RTX PRO 6000 (float8_training.py); compiled "
+ "linear 0.54-0.93x bf16 (" + _PROBE_G4 + ")",
+ ),
+ },
+ )
+)
+_add(
+ Recipe(
+ name = "fp8_rowwise_gw_hp",
+ kind = "float8_training",
+ fwd = GemmPolicy(_op(E4M3, "row"), _op(E4M3, "row")),
+ dgrad = GemmPolicy(_op(E4M3, "row"), _op(E4M3, "tensor")),
+ wgrad = HP,
+ hp_layers = _HP_FP8,
+ grad_scaler = "per_dtype",
+ backend = "torchao.float8.Float8LinearConfig.from_recipe_name('rowwise_with_gw_hp')",
+ gates = {
+ "sm75": _NO_FP8_HW,
+ "sm80": _NO_FP8_HW,
+ "sm89": Gate(
+ "explicit", evidence = "compiled linear 0.93-1.14x bf16 (" + _PROBE_L4 + ")"
+ ),
+ "sm90": Gate("explicit"),
+ "sm100": Gate("explicit", evidence = _PROBE_B200),
+ "sm120": Gate(
+ "explicit", evidence = "compiled linear 0.86-1.24x bf16 (" + _PROBE_G4 + ")"
+ ),
+ },
+ )
+)
+_add(
+ Recipe(
+ name = "fp8_blockwise",
+ kind = "float8_training",
+ fwd = GemmPolicy(_op(E4M3, "block1x128"), _op(E4M3, "block128x128")),
+ dgrad = GemmPolicy(_op(E4M3, "block1x128"), _op(E4M3, "block128x128")),
+ wgrad = GemmPolicy(_op(E4M3, "block1x128"), _op(E4M3, "block128x1")),
+ hp_layers = HPLayerPolicy(first_n = 1, last_n = 2),
+ input_dtypes = ("bf16",),
+ backend = "torchao.prototype.blockwise_fp8_training.linear.Float8BlockwiseLinear",
+ status = "planned",
+ gates = {
+ **{
+ g: Gate(
+ "refused",
+ "torchao's DeepSeek 1x128 / 128x128 scaled_mm is SM90 only in torch "
+ "(Transformer Engine Float8BlockScaling does run on sm120)",
+ _PROBE_B200 + "; " + _PROBE_G4 + "; " + _PROBE_L4,
+ )
+ for g in ("sm75", "sm80", "sm89", "sm100", "sm120", "rocm")
+ },
+ "sm90": Gate("unprobed", "H100 is not available to test on"),
+ },
+ notes = "fp16 rejected by torchao; NVFP4 paper FP8 baseline keeps first 1 + last 2 blocks in bf16",
+ )
+)
+
+# ---- microscaling formats ---------------------------------------------------------------------------
+_MX = _op(E4M3, "block1x32", "e8m0", "rceil")
+_add(
+ Recipe(
+ name = "mxfp8",
+ kind = "float8_training",
+ fwd = GemmPolicy(_MX, _MX),
+ dgrad = GemmPolicy(_MX, _MX),
+ wgrad = GemmPolicy(_MX, _MX),
+ hp_layers = HPLayerPolicy(output_head = True, embeddings = True, routers = True),
+ input_dtypes = ("bf16",),
+ grad_scaler = "none",
+ backend = "torchao.prototype.moe_training.config.MXFP8TrainingOpConfig (MXFP8_RCEIL)",
+ status = "planned",
+ gates = {
+ "sm75": _NO_MX_HW,
+ "sm80": _NO_MX_HW,
+ "sm89": Gate(
+ "refused",
+ "torchao MXFP8 kernels need block-scaled tensor cores (L4: 'needs triton')",
+ _PROBE_L4,
+ ),
+ "sm90": _NO_MX_HW,
+ "sm100": Gate("explicit", evidence = _PROBE_B200),
+ "sm120": Gate(
+ "explicit",
+ evidence = "linear fwd / dgrad / wgrad rel err 0.038 (= fp8 rowwise), eager and "
+ "compiled; compiled 1.49-1.55x bf16 at 4096 x 14336 ("
+ + _PROBE_G4
+ + "). Transformer "
+ "Engine 2.20 refuses MXFP8 on 12.0+",
+ ),
+ },
+ notes = "arXiv 2506.08027: e4m3 for every tensor incl. gradients, scale rounded up (RCEIL), all blocks quantized; "
+ "torchao to_mx rejects float16 inputs",
+ )
+)
+_add(
+ Recipe(
+ name = "mxfp6",
+ kind = "float8_training",
+ status = "refused",
+ refusal = "MXFP6 has no training kernel in torchao, Transformer Engine or Megatron (only QAT / emulation), "
+ "and no tensor-core speed over MXFP8 on Blackwell. Use mxfp8, or qat_scheme for MXFP6 deployment.",
+ )
+)
+_NV_X = _op("e2m1", "block1x16", "e4m3", "rn")
+_NV_W = _op("e2m1", "block16x16", "e4m3", "rn")
+_NV_G = _op("e2m1", "block1x16", "e4m3", "sr")
+_add(
+ Recipe(
+ name = "nvfp4",
+ kind = "float8_training",
+ fwd = GemmPolicy(_NV_X, _NV_W),
+ dgrad = GemmPolicy(_NV_G, _NV_W),
+ wgrad = GemmPolicy(
+ _op("e2m1", "block1x16", "e4m3", "sr", rht = True),
+ _op("e2m1", "block1x16", "e4m3", "rn", rht = True),
+ ),
+ hp_layers = HPLayerPolicy(output_head = True, embeddings = True, routers = True, last_frac = 0.15),
+ input_dtypes = ("bf16",),
+ grad_scaler = "none",
+ compile = "opaque",
+ backend = "torchao.prototype.moe_training.nvfp4_training.NVFP4TrainingConfig",
+ status = "planned",
+ gates = {
+ "sm75": _NO_MX_HW,
+ "sm80": _NO_MX_HW,
+ "sm89": Gate("refused", "torchao NVFP4 kernels require SM100+", _PROBE_L4),
+ "sm90": _NO_MX_HW,
+ "sm100": Gate("explicit", evidence = _PROBE_B200 + " (eager only)"),
+ "sm120": Gate(
+ "refused",
+ "torchao NVFP4 training does not build on sm_120a (Triton: cvt.rs stochastic "
+ "rounding unsupported; Inductor cannot lower the swizzled scaled_mm). Transformer Engine "
+ "2.20 NVFP4BlockScaling runs in bfloat16 (rel err 0.15-0.18, linear fwd+bwd 2.1-2.4x bf16) "
+ "and is the sm120 route",
+ _PROBE_G4,
+ ),
+ },
+ notes = "arXiv 2509.25149: 16-element blocks, e4m3 block scale + fp32 tensor scale, 2D 16x16 weights, "
+ "RHT 16x16 on wgrad inputs only, SR on gradients only, ~15% of layers (mostly last) in bf16. "
+ "Inductor cannot lower its swizzled scaled_mm, so it runs as an opaque op under compile.",
+ )
+)
+
+# ---- MoE grouped experts (torch._grouped_mm in low precision) -----------------------------------------
+_add(
+ Recipe(
+ name = "fp8_rowwise_grouped",
+ kind = "float8_training",
+ fwd = GemmPolicy(_op(E4M3, "row"), _op(E4M3, "row")),
+ dgrad = GemmPolicy(_op(E4M3, "row"), _op(E4M3, "row")),
+ wgrad = GemmPolicy(_op(E4M3, "row"), _op(E4M3, "row")),
+ input_dtypes = ("bf16",),
+ backend = "torchao.prototype.moe_training.config.Float8TrainingOpConfig",
+ status = "planned",
+ gates = {
+ "sm75": _NO_FP8_HW,
+ "sm80": _NO_FP8_HW,
+ **{
+ g: Gate(
+ "refused",
+ "torch._scaled_grouped_mm runs on compute capability 9.0 / 10.0 only",
+ _PROBE_G4 + "; " + _PROBE_L4,
+ )
+ for g in ("sm89", "sm120")
+ },
+ "sm90": Gate("unprobed"),
+ "sm100": Gate(
+ "refused",
+ "cutlass cannot run (error 7); an unspecified launch failure poisons the CUDA "
+ "context when run in a shared process",
+ _PROBE_B200,
+ ),
+ },
+ notes = "MoE experts (3-D weights); fallback is dequant + bf16 torch._grouped_mm",
+ )
+)
+_add(
+ Recipe(
+ name = "mxfp8_grouped",
+ kind = "float8_training",
+ fwd = GemmPolicy(_MX, _MX),
+ dgrad = GemmPolicy(_MX, _MX),
+ wgrad = GemmPolicy(_MX, _MX),
+ input_dtypes = ("bf16",),
+ backend = "torchao.prototype.moe_training.config.MXFP8TrainingOpConfig (grouped)",
+ status = "planned",
+ gates = {
+ "sm75": _NO_MX_HW,
+ "sm80": _NO_MX_HW,
+ "sm90": _NO_MX_HW,
+ **{
+ g: Gate(
+ "refused",
+ "torchao's MXFP8 grouped kernels are SM100 only (EMULATED runs, no speed)",
+ _PROBE_G4 + "; " + _PROBE_L4,
+ )
+ for g in ("sm89", "sm120")
+ },
+ "sm100": Gate(
+ "unprobed",
+ "DSLRuntimeError with nvidia-cutlass-dsl 4.6.2 (" + _PROBE_B200 + "); "
+ "pin a torchao / cutlass-dsl pair whose subprocess probe passes",
+ ),
+ },
+ notes = "MoE experts; fallback is dequant + bf16 torch._grouped_mm",
+ )
+)
+
+# ---- QAT (fake quant of the frozen or trained base; the GEMM itself runs in the activation dtype) --------
+for _name, _dtypes, _note in (
+ ("fp8-int4", ("bf16", "fp16", "fp32"), "Float8DynamicActivationInt4WeightConfig, group 128"),
+ ("fp8-fp8", ("bf16", "fp16", "fp32"), "Float8DynamicActivationFloat8WeightConfig PerRow"),
+ ("nvfp4", ("bf16", "fp32"), "NVFP4DynamicActivationNVFP4WeightConfig, K % 32"),
+ ("mxfp8", ("bf16",), "MXDynamicActivationMXWeightConfig, torchao 0.16+, bf16 layers only"),
+ ("int8-int4", ("bf16", "fp16", "fp32"), "phone-deployment alias"),
+ ("int4", ("bf16", "fp16", "fp32"), "Int4WeightOnlyConfig, group 128"),
+ ("int8", ("bf16", "fp16", "fp32"), "Int8 weight only"),
+ ("cactus", ("bf16", "fp16", "fp32"), "Cactus int8 / int4 per group"),
+):
+ _add(
+ Recipe(
+ name = "qat:" + _name,
+ kind = "qat",
+ input_dtypes = _dtypes,
+ modes = ("full", "lora"),
+ grad_scaler = "per_dtype",
+ compile = "opaque",
+ backend = "unsloth.models._utils._prepare_model_for_qat",
+ gates = {},
+ notes = _note,
+ )
+ )
+_add(
+ Recipe(
+ name = "qat:phone-deployment",
+ kind = "qat",
+ alias_of = "qat:int8-int4",
+ modes = ("full", "lora"),
+ backend = "unsloth.models._utils._prepare_model_for_qat",
+ )
+)
+
+# ---- frozen low-precision bases for LoRA (fwd low precision, dgrad dequantized, no wgrad) -------------
+_add(
+ Recipe(
+ name = "frozen_fp8_rowwise",
+ kind = "frozen_base",
+ modes = ("lora",),
+ fwd = GemmPolicy(_op(E4M3, "row"), _op(E4M3, "row")),
+ backend = "unsloth.kernels.fp8 (load_in_fp8 = True)",
+ gates = {
+ "sm75": _NO_FP8_HW,
+ "sm80": _NO_FP8_HW,
+ "sm89": Gate("named"),
+ "sm90": Gate("named"),
+ "sm100": Gate("named"),
+ "sm120": Gate("named"),
+ },
+ notes = "dgrad dequantizes the fp8 weight to 16 bit; fast accum in the forward",
+ )
+)
+_add(
+ Recipe(
+ name = "frozen_fp8_block",
+ kind = "frozen_base",
+ modes = ("lora",),
+ fwd = GemmPolicy(_op(E4M3, "block1x128"), _op(E4M3, "block128x128")),
+ backend = "unsloth.kernels.fp8 (load_in_fp8 = 'block')",
+ gates = {
+ "sm75": _NO_FP8_HW,
+ "sm80": _NO_FP8_HW,
+ "sm89": Gate("named"),
+ "sm90": Gate("named"),
+ "sm100": Gate("named"),
+ "sm120": Gate("named"),
+ },
+ notes = "Triton block kernels, forward only; dgrad dequantizes",
+ )
+)
+
+
+def get_recipe(name: str) -> Recipe:
+ try:
+ recipe = RECIPES[name]
+ except KeyError:
+ raise KeyError(
+ f"Unsloth: unknown precision recipe {name!r}; known: {sorted(RECIPES)}"
+ ) from None
+ return RECIPES[recipe.alias_of] if recipe.alias_of else recipe
+
+
+def recipes_of_kind(kind: str):
+ return {n: r for n, r in RECIPES.items() if r.kind == kind}
+
+
+# float8_training = '<name>' accepts these spellings; a name maps to its recipe.
+FLOAT8_TRAINING_NAMES = {
+ "tensorwise": "fp8_tensorwise",
+ "rowwise": "fp8_rowwise",
+ "rowwise_with_gw_hp": "fp8_rowwise_gw_hp",
+ "blockwise": "fp8_blockwise",
+ "mxfp8": "mxfp8",
+ "mxfp6": "mxfp6",
+ "nvfp4": "nvfp4",
+}
+
+
+def named_recipes(kind: str = "float8_training"):
+ """Float8-training names that run by name somewhere (some gate is `named`)."""
+ out = []
+ for short, full in FLOAT8_TRAINING_NAMES.items():
+ r = RECIPES[full]
+ if (
+ r.kind == kind
+ and r.status == "shipped"
+ and any(g.status == "named" for g in r.gates.values())
+ ):
+ out.append(short)
+ return tuple(out)
+
+
+_CLASS_CC = {
+ "sm120": ((12, 0), (12, 1)),
+ "sm100": ((10, 0), (10, 3)),
+ "sm90": ((9, 0),),
+ "sm89": ((8, 9),),
+}
+
+
+def measured_faster(kind: str = "float8_training"):
+ """{short name: compute capabilities where it runs by name}, the shape float8_training._MEASURED_FASTER had."""
+ out = {}
+ for short in named_recipes(kind):
+ r = RECIPES[FLOAT8_TRAINING_NAMES[short]]
+ ccs = tuple(
+ cc
+ for g, gate in r.gates.items()
+ if gate.status == "named"
+ for cc in _CLASS_CC.get(g, ())
+ )
+ if ccs:
+ out[short] = ccs
+ return out
diff --git a/unsloth/precision/selectors.py b/unsloth/precision/selectors.py
new file mode 100644
index 00000000000..6c5a19e6df9
--- /dev/null
+++ b/unsloth/precision/selectors.py
@@ -0,0 +1,350 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""High-precision layer selection by structure: output heads (and weights tied to them), embeddings, MoE
+routers and the first / last decoder layers. Names are a fallback only: a router called `wg` or `gate_proj`,
+or one in remote code, is found by its shape and its place beside the experts."""
+
+from __future__ import annotations
+
+from typing import Dict, Optional
+
+import torch
+
+from .recipes import HPLayerPolicy
+
+__all__ = [
+ "num_experts_of",
+ "find_routers",
+ "find_output_heads",
+ "find_embeddings",
+ "decoder_layers",
+ "high_precision_layers",
+ "hp_filter",
+]
+
+_EXPERT_COUNT_KEYS = (
+ "num_experts",
+ "num_local_experts",
+ "n_routed_experts",
+ "moe_num_experts",
+ "num_routed_experts",
+)
+_SUBCONFIG_KEYS = ("text_config", "language_config", "llm_config", "decoder")
+
+
+def num_experts_of(config) -> Optional[int]:
+ """Number of routed experts from a (possibly nested) HF config, None for a dense model."""
+ seen = set()
+ stack = [config]
+ while stack:
+ c = stack.pop()
+ if c is None or id(c) in seen:
+ continue
+ seen.add(id(c))
+ for key in _EXPERT_COUNT_KEYS:
+ v = c.get(key) if isinstance(c, dict) else getattr(c, key, None)
+ if isinstance(v, int) and not isinstance(v, bool) and v > 1:
+ return v
+ for key in _SUBCONFIG_KEYS:
+ stack.append(c.get(key) if isinstance(c, dict) else getattr(c, key, None))
+ return None
+
+
+def _experts_count(module) -> Optional[int]:
+ """Expert count of an experts container: ModuleList / ModuleDict length (SwitchTransformers keeps its
+ experts in a ModuleDict), `num_experts` attribute, or the leading dim of its 3-D parameters (stacked
+ expert weights)."""
+ if isinstance(module, (torch.nn.ModuleList, torch.nn.ModuleDict)) and len(module) > 1:
+ return len(module)
+ for key in _EXPERT_COUNT_KEYS:
+ v = getattr(module, key, None)
+ if isinstance(v, int) and not isinstance(v, bool) and v > 1:
+ return v
+ for p in module.parameters(recurse = False):
+ if p.dim() == 3 and p.shape[0] > 1:
+ return int(p.shape[0])
+ return None
+
+
+def _out_features(module) -> Optional[int]:
+ w = getattr(module, "weight", None)
+ if isinstance(w, torch.Tensor) and w.dim() == 2:
+ return int(w.shape[0])
+ return getattr(module, "out_features", None)
+
+
+def _router_width(module) -> Optional[int]:
+ """Output width of a router: its own 2-D weight, else the one 2-D weight inside it (a gating module that
+ wraps a Linear, GraniteMoE's `router.layer`)."""
+ out = _out_features(module)
+ if out is not None:
+ return out
+ inner = [p for p in module.parameters() if p.dim() == 2]
+ return int(inner[0].shape[0]) if len(inner) == 1 else None
+
+
+def _is_stacked_experts(module, n_cfg) -> bool:
+ """A container holding all experts' weights as 3-D parameters [num_experts, ...] (GraniteMoE's
+ ParallelExperts `input_linear`), whatever it is called. Convolutions (Mamba's conv1d) are 3-D too."""
+ if not n_cfg or isinstance(module, torch.nn.modules.conv._ConvNd):
+ return False
+ return any(p.dim() == 3 and p.shape[0] == n_cfg for p in module.parameters(recurse = False))
+
+
+def _is_experts_child(name, module, n_cfg) -> bool:
+ return ("expert" in name.lower() and "shared" not in name.lower()) or _is_stacked_experts(
+ module, n_cfg
+ )
+
+
+def _expert_subtrees(model, n_cfg) -> set:
+ """ids of modules inside an individual expert or a shared expert: the subtree of every experts child
+ (`experts.3`, `experts.expert_0`) and of every shared-expert MLP. An MoE container that owns stacked
+ expert weights in its children, and so routes over them itself (JetMoE's attention `experts`, whose
+ router is `experts.router`), is not a collection of experts and stays inspectable."""
+ inside = set()
+ for _, parent in model.named_modules():
+ for name, child in parent.named_children():
+ if "shared" not in name.lower() and not _is_experts_child(name, child, n_cfg):
+ continue
+ if "shared" not in name.lower() and any(
+ _is_stacked_experts(c, n_cfg) for c in child.children()
+ ):
+ continue
+ inside.update(id(m) for m in child.modules())
+ return inside
+
+
+def find_routers(model) -> Dict[str, str]:
+ """{fqn: reason} for every module that scores tokens over experts.
+
+ A module is a router when its output width equals the expert count and it sits in the same block as an
+ experts container (a child whose name contains 'expert', excluding shared experts, or one holding stacked
+ 3-D expert weights). With no experts container in its block, the config's expert count plus a 2-D weight
+ whose width matches is enough only for a module whose leaf name is a known router name."""
+ config = getattr(model, "config", None)
+ n_cfg = num_experts_of(config) if config is not None else None
+ found = {}
+ inside_experts = _expert_subtrees(model, n_cfg)
+ for parent_name, parent in model.named_modules():
+ # Inside an expert or shared expert (`experts.3`, `shared_expert`): its projections can be as wide as
+ # the expert count (Qwen3-Next: 512 experts, moe_intermediate_size 512) but never route.
+ if id(parent) in inside_experts:
+ continue
+ experts = [(n, c) for n, c in parent.named_children() if _is_experts_child(n, c, n_cfg)]
+ counts = {_experts_count(c) for _, c in experts} - {None}
+ if not counts and n_cfg is None:
+ continue
+ widths = counts or {n_cfg}
+ for child_name, child in parent.named_children():
+ if any(child is c for _, c in experts) or "shared" in child_name.lower():
+ continue
+ out = _router_width(child) if counts else _out_features(child)
+ if out is None or out not in widths:
+ continue
+ # No experts container beside it: only a router name counts; `gate_proj` is excluded here
+ # because a dense MLP's gate_proj can be as wide as the expert count.
+ if not counts and child_name.lower() not in (
+ "gate",
+ "router",
+ "wg",
+ "router_proj",
+ "score",
+ ):
+ continue
+ fqn = f"{parent_name}.{child_name}" if parent_name else child_name
+ found[fqn] = (
+ f"router: {out} outputs = expert count, beside {experts[0][0] if experts else 'config'}"
+ )
+ return found
+
+
+def find_output_heads(model) -> Dict[str, str]:
+ """Output embedding modules, and every Linear sharing its weight storage."""
+ heads = {}
+ out = None
+ try:
+ out = model.get_output_embeddings()
+ except Exception:
+ out = None
+ ptrs = set()
+ if out is not None and isinstance(getattr(out, "weight", None), torch.Tensor):
+ try:
+ ptrs.add(out.weight.data_ptr())
+ except Exception:
+ pass
+ # No output embedding (sequence classification, reward / value heads): the head is every Linear outside
+ # the base model (`score`, `classifier`).
+ base = getattr(model, "base_model", None) if out is None else None
+ outside = set()
+ if isinstance(base, torch.nn.Module) and base is not model:
+ inside = {id(m) for m in base.modules()}
+ outside = {id(m) for m in model.modules() if id(m) not in inside}
+ for name, module in model.named_modules():
+ if out is not None and module is out:
+ heads[name] = "output head (get_output_embeddings)"
+ continue
+ if id(module) in outside and isinstance(module, torch.nn.Linear):
+ heads[name] = "task head outside the base model"
+ continue
+ w = getattr(module, "weight", None)
+ if isinstance(module, torch.nn.Linear) and isinstance(w, torch.Tensor) and ptrs:
+ try:
+ if not w.is_meta and w.data_ptr() in ptrs:
+ heads[name] = "weight tied to the output head"
+ except Exception:
+ pass
+ return heads
+
+
+def find_embeddings(model) -> Dict[str, str]:
+ emb = {}
+ inp = None
+ try:
+ inp = model.get_input_embeddings()
+ except Exception:
+ inp = None
+ for name, module in model.named_modules():
+ if isinstance(module, torch.nn.Embedding) or (inp is not None and module is inp):
+ emb[name] = "embedding"
+ return emb
+
+
+_NON_DECODER_PARTS = (
+ "vision",
+ "visual",
+ "image",
+ "audio",
+ "speech",
+ "encoder",
+ "tower",
+ "projector",
+)
+
+
+def _is_block_stack(module) -> bool:
+ """A non-empty ModuleList whose entries each hold a Linear inside them (not a list of bare Linears such as
+ Gemma3n's `altup_projections`). Hybrid stacks (Jamba, Zamba) mix block classes, which is fine."""
+ if not isinstance(module, torch.nn.ModuleList) or len(module) < 1:
+ return False
+ return all(
+ any(isinstance(m, torch.nn.Linear) for m in block.modules() if m is not block)
+ for block in module
+ )
+
+
+def _language_roots(model):
+ """(fqn, module) of the language decoder: the innermost of HF `get_decoder()` (when it names a proper
+ submodule) and the nearest ancestor of the input embedding that holds a block stack; else the model."""
+ modules = dict(model.named_modules())
+ roots = []
+ get_decoder = getattr(model, "get_decoder", None)
+ try:
+ decoder = get_decoder() if callable(get_decoder) else None
+ except Exception:
+ decoder = None
+ if isinstance(decoder, torch.nn.Module) and decoder is not model:
+ roots += [(n, m) for n, m in modules.items() if m is decoder][:1]
+ try:
+ inp = model.get_input_embeddings()
+ except Exception:
+ inp = None
+ emb = next((n for n, m in modules.items() if inp is not None and m is inp), None)
+ if emb:
+ parts = emb.split(".")
+ for i in range(len(parts) - 1, -1, -1):
+ name = ".".join(parts[:i])
+ if any(_is_block_stack(m) for m in modules[name].modules()):
+ roots.append((name, modules[name]))
+ break
+ # Keep a root only when no other candidate lies inside it (Idefics3: get_decoder() is the whole
+ # multimodal model, the embedding's ancestor is its text model). Unrelated roots: get_decoder() wins
+ # (T5: the shared embedding sits above both stacks).
+ roots = [
+ (n, m)
+ for n, m in roots
+ if not any(o != n and (not n or o.startswith(n + ".")) for o, _ in roots)
+ ]
+ return roots[0] if roots else ("", model)
+
+
+def decoder_layers(model):
+ """(fqn prefix of the decoder layer list, number of layers): the longest ModuleList of blocks that each
+ hold Linear layers inside the language decoder (the stack beside the input embedding, or HF
+ `get_decoder()`), ignoring experts lists and any list nested inside another such list (an MoE model can
+ have more experts per block than blocks). Searching the whole model, vision / audio / encoder stacks
+ lose to any other stack; an encoder-only model (BERT) gets its single stack. One block is a stack.
+ (None, 0) when there is none."""
+ root, decoder = _language_roots(model)
+ candidates = []
+ for name, module in decoder.named_modules():
+ name = f"{root}.{name}" if root and name else (root or name)
+ if not _is_block_stack(module):
+ continue
+ if any("expert" in part.lower() for part in name.split(".")):
+ continue
+ candidates.append((name, len(module)))
+ # A list inside a block of another candidate list (`layers.0.<...>`) is part of that block, not the stack.
+ outer = [
+ (name, n)
+ for name, n in candidates
+ if not any(o != name and o and name.startswith(o + ".") for o, _ in candidates)
+ ]
+ if not root:
+ language = [
+ (name, n)
+ for name, n in outer
+ if not any(p.startswith(_NON_DECODER_PARTS) for p in name.lower().split("."))
+ ]
+ outer = language or outer
+ best = (None, 0)
+ for name, n in outer:
+ # A tie goes to the shallower list.
+ if n > best[1] or (
+ n == best[1] and best[0] is not None and name.count(".") < best[0].count(".")
+ ):
+ best = (name, n)
+ return best
+
+
+def high_precision_layers(model, policy: HPLayerPolicy) -> Dict[str, str]:
+ """{fqn: reason} of modules the policy keeps in high precision. A prefix entry (decoder layer) covers
+ every module under it."""
+ keep = {}
+ if policy.output_head:
+ keep.update(find_output_heads(model))
+ if policy.embeddings:
+ keep.update(find_embeddings(model))
+ if policy.routers:
+ keep.update(find_routers(model))
+ if policy.first_n or policy.last_n or policy.last_frac:
+ prefix, n = decoder_layers(model)
+ if prefix is not None and n:
+ import math
+ last = max(policy.last_n, math.ceil(policy.last_frac * n) if policy.last_frac else 0)
+ for i in range(n):
+ if i < policy.first_n:
+ keep[f"{prefix}.{i}"] = f"first {policy.first_n} decoder layers"
+ elif i >= n - last:
+ keep[f"{prefix}.{i}"] = f"last {last} of {n} decoder layers"
+ if policy.shared_experts:
+ for name, _ in model.named_modules():
+ if "shared_expert" in name.rsplit(".", 1)[-1]:
+ keep[name] = "shared expert"
+ return keep
+
+
+def hp_filter(model, policy: HPLayerPolicy):
+ """Predicate (module, fqn) -> True when the module (or a block containing it) stays in high precision."""
+ keep = high_precision_layers(model, policy)
+ exact = set(keep)
+
+ def is_hp(module, fqn: str) -> bool:
+ if fqn in exact:
+ return True
+ parts = fqn.split(".")
+ return any(".".join(parts[:i]) in exact for i in range(1, len(parts)))
+
+ is_hp.keep = keep
+ return is_hp