Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Add unsloth.precision: one registry for low-precision recipes, refusals and high-precision layers (#16387)
Daniel Han0bfae0173801
diff --git a/tests/test_precision_registry.py b/tests/test_precision_registry.pynew file mode 100644index 00000000000..31dca5933a0--- /dev/null+++ b/tests/test_precision_registry.py@@ -0,0 +1,773 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""unsloth.precision: recipe table, refusals before model mutation, structural high-precision layers,+engagement, and the existing float8_training / qat_scheme paths wired through it (CPU only)."""++from __future__ import annotations++import ast+import importlib.util+import inspect+import warnings++import types++import pytest+import torch++from unsloth import precision as P+from unsloth.models import float8_training as f8t++needs_torchao = pytest.mark.skipif(+ importlib.util.find_spec("torchao") is None, reason = "require torchao"+)+++def _tiny_qwen3(+ layers = 2,+ tie = False,+ dtype = torch.float32,+):+ from transformers import Qwen3Config, Qwen3ForCausalLM++ torch.manual_seed(0)+ config = Qwen3Config(+ vocab_size = 256,+ hidden_size = 128,+ intermediate_size = 256,+ num_hidden_layers = layers,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ head_dim = 32,+ tie_word_embeddings = tie,+ )+ return Qwen3ForCausalLM(config).to(dtype)+++def _tiny_qwen3_moe():+ from transformers import Qwen3MoeConfig, Qwen3MoeForCausalLM++ torch.manual_seed(0)+ config = Qwen3MoeConfig(+ vocab_size = 256,+ hidden_size = 128,+ intermediate_size = 256,+ moe_intermediate_size = 64,+ num_hidden_layers = 2,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ head_dim = 32,+ num_experts = 8,+ num_experts_per_tok = 2,+ decoder_sparse_step = 1,+ mlp_only_layers = [],+ )+ return Qwen3MoeForCausalLM(config)+++class _Expert(torch.nn.Module):+ def __init__(self, h, i):+ super().__init__()+ self.up = torch.nn.Linear(h, i, bias = False)+ self.down = torch.nn.Linear(i, h, bias = False)++ def forward(self, x):+ return self.down(torch.relu(self.up(x)))+++class _RemoteMoE(torch.nn.Module):+ """Remote-code style MoE block: the router is called `wg`, which no name rule matches."""++ def __init__(+ self,+ h = 2048,+ e = 64,+ ):+ super().__init__()+ self.wg = torch.nn.Linear(h, e, bias = False)+ self.experts = torch.nn.ModuleList([_Expert(h, 32) for _ in range(e)])+ self.shared_experts = _Expert(h, 32)+++class _RemoteModel(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.layers = torch.nn.ModuleList([_RemoteMoE() for _ in range(2)])+ self.lm_head = torch.nn.Linear(2048, 64, bias = False)++ def get_output_embeddings(self):+ return self.lm_head+++# ---- the table ----------------------------------------------------------------------------------------+++def test_float8_names_and_gates_unchanged():+ # The literals main had before the registry; byte-identical behaviour for the shipped recipe.+ assert f8t.FLOAT8_TRAINING_RECIPES == ("tensorwise",)+ assert f8t._MEASURED_FASTER == {"tensorwise": ((12, 0), (12, 1))}+++def test_every_recipe_is_well_formed():+ gpu_classes = {"sm75", "sm80", "sm89", "sm90", "sm100", "sm120", "rocm"}+ for name, r in P.RECIPES.items():+ assert r.kind in ("dtype", "float8_training", "qat", "frozen_base"), name+ assert r.status in ("shipped", "planned", "refused"), name+ assert set(r.gates) <= gpu_classes, name+ for g in r.gates.values():+ assert g.status in ("named", "explicit", "refused", "unprobed"), name+ assert g.status != "refused" or g.reason, name+ assert r.status != "refused" or r.refusal, name+ assert set(r.input_dtypes) <= {"bf16", "fp16", "fp32"}, name+ assert r.grad_scaler in ("none", "dynamic", "per_dtype"), name+ if r.alias_of:+ assert r.alias_of in P.RECIPES, name+ for short, full in P.FLOAT8_TRAINING_NAMES.items():+ assert P.RECIPES[full].kind == "float8_training", short+++def test_paper_recipes_encoded():+ mx = P.get_recipe("mxfp8")+ for gemm in (+ mx.fwd,+ mx.dgrad,+ mx.wgrad,+ ): # arXiv 2506.08027: e4m3 everywhere, 1x32, e8m0, round up+ for op in (gemm.a, gemm.b):+ assert (op.dtype, op.granularity, op.scale_dtype, op.rounding) == (+ "e4m3",+ "block1x32",+ "e8m0",+ "rceil",+ )+ nv = P.get_recipe("nvfp4") # arXiv 2509.25149+ assert nv.fwd.b.granularity == "block16x16" and nv.dgrad.b.granularity == "block16x16"+ assert nv.fwd.a.rounding == "rn" and nv.dgrad.a.rounding == "sr" and nv.wgrad.a.rounding == "sr"+ assert nv.wgrad.a.rht and nv.wgrad.b.rht and not nv.fwd.a.rht and not nv.dgrad.a.rht+ assert all(op.scale_dtype == "e4m3" for g in (nv.fwd, nv.dgrad, nv.wgrad) for op in (g.a, g.b))+ assert nv.hp_layers.last_frac == pytest.approx(0.15)+ tw = P.get_recipe("fp8_tensorwise")+ assert tw.fwd.a.dtype == "e4m3" and tw.dgrad.a.dtype == "e5m2"+ assert P.get_recipe("fp8_rowwise_gw_hp").wgrad.low_precision is False+++def test_qat_schemes_match_prepare_model_for_qat():+ # Every string scheme _prepare_model_for_qat branches on has a registry row, and no row is stale.+ from unsloth.models import _utils++ tree = ast.parse(inspect.getsource(_utils._prepare_model_for_qat))+ names = set()+ for node in ast.walk(tree):+ if (+ isinstance(node, ast.Compare)+ and isinstance(node.left, ast.Name)+ and node.left.id == "qat_scheme"+ ):+ for c in node.comparators:+ if isinstance(c, ast.Constant) and isinstance(c.value, str):+ names.add(c.value)+ assert names, "no scheme comparisons found"+ rows = {n[len("qat:") :] for n in P.recipes_of_kind("qat")}+ assert names == rows+++# ---- refusals -----------------------------------------------------------------------------------------+++def test_resolve_shipped_values_unchanged():+ assert f8t.resolve_float8_training(None) is None+ assert f8t.resolve_float8_training(False) is None+ assert f8t.resolve_float8_training(True) == "tensorwise"+ assert f8t.resolve_float8_training("TensorWise") == "tensorwise"+ assert f8t.resolve_float8_training(True, dtype = torch.float16) == "tensorwise"+ with pytest.raises(ValueError, match = "slower than bfloat16"):+ f8t.resolve_float8_training("rowwise")+ with pytest.raises(ValueError, match = "is not supported. Use True or one of"):+ f8t.resolve_float8_training("int3")+++def test_refuse_mxfp6_training():+ with pytest.raises(P.PrecisionRefused, match = "MXFP6 has no training kernel"):+ f8t.resolve_float8_training("mxfp6")++[email protected]("name", ["mxfp8", "nvfp4", "blockwise"])+def test_refuse_fp16_for_formats_whose_quantizer_rejects_it(name):+ with pytest.raises(P.PrecisionRefused, match = "float16|fp16"):+ f8t.resolve_float8_training(name, dtype = torch.float16)+++def test_refuse_blockwise_off_sm90(monkeypatch):+ monkeypatch.setattr(f8t._precision, "current_gpu_class", lambda device = None: "sm120")+ with pytest.raises(P.PrecisionRefused, match = "SM90 only"):+ f8t.resolve_float8_training("blockwise")++[email protected]("name", ["mxfp8", "nvfp4"])+def test_planned_formats_refuse_with_their_gates(name):+ with pytest.raises(P.PrecisionRefused, match = "not available in Unsloth yet"):+ f8t.resolve_float8_training(name, dtype = torch.bfloat16)+++def test_gw_hp_refused_by_name_with_the_config_hint():+ with pytest.raises(ValueError, match = "from_recipe_name\\('rowwise_with_gw_hp'\\)"):+ f8t.resolve_float8_training("rowwise_with_gw_hp")+++def test_refusal_happens_before_any_load():+ # The loader resolves float8_training before it looks the (nonexistent) repo up or builds anything.+ from unsloth.models import loader+ with pytest.raises(P.PrecisionRefused, match = "MXFP6"):+ loader.FastModel.from_pretrained(+ model_name = "unsloth/does-not-exist", full_finetuning = True, float8_training = "mxfp6"+ )+ with pytest.raises(P.PrecisionRefused, match = "float16"):+ loader.FastModel.from_pretrained(+ model_name = "unsloth/does-not-exist",+ full_finetuning = True,+ float8_training = "mxfp8",+ dtype = torch.float16,+ )+++def test_gate_evidence_names_no_local_paths():+ # Evidence is printed in refusals; a scratch path (outputs/, plans/) means nothing to a user.+ for name, r in P.RECIPES.items():+ for gpu, gate in r.gates.items():+ assert not any(p in gate.evidence for p in ("outputs/", "plans/", ".md")), (+ name,+ gpu,+ gate.evidence,+ )+++def test_check_recipe_gate_table():+ with pytest.raises(P.PrecisionRefused, match = "fp8 tensor cores"):+ P.check_recipe("fp8_tensorwise", gpu = "sm75")+ with pytest.raises(P.PrecisionRefused, match = "NaN"):+ P.check_recipe("fp8_tensorwise", gpu = "sm89")+ assert P.check_recipe("fp8_tensorwise", gpu = "sm120").status == "named"+ assert P.check_recipe("fp8_rowwise", gpu = "sm89").status == "explicit"+ with pytest.raises(P.PrecisionRefused, match = "block-scaled"):+ P.check_recipe("mxfp8", gpu = "sm89", allow_planned = True)+ with pytest.raises(P.PrecisionRefused, match = "no bfloat16"):+ P.check_recipe("bf16", gpu = "sm75")+ assert P.check_recipe("fp16", gpu = "sm75").status == "named"+ with pytest.raises(P.PrecisionRefused, match = "training modes"):+ P.check_recipe("fp8_tensorwise", mode = "lora")+++def test_gate_table_agrees_with_float8_hardware_check():+ # float8_training._check_hardware refuses tensorwise below sm90 and everything below 8.9.+ for gpu in ("sm75", "sm80", "sm89"):+ assert P.get_recipe("fp8_tensorwise").gate(gpu).status == "refused"+ for gpu in ("sm75", "sm80"):+ assert P.get_recipe("fp8_rowwise").gate(gpu).status == "refused"+++def test_gpu_class_and_scaler_kind():+ assert (+ P.gpu_class((7, 5)) == "sm75"+ and P.gpu_class((8, 6)) == "sm80"+ and P.gpu_class((8, 9)) == "sm89"+ )+ assert (+ P.gpu_class((9, 0)) == "sm90"+ and P.gpu_class((10, 0)) == "sm100"+ and P.gpu_class((12, 0)) == "sm120"+ )+ assert P.gpu_class(None) is None and P.gpu_class((9, 0), hip = True) == "rocm"+ assert P.grad_scaler_kind("fp16", torch.float16) == "GradScaler"+ assert P.grad_scaler_kind("fp16", torch.float16, fsdp2 = True) == "ShardedGradScaler"+ assert P.grad_scaler_kind("bf16", torch.bfloat16) == "none"+ assert P.grad_scaler_kind("fp8_tensorwise", torch.float16) == "GradScaler"+ assert P.grad_scaler_kind("fp8_tensorwise", torch.bfloat16) == "none"+ assert P.grad_scaler_kind("mxfp8", "bfloat16") == "none"+++# ---- structural high-precision layers ------------------------------------------------------------------+++def test_routers_found_by_structure():+ moe = _tiny_qwen3_moe()+ assert set(P.find_routers(moe)) == {"model.layers.0.mlp.gate", "model.layers.1.mlp.gate"}+ remote = _RemoteModel()+ assert set(P.find_routers(remote)) == {"layers.0.wg", "layers.1.wg"}+ assert P.find_routers(_tiny_qwen3()) == {}+++def _tiny_moe_expert_width_collision():+ # Qwen3-Next-80B-A3B has num_experts == moe_intermediate_size == shared_expert_intermediate_size == 512:+ # an expert / shared expert / dense MLP gate_proj is as wide as the router but is not one.+ from transformers import Qwen2MoeConfig, Qwen2MoeForCausalLM++ torch.manual_seed(0)+ config = Qwen2MoeConfig(+ vocab_size = 256,+ hidden_size = 128,+ intermediate_size = 16,+ moe_intermediate_size = 16,+ shared_expert_intermediate_size = 16,+ num_hidden_layers = 2,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ num_experts = 16,+ num_experts_per_tok = 2,+ decoder_sparse_step = 1,+ mlp_only_layers = [0],+ )+ return Qwen2MoeForCausalLM(config)+++def test_routers_ignore_mlps_as_wide_as_the_expert_count():+ m = _tiny_moe_expert_width_collision()+ assert set(P.find_routers(m)) == {"model.layers.1.mlp.gate"}+++def test_hp_layers_policy():+ m = _tiny_qwen3(layers = 8, tie = True)+ keep = P.high_precision_layers(m, P.get_recipe("nvfp4").hp_layers)+ assert "lm_head" in keep and "model.embed_tokens" in keep+ assert {k for k in keep if k.startswith("model.layers.")} == {+ "model.layers.6",+ "model.layers.7",+ } # ceil(0.15 * 8)+ is_hp = P.hp_filter(m, P.HPLayerPolicy(first_n = 1, last_n = 1))+ assert is_hp(None, "model.layers.0.mlp.up_proj") and is_hp(+ None, "model.layers.7.self_attn.q_proj"+ )+ assert not is_hp(None, "model.layers.3.mlp.up_proj")+ assert P.decoder_layers(m) == ("model.layers", 8)+++def test_num_experts_nested_config():+ assert P.num_experts_of({"text_config": {"num_local_experts": 16}}) == 16+ assert P.num_experts_of({"hidden_size": 8}) is None+++@needs_torchao+def test_float8_filter_skips_structural_router():+ # Failing on main: the name rules (gate / router / score / classifier) convert a router called `wg`.+ remote = _RemoteModel()+ fn = f8t.float8_training_filter(remote, "tensorwise", size_filter = False)+ assert not fn(remote.layers[0].wg, "layers.0.wg")+ assert fn(remote.layers[0].experts[0].up, "layers.0.experts.0.up")+ assert not fn(remote.lm_head, "lm_head")+++@needs_torchao[email protected]("build", [_tiny_qwen3, _tiny_qwen3_moe, _tiny_moe_expert_width_collision])+def test_float8_filter_unchanged_on_standard_models(build, monkeypatch):+ m = build()+ for size_filter in (True, False):+ new = f8t.float8_training_filter(m, "tensorwise", size_filter = size_filter)+ new_set = {n for n, mod in m.named_modules() if new(mod, n)}+ with monkeypatch.context() as mp:+ mp.setattr(f8t, "_find_routers", lambda model: {})+ old = f8t.float8_training_filter(m, "tensorwise", size_filter = size_filter)+ old_set = {n for n, mod in m.named_modules() if old(mod, n)}+ assert new_set == old_set+++# ---- engagement ------------------------------------------------------------------------------------------+++def _emulated_config(recipe = "tensorwise"):+ from torchao.float8 import Float8LinearConfig++ config = Float8LinearConfig.from_recipe_name(recipe)+ object.__setattr__(config, "emulate", True)+ return config+++@needs_torchao+def test_engagement_recorded_and_strict(monkeypatch):+ m = _tiny_qwen3()+ converted = f8t.convert_model_to_float8_training(m, _emulated_config(), size_filter = False)+ assert converted and P.engaged(m)["float8_training:Float8LinearConfig(tensorwise)"][+ "modules"+ ] == len(converted)++ tiny = _tiny_qwen3() # 128 / 256 wide: the size filter keeps every layer out+ monkeypatch.delenv("UNSLOTH_STRICT_ENGAGE", raising = False)+ with warnings.catch_warnings(record = True) as w:+ warnings.simplefilter("always")+ assert f8t.convert_model_to_float8_training(tiny, _emulated_config()) == []+ assert any("no eligible Linear" in str(x.message) for x in w)+ assert P.engaged(tiny)["float8_training:Float8LinearConfig(tensorwise)"]["modules"] == 0+ monkeypatch.setenv("UNSLOTH_STRICT_ENGAGE", "1")+ with pytest.raises(P.EngagementError, match = "engaged on 0 modules"):+ f8t.convert_model_to_float8_training(_tiny_qwen3(), _emulated_config())+++@needs_torchao+def test_qat_engagement_recorded():+ from unsloth.models._utils import _prepare_model_for_qat++ m = _prepare_model_for_qat(_tiny_qwen3(dtype = torch.bfloat16), "int4")+ rec = P.engaged(m)["qat:int4"]+ assert rec["modules"] == sum(1 for x in m.modules() if "FakeQuantized" in type(x).__name__) > 0+++def test_require_and_counter(monkeypatch):+ monkeypatch.delenv("UNSLOTH_STRICT_ENGAGE", raising = False)+ assert P.require("x", 3) and not P.require("x", 0)+ monkeypatch.setenv("UNSLOTH_STRICT_ENGAGE", "1")+ with pytest.raises(P.EngagementError):+ P.require("x", 0)+ with P.count_lowp_gemms() as c:+ torch.ones(4, 4) @ torch.ones(4, 4)+ assert all(v == 0 for v in c.values())+ a = torch.ones(16, 16).to(torch.float8_e4m3fn)+ b = torch.ones(16, 16).to(torch.float8_e4m3fn).t()+ one = torch.ones((), dtype = torch.float32)+ try:+ with P.count_lowp_gemms() as c:+ torch._scaled_mm(a, b, scale_a = one, scale_b = one, out_dtype = torch.bfloat16)+ except (RuntimeError, NotImplementedError):+ pytest.skip("no CPU _scaled_mm in this torch")+ assert c["_scaled_mm"] == 1+++def test_counter_runs_fullgraph_compiled_calls():+ # Failing before: a plain TorchDispatchMode makes Dynamo refuse fullgraph frames ("non-infra torch dispatch+ # mode present"), so the engagement check crashed around compiled models.+ a = torch.ones(16, 16).to(torch.float8_e4m3fn)+ b = torch.ones(16, 16).to(torch.float8_e4m3fn).t()+ one = torch.ones((), dtype = torch.float32)++ def f(a, b):+ return torch._scaled_mm(a, b, scale_a = one, scale_b = one, out_dtype = torch.bfloat16)++ try:+ expected = f(a, b)+ except (RuntimeError, NotImplementedError):+ pytest.skip("no CPU _scaled_mm in this torch")+ torch._dynamo.reset()+ compiled = torch.compile(f, backend = "aot_eager", fullgraph = True)+ compiled(a, b)+ with P.count_lowp_gemms() as c:+ out = compiled(a, b)+ assert torch.equal(out, expected)+ assert c["_scaled_mm"] == 1+++def test_counter_warns_when_torch_cannot_keep_compiled_calls(monkeypatch):+ # Failing before: torch < 2.9 has no ignore_compile_internals hook, so Dynamo ran compiled callables+ # eagerly inside the counter without saying so.+ from torch.utils._python_dispatch import TorchDispatchMode+ from unsloth.precision import engage++ monkeypatch.delattr(TorchDispatchMode, "ignore_compile_internals", raising = False)+ monkeypatch.setattr(engage, "_EAGER_UNDER_COUNTER_WARNED", False)+ with pytest.warns(UserWarning, match = "eagerly"):+ with P.count_lowp_gemms() as c:+ torch.ones(4, 4) @ torch.ones(4, 4)+ assert all(v == 0 for v in c.values())+++def test_decoder_layers_is_the_block_stack_not_an_experts_list():+ # Failing before: the longest ModuleList won, so a model with more experts than layers returned+ # model.layers.0.mlp.experts and first_n / last_n kept experts in high precision instead of blocks.+ from transformers import Qwen2MoeConfig, Qwen2MoeForCausalLM++ config = Qwen2MoeConfig(+ vocab_size = 256,+ hidden_size = 64,+ intermediate_size = 128,+ moe_intermediate_size = 32,+ shared_expert_intermediate_size = 64,+ num_hidden_layers = 4,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ num_experts = 12,+ num_experts_per_tok = 2,+ )+ with torch.device("meta"):+ m = Qwen2MoeForCausalLM(config)+ assert P.decoder_layers(m) == ("model.layers", 4)+ is_hp = P.hp_filter(m, P.HPLayerPolicy(first_n = 1, last_n = 1))+ assert is_hp(None, "model.layers.0.self_attn.q_proj")+ assert is_hp(None, "model.layers.3.mlp.experts.5.up_proj")+ assert not is_hp(None, "model.layers.1.self_attn.q_proj")+ assert not is_hp(None, "model.layers.1.mlp.experts.0.up_proj")+++def test_decoder_layers_is_the_language_decoder_not_a_longer_encoder():+ # Failing before: the longest outer list won anywhere, so a vision tower or speech encoder deeper than+ # the language decoder got the first / last block protection.+ from transformers import (+ CLIPVisionConfig,+ LlamaConfig,+ LlavaConfig,+ LlavaForConditionalGeneration,+ WhisperConfig,+ WhisperForConditionalGeneration,+ )++ text = LlamaConfig(+ vocab_size = 256,+ hidden_size = 64,+ intermediate_size = 128,+ num_hidden_layers = 2,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ )+ vision = CLIPVisionConfig(+ hidden_size = 32,+ intermediate_size = 64,+ num_hidden_layers = 4,+ num_attention_heads = 4,+ image_size = 32,+ patch_size = 16,+ )+ with torch.device("meta"):+ llava = LlavaForConditionalGeneration(LlavaConfig(vision_config = vision, text_config = text))+ name, n = P.decoder_layers(llava)+ assert n == 2 and "vision" not in name and name.endswith("layers"), (name, n)+ is_hp = P.hp_filter(llava, P.HPLayerPolicy(first_n = 1, last_n = 1))+ assert is_hp(None, f"{name}.0.self_attn.q_proj") and is_hp(None, f"{name}.1.mlp.down_proj")++ whisper = WhisperConfig(+ vocab_size = 256,+ d_model = 32,+ encoder_layers = 4,+ decoder_layers = 2,+ encoder_attention_heads = 4,+ decoder_attention_heads = 4,+ encoder_ffn_dim = 64,+ decoder_ffn_dim = 64,+ pad_token_id = 0,+ bos_token_id = 1,+ eos_token_id = 2,+ decoder_start_token_id = 1,+ )+ with torch.device("meta"):+ w = WhisperForConditionalGeneration(whisper)+ assert P.decoder_layers(w) == ("model.decoder.layers", 2)+++def test_decoder_layers_accepts_a_hybrid_stack_of_mixed_block_classes():+ # Failing before: a stack had to be one block class, so Jamba (Mamba + attention blocks) returned+ # (None, 0) and first_n / last_n protected nothing.+ from transformers import JambaConfig, JambaForCausalLM++ config = JambaConfig(+ vocab_size = 256,+ hidden_size = 64,+ intermediate_size = 128,+ num_hidden_layers = 4,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ num_experts = 1,+ attn_layer_period = 2,+ attn_layer_offset = 1,+ mamba_d_state = 8,+ mamba_d_conv = 4,+ mamba_expand = 2,+ use_mamba_kernels = False,+ )+ with torch.device("meta"):+ m = JambaForCausalLM(config)+ assert len({type(b) for b in m.model.layers}) == 2+ assert P.decoder_layers(m) == ("model.layers", 4)+ is_hp = P.hp_filter(m, P.HPLayerPolicy(first_n = 1, last_n = 1))+ assert is_hp(None, "model.layers.0.mamba.in_proj") and is_hp(+ None, "model.layers.3.self_attn.q_proj"+ )+ assert not is_hp(None, "model.layers.1.self_attn.q_proj")+++def test_counter_counts_opaque_custom_ops_once_at_their_boundary(monkeypatch):+ # Failing before: only torch.ops.aten targets were resolved, so a low-precision GEMM inside an opaque+ # custom op (unsloth::nvfp4_qat_linear, fbgemm::f8f8bf16_rowwise) counted zero.+ from unsloth.precision import engage++ a = torch.ones(16, 16).to(torch.float8_e4m3fn)+ b = torch.ones(16, 16).to(torch.float8_e4m3fn).t()+ one = torch.ones((), dtype = torch.float32)+ try:+ torch._scaled_mm(a, b, scale_a = one, scale_b = one, out_dtype = torch.bfloat16)+ except (RuntimeError, NotImplementedError):+ pytest.skip("no CPU _scaled_mm in this torch")++ name = "unsloth_precision_test::opaque_fp8_mm"+ if not hasattr(torch.ops.unsloth_precision_test, "opaque_fp8_mm"):++ @torch.library.custom_op(name, mutates_args = ())+ def _opaque(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:+ one = torch.ones((), dtype = torch.float32)+ return torch._scaled_mm(a, b, scale_a = one, scale_b = one, out_dtype = torch.bfloat16)++ monkeypatch.setattr(engage, "LOWP_GEMM_CUSTOM_OPS", engage.LOWP_GEMM_CUSTOM_OPS + (name,))+ with P.count_lowp_gemms() as c:+ torch.ops.unsloth_precision_test.opaque_fp8_mm(a, b)+ assert c[name] == 1 and c["_scaled_mm"] == 0 # the boundary, not the hidden inner call+ assert "fbgemm::f8f8bf16_rowwise" in c # absent namespace: listed, never resolved+++@needs_torchao+def test_counter_counts_the_real_nvfp4_qat_linear_op():+ from unsloth.models._utils import _nvfp4_qat_linear_op++ try:+ op = _nvfp4_qat_linear_op()+ except Exception as e:+ pytest.skip(f"torchao NVFP4 internals unavailable: {e!r}")+ x = torch.empty(16, 64, device = "meta", dtype = torch.bfloat16)+ w = torch.empty(32, 64, device = "meta", dtype = torch.bfloat16)+ with P.count_lowp_gemms() as c:+ op(x, w, None, False, False, False, False, None)+ assert c["unsloth::nvfp4_qat_linear"] == 1+++def test_counter_hears_eager_fp8_triton_launchers_and_not_the_fp16_gemv():+ # Failing before: eager fp8 block launchers run Triton without dispatching their op (counted 0), and+ # fp8_block_gemv, which upcasts the weight before its dot, was counted as a low-precision GEMM.+ from unsloth.precision import engage++ assert "unsloth::fp8_block_gemv" not in engage.LOWP_GEMM_CUSTOM_OPS+ with P.count_lowp_gemms() as c:+ engage.release_lowp_gemm(engage.note_lowp_gemm("unsloth::fp8_block_small_m_linear"))+ assert c["unsloth::fp8_block_small_m_linear"] == 1+ engage.note_lowp_gemm("unsloth::fp8_block_small_m_linear") # no counter open: a no-op+ assert c["unsloth::fp8_block_small_m_linear"] == 1 and not engage._active()++ fp8 = pytest.importorskip("unsloth.kernels.fp8")+ names = fp8._EAGER_COUNTED_LOWP_GEMMS+ assert "fp8_block_gemv" not in names+ for name in ("fp8_block_matmul_triton", "fp8_block_small_m_linear", "fp8_block_matmul_torchao"):+ assert name in names and f"unsloth::{name}" in engage.LOWP_GEMM_CUSTOM_OPS+++def test_mxfp8_qat_registry_admits_only_bfloat16_like_its_backend():+ # Failing before: the qat:mxfp8 row allowed fp32 while _prepare_model_for_qat quantizes bf16 layers only.+ assert P.check_recipe("qat:mxfp8", gpu = "sm100", input_dtype = torch.bfloat16, mode = "full")+ with pytest.raises(ValueError):+ P.check_recipe("qat:mxfp8", gpu = "sm100", input_dtype = torch.float32, mode = "full")+++def test_nested_counters_close_by_identity_and_delegated_gemms_count_once():+ # Failing before: list.remove matched the outer (equal, all-zero) dict when the inner scope closed, so+ # the outer cleanup raised; and a launcher that delegates to another counted launcher counted twice.+ from unsloth.precision import engage++ with P.count_lowp_gemms() as outer:+ with P.count_lowp_gemms() as inner:+ pass+ assert any(a is outer for a in engage._active()) and not any(+ a is inner for a in engage._active()+ )+ entered = engage.note_lowp_gemm("unsloth::fp8_block_matmul_torchao")+ try: # the delegated Triton launch inside the torchao one+ engage.release_lowp_gemm(engage.note_lowp_gemm("unsloth::fp8_block_matmul_triton"))+ finally:+ engage.release_lowp_gemm(entered)+ assert not engage._active()+ assert outer["unsloth::fp8_block_matmul_torchao"] == 1+ assert outer.get("unsloth::fp8_block_matmul_triton", 0) == 0+ assert inner.get("unsloth::fp8_block_matmul_torchao", 0) == 0+++def test_float8_filter_keeps_a_wrapped_router_and_its_inner_linear_in_high_precision():+ # Failing before: find_routers returns the gating wrapper `wg`, and only that exact fqn was excluded, so+ # its inner `wg.proj` Linear was converted to fp8 and could flip expert choices.+ class Gate(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(64, 16, bias = False)++ def forward(self, x):+ return self.proj(x)++ class Moe(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.wg = Gate()+ self.experts = torch.nn.ModuleList(+ torch.nn.Linear(64, 64, bias = False) for _ in range(16)+ )++ class Block(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.moe = Moe()++ class Model(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.config = types.SimpleNamespace(num_experts = 16, num_local_experts = 16)+ self.layers = torch.nn.ModuleList([Block(), Block()])++ m = Model()+ routers = P.find_routers(m)+ assert any(r.endswith("moe.wg") for r in routers), routers+ fn = f8t.float8_training_filter(m, "tensorwise", size_filter = False)+ assert not fn(m.layers[0].moe.wg.proj, "layers.0.moe.wg.proj")+ assert fn(m.layers[0].moe.experts[0], "layers.0.moe.experts.0")+++def test_float8_filter_keeps_a_router_beside_moduledict_experts_in_high_precision():+ # Failing before: only a ModuleList counted as an experts collection, so with 16 experts in a ModuleDict+ # (SwitchTransformers' layout) the wrapped router `wg` was not found and `wg.proj` was converted to fp8.+ class Gate(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(64, 16, bias = False)++ def forward(self, x):+ return self.proj(x)++ class Moe(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.wg = Gate()+ self.experts = torch.nn.ModuleDict(+ {f"expert_{i}": torch.nn.Linear(64, 64, bias = False) for i in range(16)}+ )++ class Model(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.config = types.SimpleNamespace(num_experts = 16)+ self.layers = torch.nn.ModuleList([Moe(), Moe()])++ m = Model()+ assert set(P.find_routers(m)) == {"layers.0.wg", "layers.1.wg"}+ fn = f8t.float8_training_filter(m, "tensorwise", size_filter = False)+ assert not fn(m.layers[0].wg.proj, "layers.0.wg.proj")+ assert fn(m.layers[0].experts.expert_0, "layers.0.experts.expert_0")+++def test_gemm_counters_are_per_thread():+ # Failing before: the active-counter list was process-global, so an eager launch on one thread was+ # counted by a counter open on another.+ import threading++ from unsloth.precision import engage++ seen = {}+ ready, go = threading.Barrier(2), threading.Event()++ def worker(tag, launches):+ with P.count_lowp_gemms() as c:+ ready.wait()+ for _ in range(launches):+ engage.release_lowp_gemm(engage.note_lowp_gemm("unsloth::fp8_block_small_m_linear"))+ go.wait(5)+ seen[tag] = c.get("unsloth::fp8_block_small_m_linear", 0)++ threads = [+ threading.Thread(target = worker, args = ("busy", 1)),+ threading.Thread(target = worker, args = ("idle", 0)),+ ]+ for t in threads:+ t.start()+ import time++ time.sleep(0.2)+ go.set()+ for t in threads:+ t.join()+ assert seen == {"busy": 1, "idle": 0}, seendiff --git a/tests/test_precision_selectors_families.py b/tests/test_precision_selectors_families.pynew file mode 100644index 00000000000..92bbcc28fc2--- /dev/null+++ b/tests/test_precision_selectors_families.py@@ -0,0 +1,767 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""unsloth.precision selectors over every model family transformers ships (tiny configs on the meta device, CPU+only): decoder_layers is the language decoder's block list, find_output_heads / find_embeddings / find_routers+return the expected modules (routers only for MoE families, never an expert / shared expert / dense MLP), and+hp_filter(first_n = 1, last_n = 1) protects exactly the first and last language blocks.++unsloth.precision is loaded standalone (it imports only torch), so this runs in any venv with transformers."""++from __future__ import annotations++import importlib.util+import os+import sys++import pytest+import torch+import transformers++_PKG = os.path.join(+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "unsloth", "precision"+)+++def _load_precision():+ name = "_unsloth_precision_standalone"+ if name in sys.modules:+ return sys.modules[name]+ spec = importlib.util.spec_from_file_location(+ name, os.path.join(_PKG, "__init__.py"), submodule_search_locations = [_PKG]+ )+ module = importlib.util.module_from_spec(spec)+ sys.modules[name] = module+ spec.loader.exec_module(module)+ return module+++P = _load_precision()++TXT = dict(+ vocab_size = 128,+ hidden_size = 64,+ intermediate_size = 128,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ head_dim = 16,+ pad_token_id = 0,+ bos_token_id = 1,+ eos_token_id = 2,+)+MAMBA = dict(mamba_d_state = 8, mamba_d_conv = 4, mamba_expand = 2)+++def _cls(name):+ c = getattr(transformers, name, None)+ if c is None:+ pytest.skip(f"{name} not in transformers {transformers.__version__}")+ return c+++def _build(+ model_cls,+ config_cls,+ meta = True,+ **kwargs,+):+ config = _cls(config_cls)(**kwargs)+ if meta:+ with torch.device("meta"):+ return _cls(model_cls)(config)+ torch.manual_seed(0)+ return _cls(model_cls)(config)+++def _causal(+ arch,+ n = 3,+ **kw,+):+ return lambda: _build(+ f"{arch}ForCausalLM", f"{arch}Config", **{**TXT, "num_hidden_layers": n, **kw}+ )+++def _siglip(n = 5):+ return dict(+ hidden_size = 32,+ intermediate_size = 64,+ num_hidden_layers = n,+ num_attention_heads = 4,+ image_size = 32,+ patch_size = 8,+ )+++def _text(n = 3, **kw):+ return {**TXT, "num_hidden_layers": n, **kw}+++# name -> (builder, decoder list paths (first one present wins), decoder length, router fqn suffixes, heads)+# heads: None = {the module get_output_embeddings returns}; otherwise the exact expected set.+FAMILIES = {+ "llama": (_causal("Llama"), ["model.layers"], 3, (), None),+ "mistral": (_causal("Mistral"), ["model.layers"], 3, (), None),+ "qwen2": (_causal("Qwen2"), ["model.layers"], 3, (), None),+ "qwen3_1layer": (_causal("Qwen3", 1), ["model.layers"], 1, (), None),+ "qwen3_4layer": (_causal("Qwen3", 4), ["model.layers"], 4, (), None),+ "gemma2": (_causal("Gemma2"), ["model.layers"], 3, (), None),+ "gemma3_text": (+ lambda: _build("Gemma3ForCausalLM", "Gemma3TextConfig", **_text()),+ ["model.layers"],+ 3,+ (),+ None,+ ),+ "gemma3_mm": (+ lambda: _build(+ "Gemma3ForConditionalGeneration",+ "Gemma3Config",+ text_config = _text(),+ vision_config = _siglip(),+ mm_tokens_per_image = 4,+ ),+ ["model.language_model.layers", "language_model.model.layers"],+ 3,+ (),+ None,+ ),+ "gemma3n_text": (+ lambda: _build(+ "Gemma3nForCausalLM",+ "Gemma3nTextConfig",+ **_text(+ 4,+ laurel_rank = 8,+ hidden_size_per_layer_input = 8,+ altup_num_inputs = 4,+ num_kv_shared_layers = 0,+ layer_types = ["sliding_attention", "full_attention"] * 2,+ activation_sparsity_pattern = [0.0] * 4,+ ),+ ),+ ["model.layers"],+ 4,+ (),+ None,+ ),+ "phi3": (_causal("Phi3"), ["model.layers"], 3, (), None),+ "granite": (_causal("Granite"), ["model.layers"], 3, (), None),+ "granitemoe": (+ _causal("GraniteMoe", num_local_experts = 4, num_experts_per_tok = 2),+ ["model.layers"],+ 3,+ (".block_sparse_moe.router",),+ None,+ ),+ "granitemoehybrid": (+ _causal(+ "GraniteMoeHybrid",+ 4,+ num_local_experts = 4,+ num_experts_per_tok = 2,+ shared_intermediate_size = 64,+ layer_types = ["mamba", "attention", "mamba", "attention"],+ mamba_n_heads = 8,+ mamba_d_head = 16,+ mamba_n_groups = 1,+ mamba_d_state = 8,+ mamba_d_conv = 4,+ mamba_expand = 2,+ mamba_chunk_size = 16,+ ),+ ["model.layers"],+ 4,+ (".block_sparse_moe.router",),+ None,+ ),+ "qwen2_moe": (+ _causal(+ "Qwen2Moe",+ num_experts = 8,+ num_experts_per_tok = 2,+ moe_intermediate_size = 32,+ shared_expert_intermediate_size = 64,+ ),+ ["model.layers"],+ 3,+ (".mlp.gate",),+ None,+ ),+ "qwen3_moe": (+ _causal("Qwen3Moe", num_experts = 8, num_experts_per_tok = 2, moe_intermediate_size = 32),+ ["model.layers"],+ 3,+ (".mlp.gate",),+ None,+ ),+ "qwen3_next": (+ _causal(+ "Qwen3Next",+ 4,+ num_experts = 8,+ num_experts_per_tok = 2,+ moe_intermediate_size = 32,+ shared_expert_intermediate_size = 64,+ layer_types = ["linear_attention", "full_attention"] * 2,+ linear_num_value_heads = 4,+ linear_num_key_heads = 2,+ linear_key_head_dim = 16,+ linear_value_head_dim = 16,+ linear_conv_kernel_dim = 4,+ ),+ ["model.layers"],+ 4,+ (".mlp.gate",),+ None,+ ),+ "mixtral": (+ _causal("Mixtral", num_local_experts = 4, num_experts_per_tok = 2),+ ["model.layers"],+ 3,+ (".block_sparse_moe.gate", ".mlp.gate"),+ None,+ ),+ "olmoe": (+ _causal("Olmoe", num_experts = 8, num_experts_per_tok = 2),+ ["model.layers"],+ 3,+ (".mlp.gate",),+ None,+ ),+ "gpt_oss": (+ _causal(+ "GptOss",+ num_local_experts = 4,+ num_experts_per_tok = 2,+ layer_types = ["sliding_attention", "full_attention", "sliding_attention"],+ ),+ ["model.layers"],+ 3,+ (".mlp.router",),+ None,+ ),+ "deepseek_v3": (+ _causal(+ "DeepseekV3",+ n_routed_experts = 8,+ num_experts_per_tok = 2,+ n_shared_experts = 1,+ first_k_dense_replace = 1,+ moe_intermediate_size = 32,+ n_group = 1,+ topk_group = 1,+ kv_lora_rank = 16,+ q_lora_rank = 16,+ qk_rope_head_dim = 8,+ qk_nope_head_dim = 8,+ v_head_dim = 16,+ ),+ ["model.layers"],+ 3,+ (".mlp.gate",),+ None,+ ),+ "glm4_moe": (+ _causal(+ "Glm4Moe",+ n_routed_experts = 8,+ num_experts_per_tok = 2,+ n_shared_experts = 1,+ first_k_dense_replace = 1,+ moe_intermediate_size = 32,+ n_group = 1,+ topk_group = 1,+ ),+ ["model.layers"],+ 3,+ (".mlp.gate",),+ None,+ ),+ "llama4_text": (+ lambda: _build(+ "Llama4ForCausalLM",+ "Llama4TextConfig",+ **_text(+ num_local_experts = 4,+ num_experts_per_tok = 1,+ interleave_moe_layer_step = 1,+ intermediate_size_mlp = 128,+ ),+ ),+ ["model.layers"],+ 3,+ (".feed_forward.router",),+ None,+ ),+ "llama4_mm": (+ lambda: _build(+ "Llama4ForConditionalGeneration",+ "Llama4Config",+ text_config = _text(+ num_local_experts = 4,+ num_experts_per_tok = 1,+ interleave_moe_layer_step = 1,+ intermediate_size_mlp = 128,+ ),+ vision_config = dict(+ hidden_size = 32,+ intermediate_size = 64,+ num_hidden_layers = 5,+ num_attention_heads = 4,+ image_size = 32,+ patch_size = 8,+ vision_output_dim = 64,+ projector_input_dim = 64,+ projector_output_dim = 64,+ pixel_shuffle_ratio = 0.5,+ ),+ ),+ ["language_model.model.layers", "model.language_model.layers"],+ 3,+ (".feed_forward.router",),+ None,+ ),+ "jamba": (+ _causal(+ "Jamba",+ 4,+ num_experts = 4,+ num_experts_per_tok = 2,+ expert_layer_period = 2,+ expert_layer_offset = 1,+ attn_layer_period = 2,+ attn_layer_offset = 1,+ use_mamba_kernels = False,+ **MAMBA,+ ),+ ["model.layers"],+ 4,+ (".feed_forward.router",),+ None,+ ),+ "zamba": (+ _causal(+ "Zamba",+ 6,+ attn_layer_period = 2,+ attn_layer_offset = 1,+ mamba_dt_rank = 8,+ n_mamba_heads = 2,+ use_mamba_kernels = False,+ **MAMBA,+ ),+ ["model.layers"],+ 6,+ (),+ None,+ ),+ "zamba2": (+ _causal(+ "Zamba2",+ 4,+ layers_block_type = ["mamba", "hybrid", "mamba", "hybrid"],+ mamba_headdim = 16,+ mamba_ngroups = 1,+ n_mamba_heads = 8,+ num_mem_blocks = 1,+ use_mamba_kernels = False,+ **MAMBA,+ ),+ ["model.layers"],+ 4,+ (),+ None,+ ),+ "falcon_h1": (+ _causal(+ "FalconH1",+ mamba_d_ssm = 128,+ mamba_n_heads = 8,+ mamba_d_head = 16,+ mamba_n_groups = 1,+ mamba_chunk_size = 16,+ **MAMBA,+ ),+ ["model.layers"],+ 3,+ (),+ None,+ ),+ "nemotron_h": (+ _causal(+ "NemotronH",+ hybrid_override_pattern = "M*-",+ mamba_num_heads = 8,+ mamba_head_dim = 16,+ n_groups = 1,+ ssm_state_size = 8,+ conv_kernel = 4,+ expand = 2,+ chunk_size = 16,+ ),+ ["model.layers", "backbone.layers"],+ 3,+ (),+ None,+ ),+ "nemotron_h_moe": (+ _causal(+ "NemotronH",+ 4,+ layers_block_type = ["linear_attention", "moe", "full_attention", "moe"],+ n_routed_experts = 8,+ num_experts_per_tok = 2,+ moe_intermediate_size = 32,+ moe_shared_expert_intermediate_size = 64,+ mamba_num_heads = 8,+ mamba_head_dim = 16,+ n_groups = 1,+ ssm_state_size = 8,+ conv_kernel = 4,+ expand = 2,+ chunk_size = 16,+ ),+ ["model.layers"],+ 4,+ (".mixer.gate",),+ None,+ ),+ "mamba": (+ lambda: _build(+ "MambaForCausalLM",+ "MambaConfig",+ vocab_size = 128,+ hidden_size = 64,+ state_size = 8,+ num_hidden_layers = 3,+ use_mambapy = False,+ ),+ ["backbone.layers"],+ 3,+ (),+ None,+ ),+ "llava": (+ lambda: _build(+ "LlavaForConditionalGeneration",+ "LlavaConfig",+ text_config = dict(model_type = "llama", **_text()),+ vision_config = dict(model_type = "clip_vision_model", **_siglip()),+ ),+ ["model.language_model.layers", "language_model.model.layers"],+ 3,+ (),+ None,+ ),+ "qwen2_vl": (+ lambda: _build(+ "Qwen2VLForConditionalGeneration",+ "Qwen2VLConfig",+ text_config = _text(rope_scaling = {"type": "mrope", "mrope_section": [2, 3, 3]}),+ vision_config = dict(depth = 5, embed_dim = 32, hidden_size = 64, num_heads = 4, mlp_ratio = 2),+ **_text(rope_scaling = {"type": "mrope", "mrope_section": [2, 3, 3]}),+ ),+ ["model.language_model.layers", "model.layers"],+ 3,+ (),+ None,+ ),+ "qwen2_5_vl": (+ lambda: _build(+ "Qwen2_5_VLForConditionalGeneration",+ "Qwen2_5_VLConfig",+ text_config = _text(rope_scaling = {"type": "mrope", "mrope_section": [2, 3, 3]}),+ vision_config = dict(+ depth = 5,+ hidden_size = 32,+ intermediate_size = 64,+ out_hidden_size = 64,+ num_heads = 4,+ fullatt_block_indexes = [4],+ ),+ **_text(rope_scaling = {"type": "mrope", "mrope_section": [2, 3, 3]}),+ ),+ ["model.language_model.layers", "model.layers"],+ 3,+ (),+ None,+ ),+ "mllama": (+ lambda: _build(+ "MllamaForConditionalGeneration",+ "MllamaConfig",+ text_config = dict(+ vocab_size = 128,+ hidden_size = 64,+ intermediate_size = 128,+ num_hidden_layers = 4,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ cross_attention_layers = [1],+ pad_token_id = 0,+ rope_scaling = {"rope_type": "default"},+ ),+ vision_config = dict(+ hidden_size = 32,+ intermediate_size = 64,+ num_hidden_layers = 5,+ num_global_layers = 2,+ attention_heads = 4,+ image_size = 32,+ patch_size = 8,+ vision_output_dim = 64,+ intermediate_layers_indices = [1],+ ),+ ),+ ["model.language_model.layers", "language_model.model.layers"],+ 4,+ (),+ None,+ ),+ "idefics3": (+ lambda: _build(+ "Idefics3ForConditionalGeneration",+ "Idefics3Config",+ text_config = dict(model_type = "llama", **_text()),+ vision_config = dict(+ hidden_size = 32,+ intermediate_size = 64,+ num_hidden_layers = 5,+ num_attention_heads = 4,+ image_size = 32,+ patch_size = 8,+ ),+ scale_factor = 2,+ ),+ ["model.text_model.layers"],+ 3,+ (),+ None,+ ),+ "smolvlm": (+ lambda: _build(+ "SmolVLMForConditionalGeneration",+ "SmolVLMConfig",+ text_config = dict(model_type = "llama", **_text()),+ vision_config = dict(+ hidden_size = 32,+ intermediate_size = 64,+ num_hidden_layers = 5,+ num_attention_heads = 4,+ image_size = 32,+ patch_size = 8,+ ),+ scale_factor = 2,+ ),+ ["model.text_model.layers"],+ 3,+ (),+ None,+ ),+ "paligemma": (+ lambda: _build(+ "PaliGemmaForConditionalGeneration",+ "PaliGemmaConfig",+ text_config = dict(model_type = "gemma", **_text()),+ vision_config = dict(model_type = "siglip_vision_model", **_siglip()),+ projection_dim = 64,+ hidden_size = 64,+ ),+ ["model.language_model.layers", "language_model.model.layers"],+ 3,+ (),+ None,+ ),+ "whisper": (+ lambda: _build(+ "WhisperForConditionalGeneration",+ "WhisperConfig",+ vocab_size = 128,+ d_model = 32,+ encoder_layers = 5,+ decoder_layers = 3,+ encoder_attention_heads = 4,+ decoder_attention_heads = 4,+ encoder_ffn_dim = 64,+ decoder_ffn_dim = 64,+ pad_token_id = 0,+ bos_token_id = 1,+ eos_token_id = 2,+ decoder_start_token_id = 1,+ ),+ ["model.decoder.layers"],+ 3,+ (),+ None,+ ),+ "t5": (+ lambda: _build(+ "T5ForConditionalGeneration",+ "T5Config",+ vocab_size = 128,+ d_model = 32,+ d_kv = 8,+ d_ff = 64,+ num_layers = 5,+ num_decoder_layers = 3,+ num_heads = 4,+ decoder_start_token_id = 0,+ ),+ ["decoder.block"],+ 3,+ (),+ None,+ ),+ "bert_encoder": (+ lambda: _build(+ "BertModel",+ "BertConfig",+ vocab_size = 128,+ hidden_size = 32,+ num_hidden_layers = 3,+ num_attention_heads = 4,+ intermediate_size = 64,+ ),+ ["encoder.layer"],+ 3,+ (),+ set(),+ ),+ "bert_seq_cls": (+ lambda: _build(+ "BertForSequenceClassification",+ "BertConfig",+ vocab_size = 128,+ hidden_size = 32,+ num_hidden_layers = 3,+ num_attention_heads = 4,+ intermediate_size = 64,+ num_labels = 3,+ ),+ ["bert.encoder.layer"],+ 3,+ (),+ {"classifier"},+ ),+ "llama_seq_cls": (+ lambda: _build("LlamaForSequenceClassification", "LlamaConfig", **_text(), num_labels = 3),+ ["model.layers"],+ 3,+ (),+ {"score"},+ ),+ # Experts in a ModuleDict (`experts.expert_<i>`); one sparse layer each in the encoder and the decoder.+ "switch_transformers": (+ lambda: _build(+ "SwitchTransformersForConditionalGeneration",+ "SwitchTransformersConfig",+ vocab_size = 128,+ d_model = 32,+ d_kv = 8,+ d_ff = 64,+ expert_capacity = 8,+ num_layers = 3,+ num_sparse_encoder_layers = 1,+ num_decoder_layers = 3,+ num_sparse_decoder_layers = 1,+ num_heads = 4,+ num_experts = 4,+ decoder_start_token_id = 0,+ ),+ ["decoder.block"],+ 3,+ {"encoder.block.1.layer.1.mlp.router", "decoder.block.1.layer.2.mlp.router"},+ None,+ ),+ # Attention MoE too: `self_attention.experts` (JetMoeMoA) owns stacked expert weights and its own router.+ "jetmoe": (+ _causal(+ "JetMoe",+ kv_channels = 16,+ intermediate_size = 64,+ num_local_experts = 4,+ num_experts_per_tok = 2,+ ),+ ["model.layers"],+ 3,+ {+ f"model.layers.{i}.{m}.router"+ for i in range(3)+ for m in ("mlp", "self_attention.experts")+ },+ None,+ ),+ "qwen3_tied_real_weights": (+ lambda: _build(+ "Qwen3ForCausalLM", "Qwen3Config", meta = False, **_text(tie_word_embeddings = True)+ ),+ ["model.layers"],+ 3,+ (),+ {"lm_head"},+ ),+}+++def _lists(model):+ return {n: m for n, m in model.named_modules() if isinstance(m, torch.nn.ModuleList)}++[email protected]("family", list(FAMILIES))+def test_selectors_on_family(family):+ build, paths, n, router_suffixes, heads = FAMILIES[family]+ try:+ model = build()+ except pytest.skip.Exception:+ raise+ except Exception as e: # a config this transformers cannot build is not a selector result+ pytest.skip(f"{family} does not build in transformers {transformers.__version__}: {e!r}")+ names = dict(model.named_modules())+ lists = _lists(model)+ path = next((p for p in paths if p in lists), None)+ assert path is not None, (family, paths, sorted(lists)[:20])+ assert len(lists[path]) == n++ # decoder_layers: the language decoder's block list.+ assert P.decoder_layers(model) == (path, n), family++ # hp_filter(first_n = 1, last_n = 1): exactly the first and last language blocks.+ is_hp = P.hp_filter(+ model,+ P.HPLayerPolicy(output_head = False, embeddings = False, routers = False, first_n = 1, last_n = 1),+ )+ assert set(is_hp.keep) == {f"{path}.0", f"{path}.{n - 1}"}, (family, is_hp.keep)+ for i in range(n):+ for sub, _ in lists[path][i].named_modules():+ fqn = f"{path}.{i}.{sub}" if sub else f"{path}.{i}"+ assert is_hp(None, fqn) == (i in (0, n - 1)), (family, fqn)+ for fqn in names:+ if fqn and not (fqn == path or fqn.startswith(path + ".")):+ assert not is_hp(None, fqn), (family, fqn)++ # Output heads.+ out = model.get_output_embeddings()+ expected_heads = heads if heads is not None else {k for k, v in names.items() if v is out}+ assert set(P.find_output_heads(model)) == expected_heads, family++ # Embeddings: the input embedding is found, and nothing found is a Linear.+ emb = P.find_embeddings(model)+ inp = model.get_input_embeddings()+ assert any(names[k] is inp for k in emb), family+ assert not any(+ isinstance(names[k], torch.nn.Linear) and names[k] is not inp for k in emb+ ), family++ # Routers: exactly the router modules, never an expert, shared expert or dense MLP.+ # A set is the exact expected fqns; a tuple is suffixes matched outside any expert subtree.+ if isinstance(router_suffixes, set):+ expected_routers = router_suffixes+ else:+ expected_routers = {+ k+ for k in names+ if router_suffixes and k.endswith(router_suffixes) and "expert" not in k+ }+ assert set(P.find_routers(model)) == expected_routers, familydiff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex fd7bbd16329..340e9521b13 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -78,11 +78,33 @@ def _opaque_under_compile(name, fake):ValueError,): # Already registered by an earlier import of this module.pass- return fn+ if name not in _EAGER_COUNTED_LOWP_GEMMS:+ return fn+ from ..precision import engage++ # Eager calls launch Triton directly and never dispatch the op, so count_lowp_gemms() is told here.+ @functools.wraps(fn)+ def counted(*args, **kwargs):+ if torch.compiler.is_compiling() or not engage._active():+ return fn(*args, **kwargs)+ entered = engage.note_lowp_gemm(f"unsloth::{name}")+ try:+ return fn(*args, **kwargs)+ finally:+ engage.release_lowp_gemm(entered)++ return countedreturn decorator+_EAGER_COUNTED_LOWP_GEMMS = (+ "fp8_block_matmul_triton",+ "fp8_block_small_m_linear",+ "fp8_block_matmul_torchao",+)++try:from transformers.integrations.finegrained_fp8 import FP8Linearexcept:diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 0698721a41c..4184264fdde 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -10001,10 +10001,23 @@ def _prepare_model_for_qat(_reinstall_input_require_grads(require_grads_owners)_reregister_unowned_require_grads_hooks(model, all_embeddings, unowned_require_grads)_patch_microscaling_qat_forwards(model)+ _record_qat_engagement(model, torchao_config)return model+def _record_qat_engagement(model, torchao_config):+ """Store how many layers QAT fake-quantizes on the model (unsloth.precision.engaged); under+ UNSLOTH_STRICT_ENGAGE=1 a scheme that wrapped nothing raises instead of training unquantized."""+ from ..precision import record, require++ scheme = getattr(torchao_config, "qat_scheme", None)+ name = "qat:" + (scheme if isinstance(scheme, str) else "custom")+ count = sum(1 for m in model.modules() if "FakeQuantized" in type(m).__name__)+ record(model, name, count)+ require(name, count, "no Linear / Embedding layer matched the scheme's filters")++def patch_hf_quantizer():def make_trainable(self):return Truediff --git a/unsloth/models/float8_training.py b/unsloth/models/float8_training.pyindex fab1bd63f1a..1cf9916d977 100644--- a/unsloth/models/float8_training.py+++ b/unsloth/models/float8_training.py@@ -8,6 +8,42 @@ from __future__ import annotationsimport torch++def _load_precision():+ """unsloth.precision; also when this file is loaded on its own (gloo test workers import it by path+ without the unsloth package), from the sibling directory."""+ try:+ from .. import precision+ return precision+ except ImportError:+ import importlib.util+ import os+ import sys++ name = "_unsloth_precision_standalone"+ if name in sys.modules:+ return sys.modules[name]+ root = os.path.join(+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "precision"+ )+ spec = importlib.util.spec_from_file_location(+ name, os.path.join(root, "__init__.py"), submodule_search_locations = [root]+ )+ module = importlib.util.module_from_spec(spec)+ sys.modules[name] = module+ spec.loader.exec_module(module)+ return module+++_precision = _load_precision()+_PRECISION_NAMES = _precision.FLOAT8_TRAINING_NAMES+_check_precision_recipe = _precision.check_recipe+_find_routers = _precision.find_routers+_precision_measured_faster = _precision.measured_faster+_precision_named_recipes = _precision.named_recipes+_record_engagement = _precision.record+_require_engagement = _precision.require+__all__ = ["FLOAT8_TRAINING_RECIPES","resolve_float8_training",@@ -15,21 +51,25 @@ __all__ = ["convert_model_to_float8_training",]-FLOAT8_TRAINING_RECIPES = ("tensorwise",)+# Names and gates come from the precision registry (unsloth/precision/recipes.py).+FLOAT8_TRAINING_RECIPES = _precision_named_recipes("float8_training")# Named recipes run only where they were timed faster than bfloat16 at equal loss (Qwen3-1.7B / 4B full# finetuning). tensorwise: RTX PRO 6000 (12.0) 15% faster; L4 (8.9) NaN gradients with gradient# checkpointing. rowwise: 3% slower on the RTX PRO 6000 and 2% slower on the L4, so it is not offered by# name. Any other GPU needs an explicit torchao Float8LinearConfig.-_MEASURED_FASTER = {"tensorwise": ((12, 0), (12, 1))}+_MEASURED_FASTER = _precision_measured_faster("float8_training")# Never fp8: output heads (Unsloth's fused CE reads lm_head.weight directly), MoE routers (tiny, precision sensitive)._SKIP_NAME_PARTS = ("lm_head", "embed_out", "output_projection", "router")_SKIP_LEAF_NAMES = ("gate", "router", "score", "classifier")-def resolve_float8_training(float8_training):- """`float8_training` kwarg -> recipe name or a torchao Float8LinearConfig; None when off."""+def resolve_float8_training(float8_training, dtype = None):+ """`float8_training` kwarg -> recipe name or a torchao Float8LinearConfig; None when off. A format the+ precision registry knows but Unsloth cannot train (MXFP6, or one not built yet) raises with its reason,+ before the model is loaded; `dtype` (the load dtype, None = auto) refuses float16 for formats whose+ quantizer rejects it."""if float8_training is None or float8_training is False:return Noneif float8_training is True:@@ -42,6 +82,16 @@ def resolve_float8_training(float8_training):"timed on (L4, RTX PRO 6000). Use float8_training = True (tensorwise), or pass ""torchao.float8.Float8LinearConfig.from_recipe_name('rowwise') to force it.")+ if recipe not in FLOAT8_TRAINING_RECIPES and recipe in _PRECISION_NAMES:+ # Raises PrecisionRefused (a ValueError) with the format's own reason.+ _check_precision_recipe(+ _PRECISION_NAMES[recipe], gpu = _precision.current_gpu_class(), input_dtype = dtype+ )+ raise ValueError(+ f"Unsloth: float8_training = '{float8_training}' has not been measured faster than bfloat16 "+ "on any GPU, so it is not offered by name. Pass "+ f"torchao.float8.Float8LinearConfig.from_recipe_name({recipe!r}) to force it."+ )if recipe not in FLOAT8_TRAINING_RECIPES:raise ValueError(f"Unsloth: float8_training = '{float8_training}' is not supported. "@@ -80,6 +130,12 @@ def float8_training_filter(except Exception:passsize_recipe = _recipe_scaling(recipe)+ # Routers found by shape beside an experts container (a router named `wg` or in remote code), on top of the+ # name rules below.+ try:+ routers = frozenset(_find_routers(model))+ except Exception:+ routers = frozenset()def module_filter_fn(module, fqn):# Exact type: bitsandbytes / QAT / fp8 / PEFT linears subclass nn.Linear and must stay as they are.@@ -87,7 +143,10 @@ def float8_training_filter(return Falseif id(module) in out_heads or any(p in fqn for p in _SKIP_NAME_PARTS):return False- if fqn.rsplit(".", 1)[-1] in _SKIP_LEAF_NAMES:+ # A router found structurally can be a gating wrapper: its inner Linear is the router too.+ if fqn.rsplit(".", 1)[-1] in _SKIP_LEAF_NAMES or any(+ fqn == r or fqn.startswith(r + ".") for r in routers+ ):return Falseweight = module.weightif not weight.is_floating_point() or weight.element_size() < 2 or not weight.requires_grad:@@ -327,6 +386,16 @@ def convert_model_to_float8_training(converted = [name for name, m in model.named_modules() if isinstance(m, Float8Linear)]if dispatch_hooks:_reattach_dispatch_hooks(model, converted, dispatch_hooks)+ name = (+ recipe if isinstance(recipe, str) else "Float8LinearConfig(" + _recipe_scaling(recipe) + ")"+ )+ _record_engagement(model, "float8_training:" + name, len(converted))+ # UNSLOTH_STRICT_ENGAGE=1 raises here instead of training in the original precision.+ _require_engagement(+ "float8_training",+ len(converted),+ "no eligible Linear layer: dims divisible by 16 and large enough",+ )if not converted:import warningswarnings.warn(diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 5a85254c007..6d25df63c2d 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -2263,7 +2263,7 @@ class FastModel(FastBaseModel):):rope_scaling = _fold_rope_parameters(rope_scaling, kwargs)user_config = kwargs.pop("config", None)- float8_training = resolve_float8_training(kwargs.pop("float8_training", None))+ float8_training = resolve_float8_training(kwargs.pop("float8_training", None), dtype = dtype)if float8_training is not None and not full_finetuning:raise ValueError("Unsloth: `float8_training` is for `full_finetuning = True` only; LoRA / QLoRA keep "diff --git a/unsloth/precision/__init__.py b/unsloth/precision/__init__.pynew file mode 100644index 00000000000..e11634c5f7b--- /dev/null+++ b/unsloth/precision/__init__.py@@ -0,0 +1,72 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Low-precision training policy: one recipe per format (per-GEMM dtypes and scaling, high-precision+layers, GradScaler kind, per-GPU gates), refusals that run before the model is touched, structural+high-precision layer selection and engagement checks. Data and checks only: the kernels stay in torchao and+Unsloth's kernels, reached through the existing entry points (`float8_training`, `qat_scheme`, `load_in_fp8`)."""++from .recipes import (+ FLOAT8_TRAINING_NAMES,+ RECIPES,+ Gate,+ GemmPolicy,+ HPLayerPolicy,+ Operand,+ Recipe,+ get_recipe,+ measured_faster,+ named_recipes,+ recipes_of_kind,+)+from .hardware import (+ PrecisionRefused,+ check_recipe,+ current_gpu_class,+ dtype_tag,+ gpu_class,+ grad_scaler_kind,+)+from .selectors import (+ decoder_layers,+ find_embeddings,+ find_output_heads,+ find_routers,+ high_precision_layers,+ hp_filter,+ num_experts_of,+)+from .engage import EngagementError, count_lowp_gemms, engaged, record, require, strict_engage++__all__ = [+ "FLOAT8_TRAINING_NAMES",+ "RECIPES",+ "Gate",+ "GemmPolicy",+ "HPLayerPolicy",+ "Operand",+ "Recipe",+ "get_recipe",+ "measured_faster",+ "named_recipes",+ "recipes_of_kind",+ "PrecisionRefused",+ "check_recipe",+ "current_gpu_class",+ "dtype_tag",+ "gpu_class",+ "grad_scaler_kind",+ "decoder_layers",+ "find_embeddings",+ "find_output_heads",+ "find_routers",+ "high_precision_layers",+ "hp_filter",+ "num_experts_of",+ "EngagementError",+ "count_lowp_gemms",+ "engaged",+ "record",+ "require",+ "strict_engage",+]diff --git a/unsloth/precision/engage.py b/unsloth/precision/engage.pynew file mode 100644index 00000000000..ea891206763--- /dev/null+++ b/unsloth/precision/engage.py@@ -0,0 +1,205 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Engagement: proof that a requested low-precision path really ran (strict mode: UNSLOTH_STRICT_ENGAGE=1)."""++from __future__ import annotations++import contextlib+import os+import threading+import warnings+from typing import Dict++__all__ = [+ "EngagementError",+ "strict_engage",+ "record",+ "engaged",+ "require",+ "LOWP_GEMM_OPS",+ "LOWP_GEMM_CUSTOM_OPS",+ "count_lowp_gemms",+]++_ENV = "UNSLOTH_STRICT_ENGAGE"+++class EngagementError(RuntimeError):+ """A requested feature silently fell back (strict mode)."""+++def strict_engage() -> bool:+ return os.environ.get(_ENV, "0").strip().lower() in ("1", "true", "yes", "on")+++def record(+ model,+ recipe: str,+ count: int,+ detail = None,+) -> None:+ try:+ table = getattr(model, "_unsloth_precision_engaged", None)+ if not isinstance(table, dict):+ table = {}+ table[recipe] = {+ "modules": int(count),+ **({"detail": detail} if detail is not None else {}),+ }+ model._unsloth_precision_engaged = table+ except Exception:+ pass+++def engaged(model) -> Dict[str, dict]:+ return dict(getattr(model, "_unsloth_precision_engaged", None) or {})+++def require(+ recipe: str,+ count: int,+ why: str = "",+) -> bool:+ """True when engaged. Count 0 under strict mode raises EngagementError; otherwise returns False."""+ if count > 0:+ return True+ if strict_engage():+ raise EngagementError(+ f"Unsloth: {recipe} was requested but engaged on 0 modules"+ + (f" ({why})" if why else "")+ + f". {_ENV}=1 refuses silent fallbacks; unset it to train in the original precision."+ )+ return False+++# aten ops that run a low-precision GEMM (names resolved lazily: absent ops are skipped).+LOWP_GEMM_OPS = ("_scaled_mm", "_scaled_mm_v2", "_scaled_grouped_mm", "_scaled_grouped_mm_v2")++# Opaque custom ops that run a low-precision GEMM inside: dispatch sees only the outer op, so each call counts+# once at its boundary. Matched by qualified name, so a namespace that never loaded (no fbgemm) costs nothing.+# Dequantize-then-matmul ops (mxfp4_bounded_dense_matmul, fp8 weight dequant, fp8_block_gemv, which upcasts+# the weight before its dot) are not low-precision GEMMs.+LOWP_GEMM_CUSTOM_OPS = (+ "unsloth::nvfp4_qat_linear",+ "unsloth::fp8_block_matmul_triton",+ "unsloth::fp8_block_matmul_torchao",+ "unsloth::fp8_block_small_m_linear",+ "unsloth_zoo::mxfp4_ogs_moe",+ "fbgemm::f8f8bf16",+ "fbgemm::f8f8bf16_tensorwise",+ "fbgemm::f8f8bf16_rowwise",+ "fbgemm::f8f8bf16_blockwise",+ "fbgemm::f8f8bf16_cublas",+)+++# Per thread, like torch's dispatch-mode stack: the counters open on this thread, and the ids of those already+# inside a counted GEMM (a GEMM that delegates to another counted launcher or op, torchao block matmul ->+# Triton, counts once at its outer boundary). Eager launchers that bypass dispatch report via note_lowp_gemm.+_STATE = threading.local()+++def _active():+ active = getattr(_STATE, "active", None)+ if active is None:+ active = _STATE.active = []+ return active+++def _inside():+ ids = getattr(_STATE, "inside", None)+ if ids is None:+ ids = _STATE.inside = set()+ return ids+++_EAGER_UNDER_COUNTER_WARNED = False+++def _warn_eager_under_counter(torch):+ # torch < 2.9 has no TorchDispatchMode.ignore_compile_internals: Dynamo skips compiled code while any+ # user dispatch mode is active, so counting would silently swap the compiled path for eager.+ global _EAGER_UNDER_COUNTER_WARNED+ if _EAGER_UNDER_COUNTER_WARNED:+ return+ _EAGER_UNDER_COUNTER_WARNED = True+ warnings.warn(+ f"Unsloth: count_lowp_gemms() on torch {torch.__version__} runs torch.compile'd callables eagerly "+ "while counting (needs torch >= 2.9 to count compiled calls in place). Counts are exact for the "+ "eager path, not for the compiled kernels.",+ stacklevel = 3,+ )+++def note_lowp_gemm(name):+ """Count one GEMM in every open counter not already inside a counted GEMM; returns the ids to release+ with `release_lowp_gemm` once the call returns."""+ inside = _inside()+ entered = [id(counts) for counts in _active() if id(counts) not in inside]+ for counts in _active():+ if id(counts) in entered:+ counts[name] = counts.get(name, 0) + 1+ inside.update(entered)+ return entered+++def release_lowp_gemm(entered):+ _inside().difference_update(entered)+++def count_lowp_gemms():+ """Count low-precision GEMM dispatches: `with count_lowp_gemms() as c: ...; c['_scaled_mm']`, and+ `c['unsloth::nvfp4_qat_linear']` for opaque custom ops (LOWP_GEMM_CUSTOM_OPS). On torch < 2.9, compiled+ callables run eagerly inside the counter (warned once)."""+ import torch+ from torch.utils._python_dispatch import TorchDispatchMode++ if not hasattr(TorchDispatchMode, "ignore_compile_internals"):+ _warn_eager_under_counter(torch)++ names = {}+ for name in LOWP_GEMM_OPS:+ if getattr(torch.ops.aten, name, None) is not None:+ names[f"aten::{name}"] = name+ for name in LOWP_GEMM_CUSTOM_OPS:+ names[name] = name+ counts = {key: 0 for key in names.values()}++ class _Counter(TorchDispatchMode):+ # Skip Dynamo tracing (a plain mode breaks fullgraph); torch < 2.9 ignores this hook (warned above).+ @classmethod+ def ignore_compile_internals(cls):+ return True++ def __torch_dispatch__(+ self,+ func,+ types,+ args = (),+ kwargs = None,+ ):+ key = names.get(getattr(getattr(func, "_schema", None), "name", None))+ inside = _inside()+ if key is None or id(counts) in inside:+ return func(*args, **(kwargs or {}))+ counts[key] += 1+ inside.add(id(counts))+ try:+ return func(*args, **(kwargs or {}))+ finally:+ inside.discard(id(counts))++ _active().append(counts)+ try:+ with _Counter():+ yield counts+ finally:+ # By identity: nested counters start as equal dicts, and list.remove compares by value.+ active = _active()+ for i, entry in enumerate(active):+ if entry is counts:+ del active[i]+ break+ _inside().discard(id(counts))diff --git a/unsloth/precision/hardware.py b/unsloth/precision/hardware.pynew file mode 100644index 00000000000..5afaf5f600b--- /dev/null+++ b/unsloth/precision/hardware.py@@ -0,0 +1,134 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""GPU class of a device and the refusal check for a recipe on it. Everything here runs before the model is+mutated: a refused request leaves the model as it was loaded."""++from __future__ import annotations++from typing import Optional, Tuple++from .recipes import Gate, Recipe, get_recipe++__all__ = [+ "PrecisionRefused",+ "gpu_class",+ "current_gpu_class",+ "dtype_tag",+ "check_recipe",+ "grad_scaler_kind",+]+++class PrecisionRefused(ValueError):+ """A precision request that cannot run here; raised before any layer is swapped."""+++def gpu_class(cc: Optional[Tuple[int, int]], hip: bool = False) -> Optional[str]:+ """Compute capability -> the gate table's GPU class (None when unknown / not CUDA)."""+ if hip:+ return "rocm"+ if cc is None:+ return None+ major, minor = int(cc[0]), int(cc[1])+ if major >= 12:+ return "sm120"+ if major == 11 or major == 10:+ return "sm100"+ if major == 9:+ return "sm90"+ if (major, minor) >= (8, 9):+ return "sm89"+ if major == 8:+ return "sm80"+ return "sm75"+++def current_gpu_class(device = None) -> Optional[str]:+ import torch+ try:+ if not torch.cuda.is_available():+ return None+ if torch.version.hip is not None:+ return "rocm"+ return gpu_class(torch.cuda.get_device_capability(device))+ except Exception:+ return None+++def dtype_tag(dtype) -> Optional[str]:+ """torch.dtype / string -> 'bf16' / 'fp16' / 'fp32' (None when unknown or auto)."""+ if dtype is None:+ return None+ s = str(dtype).replace("torch.", "").lower()+ return {+ "bfloat16": "bf16",+ "bf16": "bf16",+ "float16": "fp16",+ "half": "fp16",+ "fp16": "fp16",+ "float32": "fp32",+ "float": "fp32",+ "fp32": "fp32",+ }.get(s)+++_DTYPE_WHY = {+ "fp16": "float16 activations are rejected by its quantizer (torchao to_mx / NVFP4 training accept "+ "bfloat16 / float32 only); load with dtype = torch.bfloat16",+ "fp32": "float32 activations are not supported by this recipe; load with dtype = torch.bfloat16",+ "bf16": "bfloat16 activations are not supported by this recipe",+}+++def check_recipe(+ recipe,+ *,+ gpu: Optional[str] = None,+ input_dtype = None,+ mode: Optional[str] = None,+ allow_planned: bool = False,+) -> Gate:+ """Refuse `recipe` (a Recipe or a registry name) for this GPU class / activation dtype / training mode.++ Order: a format with no training path at all, then the dtype, then the mode, then the GPU gate, then+ "not built yet". Unknown GPU or dtype (None) skips that check. Returns the gate that applies."""+ r: Recipe = get_recipe(recipe) if isinstance(recipe, str) else recipe+ if r.status == "refused":+ raise PrecisionRefused(f"Unsloth: {r.name} training is refused: {r.refusal}")+ tag = dtype_tag(input_dtype)+ if tag is not None and tag not in r.input_dtypes:+ raise PrecisionRefused(+ f"Unsloth: {r.name} cannot train with {tag} activations: {_DTYPE_WHY[tag]}."+ )+ if mode is not None and mode not in r.modes:+ raise PrecisionRefused(+ f"Unsloth: {r.name} supports training modes {r.modes}, not {mode!r}."+ )+ gate = r.gate(gpu) if gpu is not None else Gate("unprobed")+ if gate.status == "refused":+ raise PrecisionRefused(+ f"Unsloth: {r.name} is refused on {gpu}: {gate.reason}."+ + (f" (evidence: {gate.evidence})" if gate.evidence else "")+ )+ if r.status == "planned" and not allow_planned:+ where = ", ".join(f"{g}: {v.status}" for g, v in sorted(r.gates.items()))+ raise PrecisionRefused(+ f"Unsloth: {r.name} training is not available in Unsloth yet (planned; per GPU: {where}). "+ "Train in bfloat16, or with float8_training = True on a supported GPU."+ )+ return gate+++def grad_scaler_kind(+ recipe,+ input_dtype,+ fsdp2: bool = False,+) -> str:+ """'none', 'GradScaler' or 'ShardedGradScaler': the loss scaler this recipe needs with these activations."""+ r: Recipe = get_recipe(recipe) if isinstance(recipe, str) else recipe+ tag = dtype_tag(input_dtype)+ needs = r.grad_scaler == "dynamic" or (r.grad_scaler == "per_dtype" and tag == "fp16")+ if not needs:+ return "none"+ return "ShardedGradScaler" if fsdp2 else "GradScaler"diff --git a/unsloth/precision/recipes.py b/unsloth/precision/recipes.pynew file mode 100644index 00000000000..07d99d80fe6--- /dev/null+++ b/unsloth/precision/recipes.py@@ -0,0 +1,577 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""One table entry per low-precision training format: what each GEMM runs in, how it is scaled, which+layers stay in high precision, which GradScaler it needs, and where it runs.++A recipe is data only. `hardware.check_recipe` turns the gate table into a refusal before the model is+touched, `selectors.high_precision_layers` applies the HP-layer policy, and `engage` checks that the+low-precision path really ran. The existing entry points (`float8_training`, `qat_scheme`, `load_in_fp8`) look+their recipe up here, so the table is the one place that says what is supported.++GEMM names: fwd = y = x W^T, dgrad = dx = dy W, wgrad = dW = dy^T x.+"""++from __future__ import annotations++import dataclasses+from typing import Mapping, Optional, Tuple++__all__ = [+ "Operand",+ "GemmPolicy",+ "HPLayerPolicy",+ "Gate",+ "Recipe",+ "RECIPES",+ "get_recipe",+ "recipes_of_kind",+ "named_recipes",+ "measured_faster",+]++[email protected](frozen = True)+class Operand:+ """One GEMM operand: element dtype and how its scale is computed."""++ dtype: str # e4m3, e5m2, e2m1, bf16, fp16, fp32, int4, int8, input (= the activation dtype)+ granularity: str = "none" # none, tensor, row (axiswise), block1x128, block128x128, block1x32, block1x16, block16x16, group32, group128+ scale_dtype: str = "none" # fp32, e8m0, e4m3, none+ rounding: str = (+ "rn" # rn, sr (stochastic), rceil (MX: scale rounded up), pow2 (power-of-two scales)+ )+ rht: bool = False # random Hadamard transform before quantizing (NVFP4 wgrad)++[email protected](frozen = True)+class GemmPolicy:+ """(a, b) operands of one GEMM; None = this GEMM runs in the activation dtype (high precision)."""++ a: Optional[Operand] = None+ b: Optional[Operand] = None++ @property+ def low_precision(self) -> bool:+ return self.a is not None or self.b is not None+++HP = GemmPolicy()++[email protected](frozen = True)+class HPLayerPolicy:+ """Which linear layers stay in high precision. Detected by structure (selectors.py), not by exact names."""++ output_head: bool = True # lm_head / get_output_embeddings and any weight tied to it+ embeddings: bool = True # nn.Embedding and get_input_embeddings+ routers: bool = (+ True # MoE routers: out_features == number of experts beside an experts container+ )+ first_n: int = 0 # first N decoder layers+ last_n: int = 0 # last N decoder layers+ last_frac: float = 0.0 # or the last fraction of decoder layers (NVFP4 ~0.15), rounded up+ shared_experts: bool = False # MoE shared expert (humans& NVFP4-RL keeps it bf16)++[email protected](frozen = True)+class Gate:+ """Status of a recipe on one GPU class.++ status: named = runs by name (measured faster at equal loss),+ explicit = runs only with an explicit torchao config (works, not measured faster),+ refused = raises with `reason` before the model is touched,+ unprobed = never run on this class; treated as explicit with a warning.+ """++ status: str+ reason: str = ""+ evidence: str = ""++[email protected](frozen = True)+class Recipe:+ name: str+ kind: str # dtype, float8_training, qat, frozen_base+ fwd: GemmPolicy = HP+ dgrad: GemmPolicy = HP+ wgrad: GemmPolicy = HP+ hp_layers: HPLayerPolicy = HPLayerPolicy()+ input_dtypes: Tuple[str, ...] = ("bf16", "fp16", "fp32")+ modes: Tuple[str, ...] = ("full",) # full, lora, qlora+ grad_scaler: str = "none" # none, dynamic (fp16 loss scaling), per_dtype (dynamic only when the activations are fp16)+ compile: str = (+ "ok" # ok, opaque (wrapped in a custom op inside compiled regions), fails, unknown+ )+ backend: str = "" # entry point that implements it+ status: str = "shipped" # shipped, planned, refused+ refusal: str = "" # reason when status == refused (for every GPU)+ gates: Mapping[str, Gate] = dataclasses.field(default_factory = dict) # gpu class -> Gate+ alias_of: str = ""+ notes: str = ""++ def gate(self, gpu_class: str) -> Gate:+ if self.status == "refused":+ return Gate("refused", self.refusal)+ return self.gates.get(gpu_class, Gate("unprobed"))+++E4M3 = "e4m3"+E5M2 = "e5m2"+++def _op(+ dtype,+ gran,+ scale = "fp32",+ rounding = "rn",+ rht = False,+):+ return Operand(dtype, gran, scale, rounding, rht)+++_NO_FP8_HW = Gate("refused", "needs fp8 tensor cores (compute capability 8.9+)")+_NO_MX_HW = Gate("refused", "needs block-scaled tensor cores (compute capability 10.0+ / 12.0)")+_HP_FP8 = HPLayerPolicy(output_head = True, embeddings = True, routers = True)++# GPU classes: sm75 (T4), sm80 (A100), sm89 (L4 / RTX 40), sm90 (H100), sm100 (B200 / GB200),+# sm120 (RTX PRO 6000 / RTX 50), rocm. Evidence strings name the probe that set the gate.+_PROBE_G4 = "probed on an RTX PRO 6000 (sm120), torch 2.13.0, torchao 0.18.0"+_PROBE_L4 = "probed on an L4 (sm89), torch 2.13.0, torchao 0.18.0"+_PROBE_B200 = "probed on a B200 (sm100), torch 2.13, torchao 0.18"++RECIPES = {}+++def _add(recipe: Recipe):+ assert recipe.name not in RECIPES, recipe.name+ RECIPES[recipe.name] = recipe+ return recipe+++# ---- plain dtypes -----------------------------------------------------------------------------------+_add(+ Recipe(+ name = "fp16",+ kind = "dtype",+ input_dtypes = ("fp16",),+ modes = ("full", "lora", "qlora"),+ grad_scaler = "dynamic",+ backend = "torch.amp.GradScaler (fsdp2_native._ShardedGradScaler under FSDP2)",+ gates = {+ g: Gate("named") for g in ("sm75", "sm80", "sm89", "sm90", "sm100", "sm120", "rocm")+ },+ notes = "fp32 masters for trainable params; Megatron default scaler init 2^32, window 1000, hysteresis 2",+ )+)+_add(+ Recipe(+ name = "bf16",+ kind = "dtype",+ input_dtypes = ("bf16",),+ modes = ("full", "lora", "qlora"),+ gates = {+ "sm75": Gate("refused", "no bfloat16 tensor cores; Unsloth downgrades to float16"),+ **{g: Gate("named") for g in ("sm80", "sm89", "sm90", "sm100", "sm120", "rocm")},+ },+ )+)++# ---- torchao float8 training (full finetuning) ------------------------------------------------------+_add(+ Recipe(+ name = "fp8_tensorwise",+ kind = "float8_training",+ fwd = GemmPolicy(_op(E4M3, "tensor"), _op(E4M3, "tensor")),+ dgrad = GemmPolicy(_op(E5M2, "tensor"), _op(E4M3, "tensor")),+ wgrad = GemmPolicy(_op(E5M2, "tensor"), _op(E4M3, "tensor")),+ hp_layers = _HP_FP8,+ grad_scaler = "per_dtype",+ backend = "torchao.float8.Float8LinearConfig.from_recipe_name('tensorwise')",+ gates = {+ "sm75": _NO_FP8_HW,+ "sm80": _NO_FP8_HW,+ "sm89": Gate(+ "refused",+ "NaN gradients on an L4: the compiled bfloat16 weight gradient is NaN at "+ "2048 x 4096 x 4096 (3 of 3 seeds) and was 0.50 relative off at 512 x 1024 x 2048 once, while "+ "eager and the compiled forward / input gradient are correct",+ _PROBE_L4,+ ),+ "sm90": Gate("explicit"),+ "sm100": Gate("explicit", evidence = _PROBE_B200),+ "sm120": Gate(+ "named",+ evidence = "Qwen3 full FT 15% faster on RTX PRO 6000 (float8_training.py); "+ "compiled linear fwd+bwd 1.56-1.61x bf16 at 4096 x 14336 (" + _PROBE_G4 + ")",+ ),+ },+ alias_of = "",+ notes = "float8_training = True",+ )+)+_add(+ Recipe(+ name = "fp8_rowwise",+ kind = "float8_training",+ fwd = GemmPolicy(_op(E4M3, "row", rounding = "pow2"), _op(E4M3, "row", rounding = "pow2")),+ dgrad = GemmPolicy(_op(E4M3, "row", rounding = "pow2"), _op(E4M3, "row", rounding = "pow2")),+ wgrad = GemmPolicy(_op(E4M3, "row", rounding = "pow2"), _op(E4M3, "row", rounding = "pow2")),+ hp_layers = _HP_FP8,+ grad_scaler = "per_dtype",+ backend = "torchao.float8.Float8LinearConfig.from_recipe_name('rowwise')",+ gates = {+ "sm75": _NO_FP8_HW,+ "sm80": _NO_FP8_HW,+ "sm89": Gate(+ "explicit",+ evidence = "2% slower than bf16 on an L4 (float8_training.py); compiled linear "+ "0.69-1.13x bf16 (" + _PROBE_L4 + ")",+ ),+ "sm90": Gate("explicit"),+ "sm100": Gate("explicit", evidence = _PROBE_B200),+ "sm120": Gate(+ "explicit",+ evidence = "3% slower than bf16 on RTX PRO 6000 (float8_training.py); compiled "+ "linear 0.54-0.93x bf16 (" + _PROBE_G4 + ")",+ ),+ },+ )+)+_add(+ Recipe(+ name = "fp8_rowwise_gw_hp",+ kind = "float8_training",+ fwd = GemmPolicy(_op(E4M3, "row"), _op(E4M3, "row")),+ dgrad = GemmPolicy(_op(E4M3, "row"), _op(E4M3, "tensor")),+ wgrad = HP,+ hp_layers = _HP_FP8,+ grad_scaler = "per_dtype",+ backend = "torchao.float8.Float8LinearConfig.from_recipe_name('rowwise_with_gw_hp')",+ gates = {+ "sm75": _NO_FP8_HW,+ "sm80": _NO_FP8_HW,+ "sm89": Gate(+ "explicit", evidence = "compiled linear 0.93-1.14x bf16 (" + _PROBE_L4 + ")"+ ),+ "sm90": Gate("explicit"),+ "sm100": Gate("explicit", evidence = _PROBE_B200),+ "sm120": Gate(+ "explicit", evidence = "compiled linear 0.86-1.24x bf16 (" + _PROBE_G4 + ")"+ ),+ },+ )+)+_add(+ Recipe(+ name = "fp8_blockwise",+ kind = "float8_training",+ fwd = GemmPolicy(_op(E4M3, "block1x128"), _op(E4M3, "block128x128")),+ dgrad = GemmPolicy(_op(E4M3, "block1x128"), _op(E4M3, "block128x128")),+ wgrad = GemmPolicy(_op(E4M3, "block1x128"), _op(E4M3, "block128x1")),+ hp_layers = HPLayerPolicy(first_n = 1, last_n = 2),+ input_dtypes = ("bf16",),+ backend = "torchao.prototype.blockwise_fp8_training.linear.Float8BlockwiseLinear",+ status = "planned",+ gates = {+ **{+ g: Gate(+ "refused",+ "torchao's DeepSeek 1x128 / 128x128 scaled_mm is SM90 only in torch "+ "(Transformer Engine Float8BlockScaling does run on sm120)",+ _PROBE_B200 + "; " + _PROBE_G4 + "; " + _PROBE_L4,+ )+ for g in ("sm75", "sm80", "sm89", "sm100", "sm120", "rocm")+ },+ "sm90": Gate("unprobed", "H100 is not available to test on"),+ },+ notes = "fp16 rejected by torchao; NVFP4 paper FP8 baseline keeps first 1 + last 2 blocks in bf16",+ )+)++# ---- microscaling formats ---------------------------------------------------------------------------+_MX = _op(E4M3, "block1x32", "e8m0", "rceil")+_add(+ Recipe(+ name = "mxfp8",+ kind = "float8_training",+ fwd = GemmPolicy(_MX, _MX),+ dgrad = GemmPolicy(_MX, _MX),+ wgrad = GemmPolicy(_MX, _MX),+ hp_layers = HPLayerPolicy(output_head = True, embeddings = True, routers = True),+ input_dtypes = ("bf16",),+ grad_scaler = "none",+ backend = "torchao.prototype.moe_training.config.MXFP8TrainingOpConfig (MXFP8_RCEIL)",+ status = "planned",+ gates = {+ "sm75": _NO_MX_HW,+ "sm80": _NO_MX_HW,+ "sm89": Gate(+ "refused",+ "torchao MXFP8 kernels need block-scaled tensor cores (L4: 'needs triton')",+ _PROBE_L4,+ ),+ "sm90": _NO_MX_HW,+ "sm100": Gate("explicit", evidence = _PROBE_B200),+ "sm120": Gate(+ "explicit",+ evidence = "linear fwd / dgrad / wgrad rel err 0.038 (= fp8 rowwise), eager and "+ "compiled; compiled 1.49-1.55x bf16 at 4096 x 14336 ("+ + _PROBE_G4+ + "). Transformer "+ "Engine 2.20 refuses MXFP8 on 12.0+",+ ),+ },+ notes = "arXiv 2506.08027: e4m3 for every tensor incl. gradients, scale rounded up (RCEIL), all blocks quantized; "+ "torchao to_mx rejects float16 inputs",+ )+)+_add(+ Recipe(+ name = "mxfp6",+ kind = "float8_training",+ status = "refused",+ refusal = "MXFP6 has no training kernel in torchao, Transformer Engine or Megatron (only QAT / emulation), "+ "and no tensor-core speed over MXFP8 on Blackwell. Use mxfp8, or qat_scheme for MXFP6 deployment.",+ )+)+_NV_X = _op("e2m1", "block1x16", "e4m3", "rn")+_NV_W = _op("e2m1", "block16x16", "e4m3", "rn")+_NV_G = _op("e2m1", "block1x16", "e4m3", "sr")+_add(+ Recipe(+ name = "nvfp4",+ kind = "float8_training",+ fwd = GemmPolicy(_NV_X, _NV_W),+ dgrad = GemmPolicy(_NV_G, _NV_W),+ wgrad = GemmPolicy(+ _op("e2m1", "block1x16", "e4m3", "sr", rht = True),+ _op("e2m1", "block1x16", "e4m3", "rn", rht = True),+ ),+ hp_layers = HPLayerPolicy(output_head = True, embeddings = True, routers = True, last_frac = 0.15),+ input_dtypes = ("bf16",),+ grad_scaler = "none",+ compile = "opaque",+ backend = "torchao.prototype.moe_training.nvfp4_training.NVFP4TrainingConfig",+ status = "planned",+ gates = {+ "sm75": _NO_MX_HW,+ "sm80": _NO_MX_HW,+ "sm89": Gate("refused", "torchao NVFP4 kernels require SM100+", _PROBE_L4),+ "sm90": _NO_MX_HW,+ "sm100": Gate("explicit", evidence = _PROBE_B200 + " (eager only)"),+ "sm120": Gate(+ "refused",+ "torchao NVFP4 training does not build on sm_120a (Triton: cvt.rs stochastic "+ "rounding unsupported; Inductor cannot lower the swizzled scaled_mm). Transformer Engine "+ "2.20 NVFP4BlockScaling runs in bfloat16 (rel err 0.15-0.18, linear fwd+bwd 2.1-2.4x bf16) "+ "and is the sm120 route",+ _PROBE_G4,+ ),+ },+ notes = "arXiv 2509.25149: 16-element blocks, e4m3 block scale + fp32 tensor scale, 2D 16x16 weights, "+ "RHT 16x16 on wgrad inputs only, SR on gradients only, ~15% of layers (mostly last) in bf16. "+ "Inductor cannot lower its swizzled scaled_mm, so it runs as an opaque op under compile.",+ )+)++# ---- MoE grouped experts (torch._grouped_mm in low precision) -----------------------------------------+_add(+ Recipe(+ name = "fp8_rowwise_grouped",+ kind = "float8_training",+ fwd = GemmPolicy(_op(E4M3, "row"), _op(E4M3, "row")),+ dgrad = GemmPolicy(_op(E4M3, "row"), _op(E4M3, "row")),+ wgrad = GemmPolicy(_op(E4M3, "row"), _op(E4M3, "row")),+ input_dtypes = ("bf16",),+ backend = "torchao.prototype.moe_training.config.Float8TrainingOpConfig",+ status = "planned",+ gates = {+ "sm75": _NO_FP8_HW,+ "sm80": _NO_FP8_HW,+ **{+ g: Gate(+ "refused",+ "torch._scaled_grouped_mm runs on compute capability 9.0 / 10.0 only",+ _PROBE_G4 + "; " + _PROBE_L4,+ )+ for g in ("sm89", "sm120")+ },+ "sm90": Gate("unprobed"),+ "sm100": Gate(+ "refused",+ "cutlass cannot run (error 7); an unspecified launch failure poisons the CUDA "+ "context when run in a shared process",+ _PROBE_B200,+ ),+ },+ notes = "MoE experts (3-D weights); fallback is dequant + bf16 torch._grouped_mm",+ )+)+_add(+ Recipe(+ name = "mxfp8_grouped",+ kind = "float8_training",+ fwd = GemmPolicy(_MX, _MX),+ dgrad = GemmPolicy(_MX, _MX),+ wgrad = GemmPolicy(_MX, _MX),+ input_dtypes = ("bf16",),+ backend = "torchao.prototype.moe_training.config.MXFP8TrainingOpConfig (grouped)",+ status = "planned",+ gates = {+ "sm75": _NO_MX_HW,+ "sm80": _NO_MX_HW,+ "sm90": _NO_MX_HW,+ **{+ g: Gate(+ "refused",+ "torchao's MXFP8 grouped kernels are SM100 only (EMULATED runs, no speed)",+ _PROBE_G4 + "; " + _PROBE_L4,+ )+ for g in ("sm89", "sm120")+ },+ "sm100": Gate(+ "unprobed",+ "DSLRuntimeError with nvidia-cutlass-dsl 4.6.2 (" + _PROBE_B200 + "); "+ "pin a torchao / cutlass-dsl pair whose subprocess probe passes",+ ),+ },+ notes = "MoE experts; fallback is dequant + bf16 torch._grouped_mm",+ )+)++# ---- QAT (fake quant of the frozen or trained base; the GEMM itself runs in the activation dtype) --------+for _name, _dtypes, _note in (+ ("fp8-int4", ("bf16", "fp16", "fp32"), "Float8DynamicActivationInt4WeightConfig, group 128"),+ ("fp8-fp8", ("bf16", "fp16", "fp32"), "Float8DynamicActivationFloat8WeightConfig PerRow"),+ ("nvfp4", ("bf16", "fp32"), "NVFP4DynamicActivationNVFP4WeightConfig, K % 32"),+ ("mxfp8", ("bf16",), "MXDynamicActivationMXWeightConfig, torchao 0.16+, bf16 layers only"),+ ("int8-int4", ("bf16", "fp16", "fp32"), "phone-deployment alias"),+ ("int4", ("bf16", "fp16", "fp32"), "Int4WeightOnlyConfig, group 128"),+ ("int8", ("bf16", "fp16", "fp32"), "Int8 weight only"),+ ("cactus", ("bf16", "fp16", "fp32"), "Cactus int8 / int4 per group"),+):+ _add(+ Recipe(+ name = "qat:" + _name,+ kind = "qat",+ input_dtypes = _dtypes,+ modes = ("full", "lora"),+ grad_scaler = "per_dtype",+ compile = "opaque",+ backend = "unsloth.models._utils._prepare_model_for_qat",+ gates = {},+ notes = _note,+ )+ )+_add(+ Recipe(+ name = "qat:phone-deployment",+ kind = "qat",+ alias_of = "qat:int8-int4",+ modes = ("full", "lora"),+ backend = "unsloth.models._utils._prepare_model_for_qat",+ )+)++# ---- frozen low-precision bases for LoRA (fwd low precision, dgrad dequantized, no wgrad) -------------+_add(+ Recipe(+ name = "frozen_fp8_rowwise",+ kind = "frozen_base",+ modes = ("lora",),+ fwd = GemmPolicy(_op(E4M3, "row"), _op(E4M3, "row")),+ backend = "unsloth.kernels.fp8 (load_in_fp8 = True)",+ gates = {+ "sm75": _NO_FP8_HW,+ "sm80": _NO_FP8_HW,+ "sm89": Gate("named"),+ "sm90": Gate("named"),+ "sm100": Gate("named"),+ "sm120": Gate("named"),+ },+ notes = "dgrad dequantizes the fp8 weight to 16 bit; fast accum in the forward",+ )+)+_add(+ Recipe(+ name = "frozen_fp8_block",+ kind = "frozen_base",+ modes = ("lora",),+ fwd = GemmPolicy(_op(E4M3, "block1x128"), _op(E4M3, "block128x128")),+ backend = "unsloth.kernels.fp8 (load_in_fp8 = 'block')",+ gates = {+ "sm75": _NO_FP8_HW,+ "sm80": _NO_FP8_HW,+ "sm89": Gate("named"),+ "sm90": Gate("named"),+ "sm100": Gate("named"),+ "sm120": Gate("named"),+ },+ notes = "Triton block kernels, forward only; dgrad dequantizes",+ )+)+++def get_recipe(name: str) -> Recipe:+ try:+ recipe = RECIPES[name]+ except KeyError:+ raise KeyError(+ f"Unsloth: unknown precision recipe {name!r}; known: {sorted(RECIPES)}"+ ) from None+ return RECIPES[recipe.alias_of] if recipe.alias_of else recipe+++def recipes_of_kind(kind: str):+ return {n: r for n, r in RECIPES.items() if r.kind == kind}+++# float8_training = '<name>' accepts these spellings; a name maps to its recipe.+FLOAT8_TRAINING_NAMES = {+ "tensorwise": "fp8_tensorwise",+ "rowwise": "fp8_rowwise",+ "rowwise_with_gw_hp": "fp8_rowwise_gw_hp",+ "blockwise": "fp8_blockwise",+ "mxfp8": "mxfp8",+ "mxfp6": "mxfp6",+ "nvfp4": "nvfp4",+}+++def named_recipes(kind: str = "float8_training"):+ """Float8-training names that run by name somewhere (some gate is `named`)."""+ out = []+ for short, full in FLOAT8_TRAINING_NAMES.items():+ r = RECIPES[full]+ if (+ r.kind == kind+ and r.status == "shipped"+ and any(g.status == "named" for g in r.gates.values())+ ):+ out.append(short)+ return tuple(out)+++_CLASS_CC = {+ "sm120": ((12, 0), (12, 1)),+ "sm100": ((10, 0), (10, 3)),+ "sm90": ((9, 0),),+ "sm89": ((8, 9),),+}+++def measured_faster(kind: str = "float8_training"):+ """{short name: compute capabilities where it runs by name}, the shape float8_training._MEASURED_FASTER had."""+ out = {}+ for short in named_recipes(kind):+ r = RECIPES[FLOAT8_TRAINING_NAMES[short]]+ ccs = tuple(+ cc+ for g, gate in r.gates.items()+ if gate.status == "named"+ for cc in _CLASS_CC.get(g, ())+ )+ if ccs:+ out[short] = ccs+ return outdiff --git a/unsloth/precision/selectors.py b/unsloth/precision/selectors.pynew file mode 100644index 00000000000..6c5a19e6df9--- /dev/null+++ b/unsloth/precision/selectors.py@@ -0,0 +1,350 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""High-precision layer selection by structure: output heads (and weights tied to them), embeddings, MoE+routers and the first / last decoder layers. Names are a fallback only: a router called `wg` or `gate_proj`,+or one in remote code, is found by its shape and its place beside the experts."""++from __future__ import annotations++from typing import Dict, Optional++import torch++from .recipes import HPLayerPolicy++__all__ = [+ "num_experts_of",+ "find_routers",+ "find_output_heads",+ "find_embeddings",+ "decoder_layers",+ "high_precision_layers",+ "hp_filter",+]++_EXPERT_COUNT_KEYS = (+ "num_experts",+ "num_local_experts",+ "n_routed_experts",+ "moe_num_experts",+ "num_routed_experts",+)+_SUBCONFIG_KEYS = ("text_config", "language_config", "llm_config", "decoder")+++def num_experts_of(config) -> Optional[int]:+ """Number of routed experts from a (possibly nested) HF config, None for a dense model."""+ seen = set()+ stack = [config]+ while stack:+ c = stack.pop()+ if c is None or id(c) in seen:+ continue+ seen.add(id(c))+ for key in _EXPERT_COUNT_KEYS:+ v = c.get(key) if isinstance(c, dict) else getattr(c, key, None)+ if isinstance(v, int) and not isinstance(v, bool) and v > 1:+ return v+ for key in _SUBCONFIG_KEYS:+ stack.append(c.get(key) if isinstance(c, dict) else getattr(c, key, None))+ return None+++def _experts_count(module) -> Optional[int]:+ """Expert count of an experts container: ModuleList / ModuleDict length (SwitchTransformers keeps its+ experts in a ModuleDict), `num_experts` attribute, or the leading dim of its 3-D parameters (stacked+ expert weights)."""+ if isinstance(module, (torch.nn.ModuleList, torch.nn.ModuleDict)) and len(module) > 1:+ return len(module)+ for key in _EXPERT_COUNT_KEYS:+ v = getattr(module, key, None)+ if isinstance(v, int) and not isinstance(v, bool) and v > 1:+ return v+ for p in module.parameters(recurse = False):+ if p.dim() == 3 and p.shape[0] > 1:+ return int(p.shape[0])+ return None+++def _out_features(module) -> Optional[int]:+ w = getattr(module, "weight", None)+ if isinstance(w, torch.Tensor) and w.dim() == 2:+ return int(w.shape[0])+ return getattr(module, "out_features", None)+++def _router_width(module) -> Optional[int]:+ """Output width of a router: its own 2-D weight, else the one 2-D weight inside it (a gating module that+ wraps a Linear, GraniteMoE's `router.layer`)."""+ out = _out_features(module)+ if out is not None:+ return out+ inner = [p for p in module.parameters() if p.dim() == 2]+ return int(inner[0].shape[0]) if len(inner) == 1 else None+++def _is_stacked_experts(module, n_cfg) -> bool:+ """A container holding all experts' weights as 3-D parameters [num_experts, ...] (GraniteMoE's+ ParallelExperts `input_linear`), whatever it is called. Convolutions (Mamba's conv1d) are 3-D too."""+ if not n_cfg or isinstance(module, torch.nn.modules.conv._ConvNd):+ return False+ return any(p.dim() == 3 and p.shape[0] == n_cfg for p in module.parameters(recurse = False))+++def _is_experts_child(name, module, n_cfg) -> bool:+ return ("expert" in name.lower() and "shared" not in name.lower()) or _is_stacked_experts(+ module, n_cfg+ )+++def _expert_subtrees(model, n_cfg) -> set:+ """ids of modules inside an individual expert or a shared expert: the subtree of every experts child+ (`experts.3`, `experts.expert_0`) and of every shared-expert MLP. An MoE container that owns stacked+ expert weights in its children, and so routes over them itself (JetMoE's attention `experts`, whose+ router is `experts.router`), is not a collection of experts and stays inspectable."""+ inside = set()+ for _, parent in model.named_modules():+ for name, child in parent.named_children():+ if "shared" not in name.lower() and not _is_experts_child(name, child, n_cfg):+ continue+ if "shared" not in name.lower() and any(+ _is_stacked_experts(c, n_cfg) for c in child.children()+ ):+ continue+ inside.update(id(m) for m in child.modules())+ return inside+++def find_routers(model) -> Dict[str, str]:+ """{fqn: reason} for every module that scores tokens over experts.++ A module is a router when its output width equals the expert count and it sits in the same block as an+ experts container (a child whose name contains 'expert', excluding shared experts, or one holding stacked+ 3-D expert weights). With no experts container in its block, the config's expert count plus a 2-D weight+ whose width matches is enough only for a module whose leaf name is a known router name."""+ config = getattr(model, "config", None)+ n_cfg = num_experts_of(config) if config is not None else None+ found = {}+ inside_experts = _expert_subtrees(model, n_cfg)+ for parent_name, parent in model.named_modules():+ # Inside an expert or shared expert (`experts.3`, `shared_expert`): its projections can be as wide as+ # the expert count (Qwen3-Next: 512 experts, moe_intermediate_size 512) but never route.+ if id(parent) in inside_experts:+ continue+ experts = [(n, c) for n, c in parent.named_children() if _is_experts_child(n, c, n_cfg)]+ counts = {_experts_count(c) for _, c in experts} - {None}+ if not counts and n_cfg is None:+ continue+ widths = counts or {n_cfg}+ for child_name, child in parent.named_children():+ if any(child is c for _, c in experts) or "shared" in child_name.lower():+ continue+ out = _router_width(child) if counts else _out_features(child)+ if out is None or out not in widths:+ continue+ # No experts container beside it: only a router name counts; `gate_proj` is excluded here+ # because a dense MLP's gate_proj can be as wide as the expert count.+ if not counts and child_name.lower() not in (+ "gate",+ "router",+ "wg",+ "router_proj",+ "score",+ ):+ continue+ fqn = f"{parent_name}.{child_name}" if parent_name else child_name+ found[fqn] = (+ f"router: {out} outputs = expert count, beside {experts[0][0] if experts else 'config'}"+ )+ return found+++def find_output_heads(model) -> Dict[str, str]:+ """Output embedding modules, and every Linear sharing its weight storage."""+ heads = {}+ out = None+ try:+ out = model.get_output_embeddings()+ except Exception:+ out = None+ ptrs = set()+ if out is not None and isinstance(getattr(out, "weight", None), torch.Tensor):+ try:+ ptrs.add(out.weight.data_ptr())+ except Exception:+ pass+ # No output embedding (sequence classification, reward / value heads): the head is every Linear outside+ # the base model (`score`, `classifier`).+ base = getattr(model, "base_model", None) if out is None else None+ outside = set()+ if isinstance(base, torch.nn.Module) and base is not model:+ inside = {id(m) for m in base.modules()}+ outside = {id(m) for m in model.modules() if id(m) not in inside}+ for name, module in model.named_modules():+ if out is not None and module is out:+ heads[name] = "output head (get_output_embeddings)"+ continue+ if id(module) in outside and isinstance(module, torch.nn.Linear):+ heads[name] = "task head outside the base model"+ continue+ w = getattr(module, "weight", None)+ if isinstance(module, torch.nn.Linear) and isinstance(w, torch.Tensor) and ptrs:+ try:+ if not w.is_meta and w.data_ptr() in ptrs:+ heads[name] = "weight tied to the output head"+ except Exception:+ pass+ return heads+++def find_embeddings(model) -> Dict[str, str]:+ emb = {}+ inp = None+ try:+ inp = model.get_input_embeddings()+ except Exception:+ inp = None+ for name, module in model.named_modules():+ if isinstance(module, torch.nn.Embedding) or (inp is not None and module is inp):+ emb[name] = "embedding"+ return emb+++_NON_DECODER_PARTS = (+ "vision",+ "visual",+ "image",+ "audio",+ "speech",+ "encoder",+ "tower",+ "projector",+)+++def _is_block_stack(module) -> bool:+ """A non-empty ModuleList whose entries each hold a Linear inside them (not a list of bare Linears such as+ Gemma3n's `altup_projections`). Hybrid stacks (Jamba, Zamba) mix block classes, which is fine."""+ if not isinstance(module, torch.nn.ModuleList) or len(module) < 1:+ return False+ return all(+ any(isinstance(m, torch.nn.Linear) for m in block.modules() if m is not block)+ for block in module+ )+++def _language_roots(model):+ """(fqn, module) of the language decoder: the innermost of HF `get_decoder()` (when it names a proper+ submodule) and the nearest ancestor of the input embedding that holds a block stack; else the model."""+ modules = dict(model.named_modules())+ roots = []+ get_decoder = getattr(model, "get_decoder", None)+ try:+ decoder = get_decoder() if callable(get_decoder) else None+ except Exception:+ decoder = None+ if isinstance(decoder, torch.nn.Module) and decoder is not model:+ roots += [(n, m) for n, m in modules.items() if m is decoder][:1]+ try:+ inp = model.get_input_embeddings()+ except Exception:+ inp = None+ emb = next((n for n, m in modules.items() if inp is not None and m is inp), None)+ if emb:+ parts = emb.split(".")+ for i in range(len(parts) - 1, -1, -1):+ name = ".".join(parts[:i])+ if any(_is_block_stack(m) for m in modules[name].modules()):+ roots.append((name, modules[name]))+ break+ # Keep a root only when no other candidate lies inside it (Idefics3: get_decoder() is the whole+ # multimodal model, the embedding's ancestor is its text model). Unrelated roots: get_decoder() wins+ # (T5: the shared embedding sits above both stacks).+ roots = [+ (n, m)+ for n, m in roots+ if not any(o != n and (not n or o.startswith(n + ".")) for o, _ in roots)+ ]+ return roots[0] if roots else ("", model)+++def decoder_layers(model):+ """(fqn prefix of the decoder layer list, number of layers): the longest ModuleList of blocks that each+ hold Linear layers inside the language decoder (the stack beside the input embedding, or HF+ `get_decoder()`), ignoring experts lists and any list nested inside another such list (an MoE model can+ have more experts per block than blocks). Searching the whole model, vision / audio / encoder stacks+ lose to any other stack; an encoder-only model (BERT) gets its single stack. One block is a stack.+ (None, 0) when there is none."""+ root, decoder = _language_roots(model)+ candidates = []+ for name, module in decoder.named_modules():+ name = f"{root}.{name}" if root and name else (root or name)+ if not _is_block_stack(module):+ continue+ if any("expert" in part.lower() for part in name.split(".")):+ continue+ candidates.append((name, len(module)))+ # A list inside a block of another candidate list (`layers.0.<...>`) is part of that block, not the stack.+ outer = [+ (name, n)+ for name, n in candidates+ if not any(o != name and o and name.startswith(o + ".") for o, _ in candidates)+ ]+ if not root:+ language = [+ (name, n)+ for name, n in outer+ if not any(p.startswith(_NON_DECODER_PARTS) for p in name.lower().split("."))+ ]+ outer = language or outer+ best = (None, 0)+ for name, n in outer:+ # A tie goes to the shallower list.+ if n > best[1] or (+ n == best[1] and best[0] is not None and name.count(".") < best[0].count(".")+ ):+ best = (name, n)+ return best+++def high_precision_layers(model, policy: HPLayerPolicy) -> Dict[str, str]:+ """{fqn: reason} of modules the policy keeps in high precision. A prefix entry (decoder layer) covers+ every module under it."""+ keep = {}+ if policy.output_head:+ keep.update(find_output_heads(model))+ if policy.embeddings:+ keep.update(find_embeddings(model))+ if policy.routers:+ keep.update(find_routers(model))+ if policy.first_n or policy.last_n or policy.last_frac:+ prefix, n = decoder_layers(model)+ if prefix is not None and n:+ import math+ last = max(policy.last_n, math.ceil(policy.last_frac * n) if policy.last_frac else 0)+ for i in range(n):+ if i < policy.first_n:+ keep[f"{prefix}.{i}"] = f"first {policy.first_n} decoder layers"+ elif i >= n - last:+ keep[f"{prefix}.{i}"] = f"last {last} of {n} decoder layers"+ if policy.shared_experts:+ for name, _ in model.named_modules():+ if "shared_expert" in name.rsplit(".", 1)[-1]:+ keep[name] = "shared expert"+ return keep+++def hp_filter(model, policy: HPLayerPolicy):+ """Predicate (module, fqn) -> True when the module (or a block containing it) stays in high precision."""+ keep = high_precision_layers(model, policy)+ exact = set(keep)++ def is_hp(module, fqn: str) -> bool:+ if fqn in exact:+ return True+ parts = fqn.split(".")+ return any(".".join(parts[:i]) in exact for i in range(1, len(parts)))++ is_hp.keep = keep+ return is_hp