FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(quant): load ModelOpt exports that ship no input_scale (#462)
Xiaoze Fanf7dbab7f151d
diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.pyindex c92e1d88..2f0c89fb 100644--- a/python/freetoken/engine/engine.py+++ b/python/freetoken/engine/engine.py@@ -281,6 +281,9 @@ def _materialize_loaded_weight_state_dict(for key, weight in weights:expected = model_state.get(key)if expected is None:+ # NOTE: the quant scheme may declare no input_scale for a layer whose FTW still stores one+ if key.endswith(".input_scale"):+ continuestate_dict[key] = weight.to(device=device)else:state_dict[key] = weight.to(device=device, dtype=expected.dtype)diff --git a/python/freetoken/layers/quantization/configs/base.py b/python/freetoken/layers/quantization/configs/base.pyindex 2a4ef432..e84dcbcb 100644--- a/python/freetoken/layers/quantization/configs/base.py+++ b/python/freetoken/layers/quantization/configs/base.py@@ -98,15 +98,11 @@ class QuantConfig(ABC):*,name_map: NameMap | None = None,unquantized: tuple[str, ...] = (),- hf_quant_config: dict[str, Any] | None = None,) -> "QuantConfig":- """``hf_quant_config`` is the parsed ``hf_quant_config.json`` of ModelOpt exports that keep- no ``quantization_config`` in config.json (modelopt < 0.41); ``unquantized`` lists the modules- the family keeps bf16 when the checkpoint's config does not (DeepSeek-V4's compressors)."""+ """``unquantized`` lists the modules the family keeps bf16 when the checkpoint's config does not (DeepSeek-V4's compressors).++ ``hf_config`` comes from ``cached_load_hf_config``, which folds an old ModelOpt ``hf_quant_config.json`` into ``quantization_config``."""q = quantization_config_of(hf_config)- if q is None and hf_quant_config and isinstance(hf_quant_config.get("quantization"), dict):- q = dict(hf_quant_config["quantization"])- q.setdefault("quant_method", "modelopt")if q is None:return NoQuantConfig(name_map, unquantized)for cls in dialects():diff --git a/python/freetoken/layers/quantization/configs/modelopt.py b/python/freetoken/layers/quantization/configs/modelopt.pyindex 1d89b235..6183dfc9 100644--- a/python/freetoken/layers/quantization/configs/modelopt.py+++ b/python/freetoken/layers/quantization/configs/modelopt.py@@ -18,7 +18,7 @@ class ModelOptConfig(QuantConfig):SCHEMES: ClassVar[dict[str, QuantScheme]] = {"NVFP4": nvfp4_scheme(input_scale=True),- "NVFP4_NO_INPUT": nvfp4_scheme(input_scale=False),+ "W4A16_NVFP4": nvfp4_scheme(input_scale=False),"FP8": fp8_tensor_scheme("fp32", input_scale=True),"FP8_PER_CHANNEL_PER_TOKEN": fp8_tensor_scheme("fp32", per_row=True),"FP8_PB_WO": fp8_block_scheme("fp32"),@@ -39,10 +39,15 @@ class ModelOptConfig(QuantConfig):def __init__(self, q: dict[str, Any], hf_config: Any = None, *, name_map=None, unquantized=()):super().__init__(name_map, unquantized)self.algo = str(q.get("quant_algo") or "").upper()+ # Inferact exports carry this key; false says the FP8 and NVFP4 layers store no input_scale, and an explicit value wins over the config_groups rule below+ self.with_input_scale = q.get("with_input_scale")+ groups = q.get("config_groups")+ # an export with no activation quantizer can still say NVFP4; every config group then has input_activations null (vLLM applies the same rule)+ if self.with_input_scale is None and self.algo == "NVFP4" and isinstance(groups, dict) and groups and all(isinstance(g, dict) and g.get("input_activations") is None for g in groups.values()):+ self.algo = "W4A16_NVFP4"self.ignore = name_set(tuple(q.get("ignore") or q.get("exclude_modules") or ()))layers = q.get("quantized_layers") or {}self.quantized_layers = {k: str((v or {}).get("quant_algo") or "").upper() for k, v in layers.items()} if isinstance(layers, dict) else {}- self.with_input_scale = bool(q.get("with_input_scale", True))if self.algo == "MIXED_PRECISION" and not self.quantized_layers:raise NotImplementedError("ModelOpt MIXED_PRECISION without quantized_layers in quantization_config")if self.algo != "MIXED_PRECISION":@@ -64,8 +69,11 @@ class ModelOptConfig(QuantConfig):return Nonedef _scheme_of(self, algo: str) -> QuantScheme:- if algo in ("NVFP4", "W4A16_NVFP4"):- return self.SCHEMES["NVFP4" if self.with_input_scale else "NVFP4_NO_INPUT"]+ if self.with_input_scale is False:+ if algo == "FP8":+ return fp8_tensor_scheme("fp32") # ModelOpt has no algo name for fp8 without an activation scale+ if algo == "NVFP4":+ algo = "W4A16_NVFP4"try:return self.SCHEMES[algo]except KeyError:diff --git a/python/freetoken/models/qwen3_5_moe/weight.py b/python/freetoken/models/qwen3_5_moe/weight.pyindex c74117b3..69501ad7 100644--- a/python/freetoken/models/qwen3_5_moe/weight.py+++ b/python/freetoken/models/qwen3_5_moe/weight.py@@ -118,7 +118,7 @@ class _DenseReader:for idx, part in enumerate(parts):self.by_part.setdefault(part, []).append((fused, idx))# target module -> (part count, {part: {role: tensor}}, {part: the roles its module stores})- self.pending: dict[str, tuple[int, dict[int, dict[str, torch.Tensor]], dict[int, set[str]]]] = {}+ self.pending: dict[str, tuple[int, dict[int, dict[str, torch.Tensor]], dict[int, set[str]], QuantScheme | None]] = {}def scheme(self, module: str) -> QuantScheme | None:return None if self.quant is None else self.quant.scheme_for(module)@@ -159,8 +159,10 @@ class _DenseReader:)if stored is None and tensor.dtype in _QUANT_DTYPES:raise ValueError(f"{name} is {tensor.dtype} but the checkpoint's quant config declares {module} unquantized")+ if stored is not None and role == "weight" and tensor.dtype is not _ELEM_DTYPES[stored.weight.elem]:+ raise ValueError(f"{name} is {tensor.dtype} but the checkpoint's quant config declares {module} {stored}")target, idx, count = self.target(module)- _, parts, expected = self.pending.setdefault(target, (count, {}, {}))+ _, parts, expected, _ = self.pending.setdefault(target, (count, {}, {}, stored))parts.setdefault(idx, {})[role] = tensorexpected[idx] = set(roles.values())if len(parts) < count or any(set(parts[i]) != expected[i] for i in parts):@@ -168,6 +170,20 @@ class _DenseReader:del self.pending[target]return self._emit(target, [parts[i] for i in range(count)], stored)+ def missing(self) -> list[str]:+ """One line per incomplete module: the roles its parts still lack."""+ lines = []+ for target, (count, parts, expected, stored) in sorted(self.pending.items()):+ lacking = sorted(set().union(*(expected[i] - set(parts[i]) for i in parts)))+ note = ""+ if lacking == ["input_scale"]:+ fix = "declares W4A16_NVFP4 or sets with_input_scale false" if stored is not None and stored.kind is QuantKind.NVFP4 else "sets with_input_scale false"+ note = f" (an export without activation scales {fix})"+ if len(parts) < count:+ lacking.append(f"{count - len(parts)} of {count} fused parts")+ lines.append(f"{target}: missing {lacking}{note}")+ return lines+def _emit(self, target: str, parts: list[dict[str, torch.Tensor]], stored: QuantScheme | None):if stored is not None:parts = [self._check(target, stored, part) for part in parts]@@ -265,7 +281,9 @@ def _iter_shards(model_path: str, device: torch.device, reader: _DenseReader | Nelse:yield name, tensorif reader is not None and reader.pending:- raise ValueError(f"checkpoint is missing tensors of {sorted(reader.pending)}")+ lines = reader.missing()+ shown = "\n ".join(lines[:8]) + (f"\n ... {len(lines) - 8} more" if len(lines) > 8 else "")+ raise ValueError(f"checkpoint is missing tensors the quant config declares for {len(lines)} modules:\n {shown}")def iter_weights_parallel(diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.pyindex d0666f9d..2396bb64 100644--- a/python/freetoken/models/register.py+++ b/python/freetoken/models/register.py@@ -300,23 +300,13 @@ def _load_attr(module_path: str, attr_name: str) -> Any:def checkpoint_quant_config(model_path: str, hf_config: Any, spec: ModelSpec):"""The checkpoint's QuantConfig under the family's naming, or None for GGUF, whose native-quant ops the shared parser does not model yet."""from freetoken.layers.quantization import NameMap, QuantConfig- from freetoken.utils.hf import optional_hf_fileif spec.parse_config == "parse_gguf_config":return None- # NOTE: ModelOpt exports before 0.41 keep the quantization config only in hf_quant_config.json, and the weight download fetches nothing but the safetensors shards, so this sidecar is fetched on its own.- hf_quant_config = None- sidecar = optional_hf_file(model_path, "hf_quant_config.json")- if sidecar is not None:- import json-- with open(sidecar) as f:- hf_quant_config = json.load(f)return QuantConfig.from_hf(hf_config,name_map=NameMap(roots=spec.checkpoint_roots, segments=spec.checkpoint_segments, packed=spec.packed_modules_mapping),unquantized=spec.unquantized_modules,- hf_quant_config=hf_quant_config,)diff --git a/python/freetoken/utils/hf.py b/python/freetoken/utils/hf.pyindex a28146e6..b93d3b12 100644--- a/python/freetoken/utils/hf.py+++ b/python/freetoken/utils/hf.py@@ -191,19 +191,48 @@ def _raw_config_json(model_path: str) -> dict:return json.load(f)+def sidecar_quantization_config(model_path: str) -> dict | None:+ """The ``quantization_config`` an old ModelOpt export keeps only in ``hf_quant_config.json``, or None."""+ sidecar = optional_hf_file(model_path, "hf_quant_config.json")+ if sidecar is None:+ return None+ with open(sidecar, encoding="utf-8") as f:+ quant = json.load(f).get("quantization")+ if not isinstance(quant, dict):+ return None+ return {"quant_method": "modelopt", **quant}+++def _merge_sidecar_quantization_config(config: Any, model_path: str) -> None:+ # config.json wins when it has one; the sidecar is only read for exports that never wrote it there+ if getattr(config, "quantization_config", None) is not None:+ return+ if getattr(getattr(config, "text_config", None), "quantization_config", None) is not None:+ return+ quant = sidecar_quantization_config(model_path)+ if quant is None:+ return+ if isinstance(config, RawConfigShim):+ config._data["quantization_config"] = quant+ else:+ config.quantization_config = quant++@functools.cachedef _load_hf_config(model_path: str) -> Any:# trust_remote_code: checkpoints that ship a custom config class via ``auto_map``# (e.g. MiniMax-M2) refuse to load without it. FreeToken only reads config fields# (parse_config) and never instantiates the checkpoint's modeling code.try:- return AutoConfig.from_pretrained(model_path, trust_remote_code=True)+ config = AutoConfig.from_pretrained(model_path, trust_remote_code=True)except ValueError as exc:# Unknown model_type on this transformers version: serve off the raw JSON.# Anything else (bad path, malformed JSON) stays fatal.if "model type" not in str(exc):raise- return RawConfigShim(_raw_config_json(model_path), _name_or_path=model_path)+ config = RawConfigShim(_raw_config_json(model_path), _name_or_path=model_path)+ _merge_sidecar_quantization_config(config, model_path)+ return configdef cached_load_hf_config(model_path: str) -> PretrainedConfig:diff --git a/tests/models/test_quant_config.py b/tests/models/test_quant_config.pyindex 73aad10c..6ac1b84b 100644--- a/tests/models/test_quant_config.py+++ b/tests/models/test_quant_config.py@@ -43,6 +43,7 @@ from freetoken.layers.quantization.moe import Fp8BlockMoEMethod, Mxfp4MoEMethod,from freetoken.models import create_modelfrom freetoken.models.gpt_oss.moe import GptOssMoELayer, GptOssOffloadMoELayerfrom freetoken.models.register import checkpoint_quant_config, get_model_spec+from freetoken.utils.hf import sidecar_quantization_configfrom freetoken.utils.torch_utils import torch_dtypeMODELS = "/mnt/nvme/models"@@ -391,21 +392,15 @@ def test_probed_layers_get_the_method_their_config_says(case: Case, monkeypatch)# --------------------------------------------------------------------------- config without local weights-def test_hub_ids_fetch_the_modelopt_sidecar(tmp_path, monkeypatch):- """An old ModelOpt export keeps its quantization config only in hf_quant_config.json."""- from huggingface_hub.utils import EntryNotFoundError+def test_the_modelopt_sidecar_is_folded_into_the_hf_config(tmp_path):+ """An old ModelOpt export keeps its quantization config only in hf_quant_config.json; the config loader folds it in, so every reader of the config sees it."""+ from freetoken.utils import cached_load_hf_config- sidecar = tmp_path / "hf_quant_config.json"- sidecar.write_text(json.dumps({"producer": {"name": "modelopt", "version": "0.29.0"}, "quantization": {"quant_algo": "FP8", "exclude_modules": ["lm_head"]}}))-- def fake_download(repo_id, filename, **_):- assert repo_id == "org/old-modelopt-fp8"- if filename == "hf_quant_config.json":- return str(sidecar)- raise EntryNotFoundError(f"no {filename}")-- monkeypatch.setattr("freetoken.utils.hf.hf_hub_download", fake_download)- quant = checkpoint_quant_config("org/old-modelopt-fp8", SimpleNamespace(architectures=["Qwen3MoeForCausalLM"]), get_model_spec("Qwen3MoeForCausalLM"))+ (tmp_path / "config.json").write_text(json.dumps({"model_type": "qwen3_moe", "architectures": ["Qwen3MoeForCausalLM"]}))+ (tmp_path / "hf_quant_config.json").write_text(json.dumps({"producer": {"name": "modelopt", "version": "0.29.0"}, "quantization": {"quant_algo": "FP8", "exclude_modules": ["lm_head"]}}))+ hf = cached_load_hf_config(str(tmp_path))+ assert hf.quantization_config == {"quant_method": "modelopt", "quant_algo": "FP8", "exclude_modules": ["lm_head"]}+ quant = checkpoint_quant_config(str(tmp_path), hf, get_model_spec("Qwen3MoeForCausalLM"))assert type(quant) is ModelOptConfigassert quant.scheme_for("model.layers.0.self_attn.q_proj").kind is QuantKind.FP8_TENSORassert quant.scheme_for("lm_head") is None@@ -427,6 +422,23 @@ def test_compressed_tensors_ignore_names_the_module_alone():assert quant.scheme_for("model.language_model.layers.0.linear_attn") is None+_NVFP4_GROUP = {"targets": ["Linear"], "weights": {"num_bits": 4, "type": "float", "group_size": 16}}++[email protected]("extra, has_input_scale", [+ ({"config_groups": {"group_0": {**_NVFP4_GROUP, "input_activations": None}}}, False),+ ({"config_groups": {"group_0": {**_NVFP4_GROUP, "input_activations": {"num_bits": 4, "type": "float", "group_size": 16}}}}, True),+ ({}, True),+ ({"with_input_scale": False}, False),+ ({"config_groups": {"group_0": {**_NVFP4_GROUP, "input_activations": None}}, "with_input_scale": True}, True),+], ids=["no input_activations", "input_activations", "no config_groups", "with_input_scale false", "with_input_scale true wins"])+def test_modelopt_nvfp4_reads_the_activation_quantizer_from_config_groups(extra, has_input_scale):+ """A ModelOpt export can say ``NVFP4`` with no activation quantizer: every config group then has ``input_activations`` null (vLLM #54427); an explicit ``with_input_scale`` wins."""+ q = {"quant_method": "modelopt", "quant_algo": "NVFP4", **extra}+ scheme = QuantConfig.from_hf(SimpleNamespace(quantization_config=q)).scheme_for("model.layers.0.mlp.down_proj")+ assert scheme.kind is QuantKind.NVFP4 and scheme.has("input_scale") is has_input_scale++def test_every_dialect_names_the_tensors_behind_its_schemes():from freetoken.layers.quantization.registry import dialects@@ -570,12 +582,13 @@ def test_scheme_for_agrees_with_the_stored_tensors(ckpt: Path):with pytest.raises(NotImplementedError):QuantConfig.from_hf(cfg)return- hfq = json.load(open(ckpt / "hf_quant_config.json")) if (ckpt / "hf_quant_config.json").exists() else None+ if not quant:+ cfg["quantization_config"] = sidecar_quantization_config(str(ckpt))try:spec = get_model_spec((cfg.get("architectures") or [""])[0])except Exception:spec = None- qc = QuantConfig.from_hf(cfg, hf_quant_config=hfq, unquantized=spec.unquantized_modules if spec else ())+ qc = QuantConfig.from_hf(cfg, unquantized=spec.unquantized_modules if spec else ())weight_map = _weight_map(ckpt)tensors = _tensor_info(ckpt, weight_map)mismatches, checked = [], 0diff --git a/tests/models/test_qwen3_5_moe_weight.py b/tests/models/test_qwen3_5_moe_weight.pyindex f5dc2c3c..158f1f45 100644--- a/tests/models/test_qwen3_5_moe_weight.py+++ b/tests/models/test_qwen3_5_moe_weight.py@@ -175,6 +175,10 @@ MODELOPT_MIXED = {**{f"{LM}.layers.{l}.mlp.experts": {"quant_algo": "NVFP4", "group_size": 16} for l in (0, 1)},},}+# ornith-ai/Ornith-1.5-35B-A3B-NVFP4: the same mix with W4A16 experts+MODELOPT_MIXED_A16 = {**MODELOPT_MIXED, "quantized_layers": {+ **MODELOPT_MIXED["quantized_layers"], **{f"{LM}.layers.{l}.mlp.experts": {"quant_algo": "W4A16_NVFP4", "group_size": 16} for l in (0, 1)}}}+MODELOPT_MIXED_NOINPUT = {**MODELOPT_MIXED, "with_input_scale": False}# sakamakismile/Qwen3.6-27B-NVFP4: every Linear of the dense model, GDN in_proj includedCT_NVFP4_DENSE = _ct({"group_0": {**NVFP4_GROUP, "targets": ["Linear"]}}, ["lm_head"], "nvfp4-pack-quantized")# RedHatAI/Qwen3.6-35B-A3B-NVFP4: every Linear but the GDN, the routers and lm_head@@ -209,11 +213,18 @@ def _layout(name: str) -> tuple[bool, dict | None, dict[str, torch.Tensor]]:_quantize(raw, GDN_QKVZ_OUT + ATTN + SHARED, lambda w: _fp8_block(w, ct=False))_experts(raw, lambda w: _fp8_block(w, ct=False))return moe, QWEN_FP8, raw- if name == "modelopt_mixed":+ if name.startswith("modelopt_mixed"):_quantize(raw, GDN_QKVZ_OUT + ATTN, _fp8_tensor)_quantize(raw, SHARED + ["lm_head"], lambda w: _nvfp4(w, ct=False))_experts(raw, lambda w: _nvfp4(w, ct=False))- return moe, MODELOPT_MIXED, raw+ # _a16: the W4A16 layers ship no activation scale (ornith-ai); _noinput: no layer does and the config says with_input_scale false (jhone888)+ if name == "modelopt_mixed_a16":+ stripped = lambda k: ".mlp." in k or k.startswith("lm_head")+ else:+ stripped = lambda k: name == "modelopt_mixed_noinput"+ for key in [k for k in raw if k.endswith(".input_scale") and stripped(k)]:+ del raw[key]+ return moe, {"modelopt_mixed_a16": MODELOPT_MIXED_A16, "modelopt_mixed_noinput": MODELOPT_MIXED_NOINPUT}.get(name, MODELOPT_MIXED), rawif name == "ct_nvfp4_dense":_quantize(raw, GDN_QKVZ_OUT + GDN_BA + ATTN + DENSE_MLP, lambda w: _nvfp4(w, ct=True))return moe, CT_NVFP4_DENSE, raw@@ -242,7 +253,7 @@ def _layout(name: str) -> tuple[bool, dict | None, dict[str, torch.Tensor]]:raise KeyError(name)-LAYOUTS = ["bf16", "fp8_block", "modelopt_mixed", "ct_nvfp4_dense", "ct_nvfp4_moe", "ct_mixed_fast", "ct_tensor_fp8_moe", "ct_block_moe", "ct_block_experts"]+LAYOUTS = ["bf16", "fp8_block", "modelopt_mixed", "modelopt_mixed_a16", "modelopt_mixed_noinput", "ct_nvfp4_dense", "ct_nvfp4_moe", "ct_mixed_fast", "ct_tensor_fp8_moe", "ct_block_moe", "ct_block_experts"]def _config_json(moe: bool, quantization_config) -> dict:@@ -401,25 +412,28 @@ def test_block_fp8_fuses_weight_and_scale_per_kind(checkpoint):def test_modelopt_fp8_scales_broadcast_per_part_and_input_scale_is_the_max(checkpoint):name, folder, raw = checkpoint- if name != "modelopt_mixed":- pytest.skip("modelopt layout only")+ if not name.startswith("modelopt_mixed"):+ pytest.skip("modelopt layouts only")loaded = _load(folder)attn = f"{LM}.layers.1.self_attn"scale = loaded["model.layers.1.self_attn.qkv_proj.weight_scale"]assert scale.dtype is torch.float32 and scale.shape == (2 * QH * AHD + 2 * KVH * AHD,)expected = torch.cat([raw[f"{attn}.{p}_proj.weight_scale"].expand(raw[f"{attn}.{p}_proj.weight"].shape[0]) for p in "qkv"])assert torch.equal(scale, expected)- assert torch.equal(loaded["model.layers.1.self_attn.qkv_proj.input_scale"], torch.stack([raw[f"{attn}.{p}_proj.input_scale"] for p in "qkv"]).max())- assert loaded["model.layers.1.self_attn.o_proj.input_scale"].shape == ()- # NVFP4 shared expert: each part keeps its own block scales and global; the input scale is the max of the parts+ if name == "modelopt_mixed_noinput":+ assert not any(k.endswith(".input_scale") for k in loaded)+ else:+ assert torch.equal(loaded["model.layers.1.self_attn.qkv_proj.input_scale"], torch.stack([raw[f"{attn}.{p}_proj.input_scale"] for p in "qkv"]).max())+ assert loaded["model.layers.1.self_attn.o_proj.input_scale"].shape == ()+ # NVFP4 shared expert: each part keeps its own block scales and globalshared = f"{LM}.layers.0.mlp.shared_expert"fused = "model.layers.0.mlp.shared_expert.gate_up_proj"assert _same(loaded[f"{fused}.weight"][:I], raw[f"{shared}.gate_proj.weight"])assert _same(loaded[f"{fused}.weight_scale"][I:], raw[f"{shared}.up_proj.weight_scale"])glob = loaded[f"{fused}.weight_global"]assert glob.dtype is torch.float16 and torch.equal(glob[:I], raw[f"{shared}.gate_proj.weight_scale_2"].to(torch.float16).expand(I))- assert torch.equal(loaded[f"{fused}.input_scale"], torch.stack([raw[f"{shared}.{p}_proj.input_scale"] for p in ("gate", "up")]).max())- assert loaded["lm_head.input_scale"].shape == ()+ # W4A16: no activation quantizer, so the layer has no input_scale even when the export ships one+ assert f"{fused}.input_scale" not in loaded and "lm_head.input_scale" not in loadedassert loaded["lm_head.weight"].dtype is torch.uint8 and loaded["lm_head.weight_global"].shape == (V,)@@ -486,7 +500,7 @@ def test_static_fp8_keeps_the_activation_scale_and_ignored_expert_containers_do_def test_nvfp4_expert_pieces_read_either_dialect_from_a_single_file(checkpoint):name, folder, raw = checkpoint- if name not in ("modelopt_mixed", "ct_nvfp4_moe", "ct_tensor_fp8_moe"):+ if not name.startswith("modelopt_mixed") and name not in ("ct_nvfp4_moe", "ct_tensor_fp8_moe"):pytest.skip("NVFP4 expert layouts only")_install(folder)config = parse_config(cached_load_hf_config(folder))@@ -496,7 +510,7 @@ def test_nvfp4_expert_pieces_read_either_dialect_from_a_single_file(checkpoint):layer, e0, e1, piece = next(p for p in pieces if p[0] == 1 and p[1] == 2)assert (e0, e1) == (2, 3)base = f"{LM}.layers.1.mlp.experts.2.gate_proj"- if name == "modelopt_mixed":+ if name.startswith("modelopt_mixed"):assert _same(piece["gate"][0], raw[f"{base}.weight"])assert torch.equal(piece["gate_global"].reshape(-1), raw[f"{base}.weight_scale_2"].reshape(-1).to(torch.float16))else:@@ -559,13 +573,18 @@ REJECTED = [pytest.param(None, lambda w: {f"{ATTN1}.o_proj.weight": w.to(FP8)},r"o_proj\.weight is torch\.float8", id="fp8 weight the config declares bf16"),pytest.param(MODELOPT_MIXED, lambda w: {f"{ATTN1}.o_proj.weight": w, f"{ATTN1}.o_proj.weight_scale": torch.rand(()), f"{ATTN1}.o_proj.input_scale": torch.rand(())},- r"o_proj: weight is torch\.bfloat16", id="bf16 weight the config declares fp8"),+ r"o_proj\.weight is torch\.bfloat16 but .*declares .*fp8_tensor", id="bf16 weight the config declares fp8"),+ pytest.param({"quant_method": "modelopt", "quant_algo": "W4A16_NVFP4"}, lambda w: {f"{ATTN1}.o_proj.weight": w},+ r"o_proj\.weight is torch\.bfloat16 but .*declares .*nvfp4", id="bf16 weight the config declares nvfp4"),+ pytest.param({"quant_method": "modelopt", "quant_algo": "NVFP4"},+ lambda w: {f"{ATTN1}.o_proj.weight": torch.zeros(H, QH * AHD // 2, dtype=torch.uint8), f"{ATTN1}.o_proj.weight_scale": torch.zeros(H, QH * AHD // 16, dtype=FP8), f"{ATTN1}.o_proj.weight_scale_2": torch.rand(()), f"{ATTN1}.k_proj.input_scale": torch.rand(())},+ r"(?s)o_proj: missing \['input_scale'\] \(an export without activation scales declares W4A16_NVFP4", id="w4a4 config with one input_scale missing"),pytest.param(CT_MIXED_FAST, lambda w: {f"{ATTN1}.o_proj.weight": w.to(FP8), f"{ATTN1}.o_proj.weight_scale": torch.rand(3, 1)},"expected 1 or", id="per-channel scale with the wrong row count"),pytest.param(CT_BLOCK_MOE, lambda w: {f"{ATTN1}.o_proj.weight": w.to(FP8), f"{ATTN1}.o_proj.weight_scale": torch.rand(2, 1)},r"weight_scale_inv is \(2, 1\), expected \(1, 2\)", id="block scale of the wrong shape"),pytest.param(CT_NVFP4_MOE, lambda w: {f"{ATTN1}.o_proj.weight_packed": torch.zeros(H, QH * AHD // 2, dtype=torch.uint8), f"{ATTN1}.o_proj.weight_global_scale": torch.rand(1)},- "missing tensors of .*o_proj", id="quantized module without its block scale"),+ r"(?s)missing tensors the quant config declares.*o_proj: missing \['input_scale', 'weight_scale'\]", id="quantized module without its block scale"),]