Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

fix(quant): load ModelOpt exports that ship no input_scale (#462)

Xiaoze Fanf7dbab7f151d
diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.py
index c92e1d88..2f0c89fb 100644
--- a/python/freetoken/engine/engine.py
+++ b/python/freetoken/engine/engine.py
@@ -281,6 +281,9 @@ def _materialize_loaded_weight_state_dict(
for key, weight in weights:
expected = model_state.get(key)
if expected is None:
+ # NOTE: the quant scheme may declare no input_scale for a layer whose FTW still stores one
+ if key.endswith(".input_scale"):
+ continue
state_dict[key] = weight.to(device=device)
else:
state_dict[key] = weight.to(device=device, dtype=expected.dtype)
diff --git a/python/freetoken/layers/quantization/configs/base.py b/python/freetoken/layers/quantization/configs/base.py
index 2a4ef432..e84dcbcb 100644
--- a/python/freetoken/layers/quantization/configs/base.py
+++ b/python/freetoken/layers/quantization/configs/base.py
@@ -98,15 +98,11 @@ class QuantConfig(ABC):
*,
name_map: NameMap | None = None,
unquantized: tuple[str, ...] = (),
- hf_quant_config: dict[str, Any] | None = None,
) -> "QuantConfig":
- """``hf_quant_config`` is the parsed ``hf_quant_config.json`` of ModelOpt exports that keep
- no ``quantization_config`` in config.json (modelopt < 0.41); ``unquantized`` lists the modules
- the family keeps bf16 when the checkpoint's config does not (DeepSeek-V4's compressors)."""
+ """``unquantized`` lists the modules the family keeps bf16 when the checkpoint's config does not (DeepSeek-V4's compressors).
+
+ ``hf_config`` comes from ``cached_load_hf_config``, which folds an old ModelOpt ``hf_quant_config.json`` into ``quantization_config``."""
q = quantization_config_of(hf_config)
- if q is None and hf_quant_config and isinstance(hf_quant_config.get("quantization"), dict):
- q = dict(hf_quant_config["quantization"])
- q.setdefault("quant_method", "modelopt")
if q is None:
return NoQuantConfig(name_map, unquantized)
for cls in dialects():
diff --git a/python/freetoken/layers/quantization/configs/modelopt.py b/python/freetoken/layers/quantization/configs/modelopt.py
index 1d89b235..6183dfc9 100644
--- a/python/freetoken/layers/quantization/configs/modelopt.py
+++ b/python/freetoken/layers/quantization/configs/modelopt.py
@@ -18,7 +18,7 @@ class ModelOptConfig(QuantConfig):
SCHEMES: ClassVar[dict[str, QuantScheme]] = {
"NVFP4": nvfp4_scheme(input_scale=True),
- "NVFP4_NO_INPUT": nvfp4_scheme(input_scale=False),
+ "W4A16_NVFP4": nvfp4_scheme(input_scale=False),
"FP8": fp8_tensor_scheme("fp32", input_scale=True),
"FP8_PER_CHANNEL_PER_TOKEN": fp8_tensor_scheme("fp32", per_row=True),
"FP8_PB_WO": fp8_block_scheme("fp32"),
@@ -39,10 +39,15 @@ class ModelOptConfig(QuantConfig):
def __init__(self, q: dict[str, Any], hf_config: Any = None, *, name_map=None, unquantized=()):
super().__init__(name_map, unquantized)
self.algo = str(q.get("quant_algo") or "").upper()
+ # Inferact exports carry this key; false says the FP8 and NVFP4 layers store no input_scale, and an explicit value wins over the config_groups rule below
+ self.with_input_scale = q.get("with_input_scale")
+ groups = q.get("config_groups")
+ # an export with no activation quantizer can still say NVFP4; every config group then has input_activations null (vLLM applies the same rule)
+ if self.with_input_scale is None and self.algo == "NVFP4" and isinstance(groups, dict) and groups and all(isinstance(g, dict) and g.get("input_activations") is None for g in groups.values()):
+ self.algo = "W4A16_NVFP4"
self.ignore = name_set(tuple(q.get("ignore") or q.get("exclude_modules") or ()))
layers = q.get("quantized_layers") or {}
self.quantized_layers = {k: str((v or {}).get("quant_algo") or "").upper() for k, v in layers.items()} if isinstance(layers, dict) else {}
- self.with_input_scale = bool(q.get("with_input_scale", True))
if self.algo == "MIXED_PRECISION" and not self.quantized_layers:
raise NotImplementedError("ModelOpt MIXED_PRECISION without quantized_layers in quantization_config")
if self.algo != "MIXED_PRECISION":
@@ -64,8 +69,11 @@ class ModelOptConfig(QuantConfig):
return None
def _scheme_of(self, algo: str) -> QuantScheme:
- if algo in ("NVFP4", "W4A16_NVFP4"):
- return self.SCHEMES["NVFP4" if self.with_input_scale else "NVFP4_NO_INPUT"]
+ if self.with_input_scale is False:
+ if algo == "FP8":
+ return fp8_tensor_scheme("fp32") # ModelOpt has no algo name for fp8 without an activation scale
+ if algo == "NVFP4":
+ algo = "W4A16_NVFP4"
try:
return self.SCHEMES[algo]
except KeyError:
diff --git a/python/freetoken/models/qwen3_5_moe/weight.py b/python/freetoken/models/qwen3_5_moe/weight.py
index c74117b3..69501ad7 100644
--- a/python/freetoken/models/qwen3_5_moe/weight.py
+++ b/python/freetoken/models/qwen3_5_moe/weight.py
@@ -118,7 +118,7 @@ class _DenseReader:
for idx, part in enumerate(parts):
self.by_part.setdefault(part, []).append((fused, idx))
# target module -> (part count, {part: {role: tensor}}, {part: the roles its module stores})
- self.pending: dict[str, tuple[int, dict[int, dict[str, torch.Tensor]], dict[int, set[str]]]] = {}
+ self.pending: dict[str, tuple[int, dict[int, dict[str, torch.Tensor]], dict[int, set[str]], QuantScheme | None]] = {}
def scheme(self, module: str) -> QuantScheme | None:
return None if self.quant is None else self.quant.scheme_for(module)
@@ -159,8 +159,10 @@ class _DenseReader:
)
if stored is None and tensor.dtype in _QUANT_DTYPES:
raise ValueError(f"{name} is {tensor.dtype} but the checkpoint's quant config declares {module} unquantized")
+ if stored is not None and role == "weight" and tensor.dtype is not _ELEM_DTYPES[stored.weight.elem]:
+ raise ValueError(f"{name} is {tensor.dtype} but the checkpoint's quant config declares {module} {stored}")
target, idx, count = self.target(module)
- _, parts, expected = self.pending.setdefault(target, (count, {}, {}))
+ _, parts, expected, _ = self.pending.setdefault(target, (count, {}, {}, stored))
parts.setdefault(idx, {})[role] = tensor
expected[idx] = set(roles.values())
if len(parts) < count or any(set(parts[i]) != expected[i] for i in parts):
@@ -168,6 +170,20 @@ class _DenseReader:
del self.pending[target]
return self._emit(target, [parts[i] for i in range(count)], stored)
+ def missing(self) -> list[str]:
+ """One line per incomplete module: the roles its parts still lack."""
+ lines = []
+ for target, (count, parts, expected, stored) in sorted(self.pending.items()):
+ lacking = sorted(set().union(*(expected[i] - set(parts[i]) for i in parts)))
+ note = ""
+ if lacking == ["input_scale"]:
+ fix = "declares W4A16_NVFP4 or sets with_input_scale false" if stored is not None and stored.kind is QuantKind.NVFP4 else "sets with_input_scale false"
+ note = f" (an export without activation scales {fix})"
+ if len(parts) < count:
+ lacking.append(f"{count - len(parts)} of {count} fused parts")
+ lines.append(f"{target}: missing {lacking}{note}")
+ return lines
+
def _emit(self, target: str, parts: list[dict[str, torch.Tensor]], stored: QuantScheme | None):
if stored is not None:
parts = [self._check(target, stored, part) for part in parts]
@@ -265,7 +281,9 @@ def _iter_shards(model_path: str, device: torch.device, reader: _DenseReader | N
else:
yield name, tensor
if reader is not None and reader.pending:
- raise ValueError(f"checkpoint is missing tensors of {sorted(reader.pending)}")
+ lines = reader.missing()
+ shown = "\n ".join(lines[:8]) + (f"\n ... {len(lines) - 8} more" if len(lines) > 8 else "")
+ raise ValueError(f"checkpoint is missing tensors the quant config declares for {len(lines)} modules:\n {shown}")
def iter_weights_parallel(
diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.py
index d0666f9d..2396bb64 100644
--- a/python/freetoken/models/register.py
+++ b/python/freetoken/models/register.py
@@ -300,23 +300,13 @@ def _load_attr(module_path: str, attr_name: str) -> Any:
def checkpoint_quant_config(model_path: str, hf_config: Any, spec: ModelSpec):
"""The checkpoint's QuantConfig under the family's naming, or None for GGUF, whose native-quant ops the shared parser does not model yet."""
from freetoken.layers.quantization import NameMap, QuantConfig
- from freetoken.utils.hf import optional_hf_file
if spec.parse_config == "parse_gguf_config":
return None
- # NOTE: ModelOpt exports before 0.41 keep the quantization config only in hf_quant_config.json, and the weight download fetches nothing but the safetensors shards, so this sidecar is fetched on its own.
- hf_quant_config = None
- sidecar = optional_hf_file(model_path, "hf_quant_config.json")
- if sidecar is not None:
- import json
-
- with open(sidecar) as f:
- hf_quant_config = json.load(f)
return QuantConfig.from_hf(
hf_config,
name_map=NameMap(roots=spec.checkpoint_roots, segments=spec.checkpoint_segments, packed=spec.packed_modules_mapping),
unquantized=spec.unquantized_modules,
- hf_quant_config=hf_quant_config,
)
diff --git a/python/freetoken/utils/hf.py b/python/freetoken/utils/hf.py
index a28146e6..b93d3b12 100644
--- a/python/freetoken/utils/hf.py
+++ b/python/freetoken/utils/hf.py
@@ -191,19 +191,48 @@ def _raw_config_json(model_path: str) -> dict:
return json.load(f)
+def sidecar_quantization_config(model_path: str) -> dict | None:
+ """The ``quantization_config`` an old ModelOpt export keeps only in ``hf_quant_config.json``, or None."""
+ sidecar = optional_hf_file(model_path, "hf_quant_config.json")
+ if sidecar is None:
+ return None
+ with open(sidecar, encoding="utf-8") as f:
+ quant = json.load(f).get("quantization")
+ if not isinstance(quant, dict):
+ return None
+ return {"quant_method": "modelopt", **quant}
+
+
+def _merge_sidecar_quantization_config(config: Any, model_path: str) -> None:
+ # config.json wins when it has one; the sidecar is only read for exports that never wrote it there
+ if getattr(config, "quantization_config", None) is not None:
+ return
+ if getattr(getattr(config, "text_config", None), "quantization_config", None) is not None:
+ return
+ quant = sidecar_quantization_config(model_path)
+ if quant is None:
+ return
+ if isinstance(config, RawConfigShim):
+ config._data["quantization_config"] = quant
+ else:
+ config.quantization_config = quant
+
+
@functools.cache
def _load_hf_config(model_path: str) -> Any:
# trust_remote_code: checkpoints that ship a custom config class via ``auto_map``
# (e.g. MiniMax-M2) refuse to load without it. FreeToken only reads config fields
# (parse_config) and never instantiates the checkpoint's modeling code.
try:
- return AutoConfig.from_pretrained(model_path, trust_remote_code=True)
+ config = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
except ValueError as exc:
# Unknown model_type on this transformers version: serve off the raw JSON.
# Anything else (bad path, malformed JSON) stays fatal.
if "model type" not in str(exc):
raise
- return RawConfigShim(_raw_config_json(model_path), _name_or_path=model_path)
+ config = RawConfigShim(_raw_config_json(model_path), _name_or_path=model_path)
+ _merge_sidecar_quantization_config(config, model_path)
+ return config
def cached_load_hf_config(model_path: str) -> PretrainedConfig:
diff --git a/tests/models/test_quant_config.py b/tests/models/test_quant_config.py
index 73aad10c..6ac1b84b 100644
--- a/tests/models/test_quant_config.py
+++ b/tests/models/test_quant_config.py
@@ -43,6 +43,7 @@ from freetoken.layers.quantization.moe import Fp8BlockMoEMethod, Mxfp4MoEMethod,
from freetoken.models import create_model
from freetoken.models.gpt_oss.moe import GptOssMoELayer, GptOssOffloadMoELayer
from freetoken.models.register import checkpoint_quant_config, get_model_spec
+from freetoken.utils.hf import sidecar_quantization_config
from freetoken.utils.torch_utils import torch_dtype
MODELS = "/mnt/nvme/models"
@@ -391,21 +392,15 @@ def test_probed_layers_get_the_method_their_config_says(case: Case, monkeypatch)
# --------------------------------------------------------------------------- config without local weights
-def test_hub_ids_fetch_the_modelopt_sidecar(tmp_path, monkeypatch):
- """An old ModelOpt export keeps its quantization config only in hf_quant_config.json."""
- from huggingface_hub.utils import EntryNotFoundError
+def test_the_modelopt_sidecar_is_folded_into_the_hf_config(tmp_path):
+ """An old ModelOpt export keeps its quantization config only in hf_quant_config.json; the config loader folds it in, so every reader of the config sees it."""
+ from freetoken.utils import cached_load_hf_config
- sidecar = tmp_path / "hf_quant_config.json"
- sidecar.write_text(json.dumps({"producer": {"name": "modelopt", "version": "0.29.0"}, "quantization": {"quant_algo": "FP8", "exclude_modules": ["lm_head"]}}))
-
- def fake_download(repo_id, filename, **_):
- assert repo_id == "org/old-modelopt-fp8"
- if filename == "hf_quant_config.json":
- return str(sidecar)
- raise EntryNotFoundError(f"no {filename}")
-
- monkeypatch.setattr("freetoken.utils.hf.hf_hub_download", fake_download)
- quant = checkpoint_quant_config("org/old-modelopt-fp8", SimpleNamespace(architectures=["Qwen3MoeForCausalLM"]), get_model_spec("Qwen3MoeForCausalLM"))
+ (tmp_path / "config.json").write_text(json.dumps({"model_type": "qwen3_moe", "architectures": ["Qwen3MoeForCausalLM"]}))
+ (tmp_path / "hf_quant_config.json").write_text(json.dumps({"producer": {"name": "modelopt", "version": "0.29.0"}, "quantization": {"quant_algo": "FP8", "exclude_modules": ["lm_head"]}}))
+ hf = cached_load_hf_config(str(tmp_path))
+ assert hf.quantization_config == {"quant_method": "modelopt", "quant_algo": "FP8", "exclude_modules": ["lm_head"]}
+ quant = checkpoint_quant_config(str(tmp_path), hf, get_model_spec("Qwen3MoeForCausalLM"))
assert type(quant) is ModelOptConfig
assert quant.scheme_for("model.layers.0.self_attn.q_proj").kind is QuantKind.FP8_TENSOR
assert quant.scheme_for("lm_head") is None
@@ -427,6 +422,23 @@ def test_compressed_tensors_ignore_names_the_module_alone():
assert quant.scheme_for("model.language_model.layers.0.linear_attn") is None
+_NVFP4_GROUP = {"targets": ["Linear"], "weights": {"num_bits": 4, "type": "float", "group_size": 16}}
+
+
[email protected]("extra, has_input_scale", [
+ ({"config_groups": {"group_0": {**_NVFP4_GROUP, "input_activations": None}}}, False),
+ ({"config_groups": {"group_0": {**_NVFP4_GROUP, "input_activations": {"num_bits": 4, "type": "float", "group_size": 16}}}}, True),
+ ({}, True),
+ ({"with_input_scale": False}, False),
+ ({"config_groups": {"group_0": {**_NVFP4_GROUP, "input_activations": None}}, "with_input_scale": True}, True),
+], ids=["no input_activations", "input_activations", "no config_groups", "with_input_scale false", "with_input_scale true wins"])
+def test_modelopt_nvfp4_reads_the_activation_quantizer_from_config_groups(extra, has_input_scale):
+ """A ModelOpt export can say ``NVFP4`` with no activation quantizer: every config group then has ``input_activations`` null (vLLM #54427); an explicit ``with_input_scale`` wins."""
+ q = {"quant_method": "modelopt", "quant_algo": "NVFP4", **extra}
+ scheme = QuantConfig.from_hf(SimpleNamespace(quantization_config=q)).scheme_for("model.layers.0.mlp.down_proj")
+ assert scheme.kind is QuantKind.NVFP4 and scheme.has("input_scale") is has_input_scale
+
+
def test_every_dialect_names_the_tensors_behind_its_schemes():
from freetoken.layers.quantization.registry import dialects
@@ -570,12 +582,13 @@ def test_scheme_for_agrees_with_the_stored_tensors(ckpt: Path):
with pytest.raises(NotImplementedError):
QuantConfig.from_hf(cfg)
return
- hfq = json.load(open(ckpt / "hf_quant_config.json")) if (ckpt / "hf_quant_config.json").exists() else None
+ if not quant:
+ cfg["quantization_config"] = sidecar_quantization_config(str(ckpt))
try:
spec = get_model_spec((cfg.get("architectures") or [""])[0])
except Exception:
spec = None
- qc = QuantConfig.from_hf(cfg, hf_quant_config=hfq, unquantized=spec.unquantized_modules if spec else ())
+ qc = QuantConfig.from_hf(cfg, unquantized=spec.unquantized_modules if spec else ())
weight_map = _weight_map(ckpt)
tensors = _tensor_info(ckpt, weight_map)
mismatches, checked = [], 0
diff --git a/tests/models/test_qwen3_5_moe_weight.py b/tests/models/test_qwen3_5_moe_weight.py
index f5dc2c3c..158f1f45 100644
--- a/tests/models/test_qwen3_5_moe_weight.py
+++ b/tests/models/test_qwen3_5_moe_weight.py
@@ -175,6 +175,10 @@ MODELOPT_MIXED = {
**{f"{LM}.layers.{l}.mlp.experts": {"quant_algo": "NVFP4", "group_size": 16} for l in (0, 1)},
},
}
+# ornith-ai/Ornith-1.5-35B-A3B-NVFP4: the same mix with W4A16 experts
+MODELOPT_MIXED_A16 = {**MODELOPT_MIXED, "quantized_layers": {
+ **MODELOPT_MIXED["quantized_layers"], **{f"{LM}.layers.{l}.mlp.experts": {"quant_algo": "W4A16_NVFP4", "group_size": 16} for l in (0, 1)}}}
+MODELOPT_MIXED_NOINPUT = {**MODELOPT_MIXED, "with_input_scale": False}
# sakamakismile/Qwen3.6-27B-NVFP4: every Linear of the dense model, GDN in_proj included
CT_NVFP4_DENSE = _ct({"group_0": {**NVFP4_GROUP, "targets": ["Linear"]}}, ["lm_head"], "nvfp4-pack-quantized")
# RedHatAI/Qwen3.6-35B-A3B-NVFP4: every Linear but the GDN, the routers and lm_head
@@ -209,11 +213,18 @@ def _layout(name: str) -> tuple[bool, dict | None, dict[str, torch.Tensor]]:
_quantize(raw, GDN_QKVZ_OUT + ATTN + SHARED, lambda w: _fp8_block(w, ct=False))
_experts(raw, lambda w: _fp8_block(w, ct=False))
return moe, QWEN_FP8, raw
- if name == "modelopt_mixed":
+ if name.startswith("modelopt_mixed"):
_quantize(raw, GDN_QKVZ_OUT + ATTN, _fp8_tensor)
_quantize(raw, SHARED + ["lm_head"], lambda w: _nvfp4(w, ct=False))
_experts(raw, lambda w: _nvfp4(w, ct=False))
- return moe, MODELOPT_MIXED, raw
+ # _a16: the W4A16 layers ship no activation scale (ornith-ai); _noinput: no layer does and the config says with_input_scale false (jhone888)
+ if name == "modelopt_mixed_a16":
+ stripped = lambda k: ".mlp." in k or k.startswith("lm_head")
+ else:
+ stripped = lambda k: name == "modelopt_mixed_noinput"
+ for key in [k for k in raw if k.endswith(".input_scale") and stripped(k)]:
+ del raw[key]
+ return moe, {"modelopt_mixed_a16": MODELOPT_MIXED_A16, "modelopt_mixed_noinput": MODELOPT_MIXED_NOINPUT}.get(name, MODELOPT_MIXED), raw
if name == "ct_nvfp4_dense":
_quantize(raw, GDN_QKVZ_OUT + GDN_BA + ATTN + DENSE_MLP, lambda w: _nvfp4(w, ct=True))
return moe, CT_NVFP4_DENSE, raw
@@ -242,7 +253,7 @@ def _layout(name: str) -> tuple[bool, dict | None, dict[str, torch.Tensor]]:
raise KeyError(name)
-LAYOUTS = ["bf16", "fp8_block", "modelopt_mixed", "ct_nvfp4_dense", "ct_nvfp4_moe", "ct_mixed_fast", "ct_tensor_fp8_moe", "ct_block_moe", "ct_block_experts"]
+LAYOUTS = ["bf16", "fp8_block", "modelopt_mixed", "modelopt_mixed_a16", "modelopt_mixed_noinput", "ct_nvfp4_dense", "ct_nvfp4_moe", "ct_mixed_fast", "ct_tensor_fp8_moe", "ct_block_moe", "ct_block_experts"]
def _config_json(moe: bool, quantization_config) -> dict:
@@ -401,25 +412,28 @@ def test_block_fp8_fuses_weight_and_scale_per_kind(checkpoint):
def test_modelopt_fp8_scales_broadcast_per_part_and_input_scale_is_the_max(checkpoint):
name, folder, raw = checkpoint
- if name != "modelopt_mixed":
- pytest.skip("modelopt layout only")
+ if not name.startswith("modelopt_mixed"):
+ pytest.skip("modelopt layouts only")
loaded = _load(folder)
attn = f"{LM}.layers.1.self_attn"
scale = loaded["model.layers.1.self_attn.qkv_proj.weight_scale"]
assert scale.dtype is torch.float32 and scale.shape == (2 * QH * AHD + 2 * KVH * AHD,)
expected = torch.cat([raw[f"{attn}.{p}_proj.weight_scale"].expand(raw[f"{attn}.{p}_proj.weight"].shape[0]) for p in "qkv"])
assert torch.equal(scale, expected)
- assert torch.equal(loaded["model.layers.1.self_attn.qkv_proj.input_scale"], torch.stack([raw[f"{attn}.{p}_proj.input_scale"] for p in "qkv"]).max())
- assert loaded["model.layers.1.self_attn.o_proj.input_scale"].shape == ()
- # NVFP4 shared expert: each part keeps its own block scales and global; the input scale is the max of the parts
+ if name == "modelopt_mixed_noinput":
+ assert not any(k.endswith(".input_scale") for k in loaded)
+ else:
+ assert torch.equal(loaded["model.layers.1.self_attn.qkv_proj.input_scale"], torch.stack([raw[f"{attn}.{p}_proj.input_scale"] for p in "qkv"]).max())
+ assert loaded["model.layers.1.self_attn.o_proj.input_scale"].shape == ()
+ # NVFP4 shared expert: each part keeps its own block scales and global
shared = f"{LM}.layers.0.mlp.shared_expert"
fused = "model.layers.0.mlp.shared_expert.gate_up_proj"
assert _same(loaded[f"{fused}.weight"][:I], raw[f"{shared}.gate_proj.weight"])
assert _same(loaded[f"{fused}.weight_scale"][I:], raw[f"{shared}.up_proj.weight_scale"])
glob = loaded[f"{fused}.weight_global"]
assert glob.dtype is torch.float16 and torch.equal(glob[:I], raw[f"{shared}.gate_proj.weight_scale_2"].to(torch.float16).expand(I))
- assert torch.equal(loaded[f"{fused}.input_scale"], torch.stack([raw[f"{shared}.{p}_proj.input_scale"] for p in ("gate", "up")]).max())
- assert loaded["lm_head.input_scale"].shape == ()
+ # W4A16: no activation quantizer, so the layer has no input_scale even when the export ships one
+ assert f"{fused}.input_scale" not in loaded and "lm_head.input_scale" not in loaded
assert loaded["lm_head.weight"].dtype is torch.uint8 and loaded["lm_head.weight_global"].shape == (V,)
@@ -486,7 +500,7 @@ def test_static_fp8_keeps_the_activation_scale_and_ignored_expert_containers_do_
def test_nvfp4_expert_pieces_read_either_dialect_from_a_single_file(checkpoint):
name, folder, raw = checkpoint
- if name not in ("modelopt_mixed", "ct_nvfp4_moe", "ct_tensor_fp8_moe"):
+ if not name.startswith("modelopt_mixed") and name not in ("ct_nvfp4_moe", "ct_tensor_fp8_moe"):
pytest.skip("NVFP4 expert layouts only")
_install(folder)
config = parse_config(cached_load_hf_config(folder))
@@ -496,7 +510,7 @@ def test_nvfp4_expert_pieces_read_either_dialect_from_a_single_file(checkpoint):
layer, e0, e1, piece = next(p for p in pieces if p[0] == 1 and p[1] == 2)
assert (e0, e1) == (2, 3)
base = f"{LM}.layers.1.mlp.experts.2.gate_proj"
- if name == "modelopt_mixed":
+ if name.startswith("modelopt_mixed"):
assert _same(piece["gate"][0], raw[f"{base}.weight"])
assert torch.equal(piece["gate_global"].reshape(-1), raw[f"{base}.weight_scale_2"].reshape(-1).to(torch.float16))
else:
@@ -559,13 +573,18 @@ REJECTED = [
pytest.param(None, lambda w: {f"{ATTN1}.o_proj.weight": w.to(FP8)},
r"o_proj\.weight is torch\.float8", id="fp8 weight the config declares bf16"),
pytest.param(MODELOPT_MIXED, lambda w: {f"{ATTN1}.o_proj.weight": w, f"{ATTN1}.o_proj.weight_scale": torch.rand(()), f"{ATTN1}.o_proj.input_scale": torch.rand(())},
- r"o_proj: weight is torch\.bfloat16", id="bf16 weight the config declares fp8"),
+ r"o_proj\.weight is torch\.bfloat16 but .*declares .*fp8_tensor", id="bf16 weight the config declares fp8"),
+ pytest.param({"quant_method": "modelopt", "quant_algo": "W4A16_NVFP4"}, lambda w: {f"{ATTN1}.o_proj.weight": w},
+ r"o_proj\.weight is torch\.bfloat16 but .*declares .*nvfp4", id="bf16 weight the config declares nvfp4"),
+ pytest.param({"quant_method": "modelopt", "quant_algo": "NVFP4"},
+ lambda w: {f"{ATTN1}.o_proj.weight": torch.zeros(H, QH * AHD // 2, dtype=torch.uint8), f"{ATTN1}.o_proj.weight_scale": torch.zeros(H, QH * AHD // 16, dtype=FP8), f"{ATTN1}.o_proj.weight_scale_2": torch.rand(()), f"{ATTN1}.k_proj.input_scale": torch.rand(())},
+ r"(?s)o_proj: missing \['input_scale'\] \(an export without activation scales declares W4A16_NVFP4", id="w4a4 config with one input_scale missing"),
pytest.param(CT_MIXED_FAST, lambda w: {f"{ATTN1}.o_proj.weight": w.to(FP8), f"{ATTN1}.o_proj.weight_scale": torch.rand(3, 1)},
"expected 1 or", id="per-channel scale with the wrong row count"),
pytest.param(CT_BLOCK_MOE, lambda w: {f"{ATTN1}.o_proj.weight": w.to(FP8), f"{ATTN1}.o_proj.weight_scale": torch.rand(2, 1)},
r"weight_scale_inv is \(2, 1\), expected \(1, 2\)", id="block scale of the wrong shape"),
pytest.param(CT_NVFP4_MOE, lambda w: {f"{ATTN1}.o_proj.weight_packed": torch.zeros(H, QH * AHD // 2, dtype=torch.uint8), f"{ATTN1}.o_proj.weight_global_scale": torch.rand(1)},
- "missing tensors of .*o_proj", id="quantized module without its block scale"),
+ r"(?s)missing tensors the quant config declares.*o_proj: missing \['input_scale', 'weight_scale'\]", id="quantized module without its block scale"),
]